From 1982d871475ffb8c2c160a96916a76fdcb6eb11c Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sat, 18 Jan 2025 15:37:14 -0500 Subject: [PATCH 0001/1843] Merge commit '3383cfbd3572465febc7a8f816a46304373de46a' into sync-from-portable-simd-2025-01-18 --- .github/workflows/ci.yml | 125 +++++----- .github/workflows/doc.yml | 2 +- .gitignore | 1 + Cargo.toml | 6 + Cross.toml | 2 + crates/core_simd/Cargo.toml | 3 +- crates/core_simd/src/lane_count.rs | 8 +- crates/core_simd/src/lib.rs | 4 +- crates/core_simd/src/masks.rs | 42 ---- crates/core_simd/src/masks/bitmask.rs | 17 -- crates/core_simd/src/masks/full_masks.rs | 58 +---- crates/core_simd/src/ops.rs | 25 +- crates/core_simd/src/simd/cmp/eq.rs | 2 +- crates/core_simd/src/simd/num/float.rs | 30 ++- crates/core_simd/src/simd/num/int.rs | 43 +++- crates/core_simd/src/simd/num/uint.rs | 42 +++- crates/core_simd/src/simd/ptr/const_ptr.rs | 21 ++ crates/core_simd/src/simd/ptr/mut_ptr.rs | 21 ++ crates/core_simd/src/swizzle.rs | 257 ++++++++++++++++++++- crates/core_simd/src/swizzle_dyn.rs | 59 ++++- crates/core_simd/src/vector.rs | 43 +++- crates/core_simd/src/vendor.rs | 3 + crates/core_simd/src/vendor/loongarch64.rs | 31 +++ crates/core_simd/tests/layout.rs | 35 +++ crates/core_simd/tests/masks.rs | 43 ---- crates/core_simd/tests/ops_macros.rs | 38 +++ crates/core_simd/tests/swizzle.rs | 18 ++ crates/test_helpers/Cargo.toml | 3 - crates/test_helpers/src/lib.rs | 216 ++++++++--------- rust-toolchain.toml | 3 + subtree-sync.sh | 52 +++++ 31 files changed, 865 insertions(+), 388 deletions(-) create mode 100644 Cross.toml create mode 100644 crates/core_simd/src/vendor/loongarch64.rs create mode 100644 crates/core_simd/tests/layout.rs create mode 100644 rust-toolchain.toml create mode 100755 subtree-sync.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b292be2d6f99..3984d8f0d8d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: env: CARGO_NET_RETRY: 10 RUSTUP_MAX_RETRIES: 10 + PROPTEST_CASES: 64 jobs: rustfmt: @@ -16,12 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Setup Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - rustup component add rustfmt + - uses: actions/checkout@v4 - name: Run rustfmt run: cargo fmt --all -- --check @@ -37,7 +33,9 @@ jobs: - i686-unknown-linux-gnu - i586-unknown-linux-gnu - aarch64-unknown-linux-gnu + - arm64ec-pc-windows-msvc - armv7-unknown-linux-gnueabihf + - loongarch64-unknown-linux-gnu # non-nightly since https://github.com/rust-lang/rust/pull/113274 # - mips-unknown-linux-gnu # - mips64-unknown-linux-gnuabi64 @@ -49,13 +47,9 @@ jobs: - wasm32-unknown-unknown steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - rustup target add ${{ matrix.target }} - rustup component add clippy + run: rustup target add ${{ matrix.target }} - name: Run Clippy run: cargo clippy --all-targets --target ${{ matrix.target }} @@ -65,26 +59,19 @@ jobs: strategy: fail-fast: false matrix: - target: [x86_64-pc-windows-msvc, i686-pc-windows-msvc, i586-pc-windows-msvc, x86_64-unknown-linux-gnu, x86_64-apple-darwin] + target: [x86_64-pc-windows-msvc, i686-pc-windows-msvc, i586-pc-windows-msvc, x86_64-unknown-linux-gnu] # `default` means we use the default target config for the target, # `native` means we run with `-Ctarget-cpu=native`, and anything else is # an arg to `-Ctarget-feature` target_feature: [default, native, +sse3, +ssse3, +sse4.1, +sse4.2, +avx, +avx2] exclude: - # The macos runners seem to only reliably support up to `avx`. - - { target: x86_64-apple-darwin, target_feature: +avx2 } - # These features are statically known to be present for all 64 bit - # macs, and thus are covered by the `default` test - - { target: x86_64-apple-darwin, target_feature: +sse3 } - - { target: x86_64-apple-darwin, target_feature: +ssse3 } # -Ctarget-cpu=native sounds like bad-news if target != host - { target: i686-pc-windows-msvc, target_feature: native } - { target: i586-pc-windows-msvc, target_feature: native } include: # Populate the `matrix.os` field - - { target: x86_64-apple-darwin, os: macos-latest } - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest } - { target: x86_64-pc-windows-msvc, os: windows-latest } - { target: i686-pc-windows-msvc, os: windows-latest } @@ -98,12 +85,9 @@ jobs: # avx512vl, but occasionally doesn't. Maybe one day we can enable it. steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - rustup target add ${{ matrix.target }} + run: rustup target add ${{ matrix.target }} - name: Configure RUSTFLAGS shell: bash @@ -145,6 +129,35 @@ jobs: run: cargo doc --verbose --target=${{ matrix.target }} env: RUSTDOCFLAGS: -Dwarnings + + macos-tests: + name: ${{ matrix.target }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + target: + - aarch64-apple-darwin + - x86_64-apple-darwin + steps: + - uses: actions/checkout@v4 + - name: Setup Rust + run: rustup target add ${{ matrix.target }} + + - name: Configure RUSTFLAGS + shell: bash + run: echo "RUSTFLAGS=-Dwarnings" >> $GITHUB_ENV + + - name: Test (debug) + run: cargo test --verbose --target=${{ matrix.target }} + + - name: Test (release) + run: cargo test --verbose --target=${{ matrix.target }} --release + + - name: Generate docs + run: cargo doc --verbose --target=${{ matrix.target }} + env: + RUSTDOCFLAGS: -Dwarnings wasm-tests: name: "wasm (firefox, ${{ matrix.name }})" @@ -155,11 +168,7 @@ jobs: - { name: default, RUSTFLAGS: "" } - { name: simd128, RUSTFLAGS: "-C target-feature=+simd128" } steps: - - uses: actions/checkout@v2 - - name: Setup Rust - run: | - rustup update nightly --no-self-update - rustup default nightly + - uses: actions/checkout@v4 - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: Test (debug) @@ -174,6 +183,8 @@ jobs: cross-tests: name: "${{ matrix.target_feature }} on ${{ matrix.target }} (via cross)" runs-on: ubuntu-latest + env: + PROPTEST_CASES: 16 strategy: fail-fast: false @@ -185,6 +196,7 @@ jobs: - powerpc-unknown-linux-gnu - powerpc64le-unknown-linux-gnu # includes altivec by default - riscv64gc-unknown-linux-gnu + - loongarch64-unknown-linux-gnu # MIPS uses a nonstandard binary representation for NaNs which makes it worth testing # non-nightly since https://github.com/rust-lang/rust/pull/113274 # - mips-unknown-linux-gnu @@ -201,24 +213,14 @@ jobs: # - { target: riscv64gc-unknown-linux-gnu, target_feature: "+v,+zvl128b" } steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - rustup target add ${{ matrix.target }} - rustup component add rust-src + run: rustup target add ${{ matrix.target }} - name: Install Cross - # Equivalent to `cargo install cross`, but downloading a prebuilt - # binary. Ideally we wouldn't hardcode a version, but the version number - # being part of the tarball means we can't just use the download/latest - # URL :( + # Install the latest git version for newer targets. run: | - CROSS_URL=https://github.com/cross-rs/cross/releases/download/v0.2.5/cross-x86_64-unknown-linux-gnu.tar.gz - mkdir -p "$HOME/.bin" - curl -sfSL --retry-delay 10 --retry 5 "${CROSS_URL}" | tar zxf - -C "$HOME/.bin" - echo "$HOME/.bin" >> $GITHUB_PATH + cargo install cross --git https://github.com/cross-rs/cross --rev 4090beca3cfffa44371a5bba524de3a578aa46c3 - name: Configure Emulated CPUs run: | @@ -242,34 +244,11 @@ jobs: - name: Test (release) run: cross test --verbose --target=${{ matrix.target }} --release - features: - name: "Test cargo features (${{ matrix.simd }} × ${{ matrix.features }})" + miri: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - simd: - - "" - - "avx512" - features: - - "" - - "--features std" - - "--features all_lane_counts" - - "--all-features" - + env: + PROPTEST_CASES: 16 steps: - - uses: actions/checkout@v2 - - name: Setup Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - - name: Detect AVX512 - run: echo "CPU_FEATURE=$(lscpu | grep -o avx512[a-z]* | sed s/avx/+avx/ | tr '\n' ',' )" >> $GITHUB_ENV - - name: Check build - if: ${{ matrix.simd == '' }} - run: RUSTFLAGS="-Dwarnings" cargo test --all-targets --no-default-features ${{ matrix.features }} - - name: Check AVX - if: ${{ matrix.simd == 'avx512' && contains(env.CPU_FEATURE, 'avx512') }} - run: | - echo "Found AVX features: $CPU_FEATURE" - RUSTFLAGS="-Dwarnings -Ctarget-feature=$CPU_FEATURE" cargo test --all-targets --no-default-features ${{ matrix.features }} + - uses: actions/checkout@v4 + - name: Test (Miri) + run: cargo miri test diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 9d1fa66ccb59..22c2cb3f67f1 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Setup Rust run: | diff --git a/.gitignore b/.gitignore index ea8c4bf7f35f..9673e52dcadb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +git-subtree.sh diff --git a/Cargo.toml b/Cargo.toml index d1732aaec2f9..21d4584a9f4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,3 +5,9 @@ members = [ "crates/std_float", "crates/test_helpers", ] + +[profile.test.package."*"] +opt-level = 2 + +[profile.test.package.test_helpers] +opt-level = 2 diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 000000000000..d21e76b92dd1 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,2 @@ +[build.env] +passthrough = ["PROPTEST_CASES"] diff --git a/crates/core_simd/Cargo.toml b/crates/core_simd/Cargo.toml index b4a8fd70f4c0..a7a6d43b11d3 100644 --- a/crates/core_simd/Cargo.toml +++ b/crates/core_simd/Cargo.toml @@ -9,10 +9,9 @@ categories = ["hardware-support", "no-std"] license = "MIT OR Apache-2.0" [features] -default = ["as_crate"] +default = ["as_crate", "std"] as_crate = [] std = [] -all_lane_counts = [] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen = "0.2" diff --git a/crates/core_simd/src/lane_count.rs b/crates/core_simd/src/lane_count.rs index 4cd7265ed671..280b27bc9bc6 100644 --- a/crates/core_simd/src/lane_count.rs +++ b/crates/core_simd/src/lane_count.rs @@ -33,10 +33,8 @@ macro_rules! supported_lane_count { }; } -supported_lane_count!(1, 2, 4, 8, 16, 32, 64); -#[cfg(feature = "all_lane_counts")] supported_lane_count!( - 3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63 + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 ); diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 992a7705e3c5..7f57847c9c23 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -1,7 +1,6 @@ #![no_std] #![feature( - const_refs_to_cell, - const_mut_refs, + const_eval_select, convert_float_to_int, core_intrinsics, decl_macro, @@ -26,6 +25,7 @@ all(target_arch = "arm", target_feature = "v7"), feature(stdarch_arm_neon_intrinsics) )] +#![cfg_attr(target_arch = "loongarch64", feature(stdarch_loongarch))] #![cfg_attr( any(target_arch = "powerpc", target_arch = "powerpc64"), feature(stdarch_powerpc) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 04de3a968276..b763a7c75a5a 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -308,48 +308,6 @@ where Self(mask_impl::Mask::from_bitmask_integer(bitmask)) } - /// Creates a bitmask vector from a mask. - /// - /// Each bit is set if the corresponding element in the mask is `true`. - /// The remaining bits are unset. - /// - /// The bits are packed into the first N bits of the vector: - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::mask32x8; - /// let mask = mask32x8::from_array([true, false, true, false, false, false, true, false]); - /// assert_eq!(mask.to_bitmask_vector()[0], 0b01000101); - /// ``` - #[inline] - #[must_use = "method returns a new integer and does not mutate the original value"] - pub fn to_bitmask_vector(self) -> Simd { - self.0.to_bitmask_vector() - } - - /// Creates a mask from a bitmask vector. - /// - /// For each bit, if it is set, the corresponding element in the mask is set to `true`. - /// - /// The bits are packed into the first N bits of the vector: - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{mask32x8, u8x8}; - /// let bitmask = u8x8::from_array([0b01000101, 0, 0, 0, 0, 0, 0, 0]); - /// assert_eq!( - /// mask32x8::from_bitmask_vector(bitmask), - /// mask32x8::from_array([true, false, true, false, false, false, true, false]), - /// ); - /// ``` - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn from_bitmask_vector(bitmask: Simd) -> Self { - Self(mask_impl::Mask::from_bitmask_vector(bitmask)) - } - /// Finds the index of the first set element. /// /// ``` diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 96c553426ee7..db4312d5bf88 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -122,23 +122,6 @@ where unsafe { Self(core::intrinsics::simd::simd_bitmask(value), PhantomData) } } - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_bitmask_vector(self) -> Simd { - let mut bitmask = Simd::splat(0); - bitmask.as_mut_array()[..self.0.as_ref().len()].copy_from_slice(self.0.as_ref()); - bitmask - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn from_bitmask_vector(bitmask: Simd) -> Self { - let mut bytes = as SupportedLaneCount>::BitMask::default(); - let len = bytes.as_ref().len(); - bytes.as_mut().copy_from_slice(&bitmask.as_array()[..len]); - Self(bytes, PhantomData) - } - #[inline] pub fn to_bitmask_integer(self) -> u64 { let mut bitmask = [0u8; 8]; diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 87f031a9f367..2d01946b5747 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -140,62 +140,6 @@ where unsafe { Mask(core::intrinsics::simd::simd_cast(self.0)) } } - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_bitmask_vector(self) -> Simd { - let mut bitmask = Simd::splat(0); - - // Safety: Bytes is the right size array - unsafe { - // Compute the bitmask - let mut bytes: as SupportedLaneCount>::BitMask = - core::intrinsics::simd::simd_bitmask(self.0); - - // LLVM assumes bit order should match endianness - if cfg!(target_endian = "big") { - for x in bytes.as_mut() { - *x = x.reverse_bits() - } - if N % 8 > 0 { - bytes.as_mut()[N / 8] >>= 8 - N % 8; - } - } - - bitmask.as_mut_array()[..bytes.as_ref().len()].copy_from_slice(bytes.as_ref()); - } - - bitmask - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn from_bitmask_vector(bitmask: Simd) -> Self { - let mut bytes = as SupportedLaneCount>::BitMask::default(); - - // Safety: Bytes is the right size array - unsafe { - let len = bytes.as_ref().len(); - bytes.as_mut().copy_from_slice(&bitmask.as_array()[..len]); - - // LLVM assumes bit order should match endianness - if cfg!(target_endian = "big") { - for x in bytes.as_mut() { - *x = x.reverse_bits(); - } - if N % 8 > 0 { - bytes.as_mut()[N / 8] >>= 8 - N % 8; - } - } - - // Compute the regular mask - Self::from_int_unchecked(core::intrinsics::simd::simd_select_bitmask( - bytes, - Self::splat(true).to_int(), - Self::splat(false).to_int(), - )) - } - } - #[inline] unsafe fn to_bitmask_impl(self) -> U where @@ -283,7 +227,7 @@ where } #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] + #[must_use = "method returns a new bool and does not mutate the original value"] pub fn all(self) -> bool { // Safety: use `self` as an integer vector unsafe { core::intrinsics::simd::simd_reduce_all(self.to_int()) } diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index dd7303a97b19..d3bd14a34027 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -77,7 +77,7 @@ macro_rules! int_divrem_guard { ( $lhs:ident, $rhs:ident, { const PANIC_ZERO: &'static str = $zero:literal; - $simd_call:ident + $simd_call:ident, $op:tt }, $int:ident ) => { if $rhs.simd_eq(Simd::splat(0 as _)).any() { @@ -96,8 +96,23 @@ macro_rules! int_divrem_guard { // Nice base case to make it easy to const-fold away the other branch. $rhs }; - // Safety: $lhs and rhs are vectors - unsafe { core::intrinsics::simd::$simd_call($lhs, rhs) } + + // aarch64 div fails for arbitrary `v % 0`, mod fails when rhs is MIN, for non-powers-of-two + // these operations aren't vectorized on aarch64 anyway + #[cfg(target_arch = "aarch64")] + { + let mut out = Simd::splat(0 as _); + for i in 0..Self::LEN { + out[i] = $lhs[i] $op rhs[i]; + } + out + } + + #[cfg(not(target_arch = "aarch64"))] + { + // Safety: $lhs and rhs are vectors + unsafe { core::intrinsics::simd::$simd_call($lhs, rhs) } + } } }; } @@ -205,14 +220,14 @@ for_base_ops! { impl Div::div { int_divrem_guard { const PANIC_ZERO: &'static str = "attempt to divide by zero"; - simd_div + simd_div, / } } impl Rem::rem { int_divrem_guard { const PANIC_ZERO: &'static str = "attempt to calculate the remainder with a divisor of zero"; - simd_rem + simd_rem, % } } diff --git a/crates/core_simd/src/simd/cmp/eq.rs b/crates/core_simd/src/simd/cmp/eq.rs index 5b4615ce51d7..93989ce91b89 100644 --- a/crates/core_simd/src/simd/cmp/eq.rs +++ b/crates/core_simd/src/simd/cmp/eq.rs @@ -12,7 +12,7 @@ pub trait SimdPartialEq { #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_eq(self, other: Self) -> Self::Mask; - /// Test if each element is equal to the corresponding element in `other`. + /// Test if each element is not equal to the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_ne(self, other: Self) -> Self::Mask; } diff --git a/crates/core_simd/src/simd/num/float.rs b/crates/core_simd/src/simd/num/float.rs index 59e43851ea8d..79954b937b39 100644 --- a/crates/core_simd/src/simd/num/float.rs +++ b/crates/core_simd/src/simd/num/float.rs @@ -255,6 +255,7 @@ macro_rules! impl_trait { type Bits = Simd<$bits_ty, N>; type Cast = Simd; + #[cfg(not(target_arch = "aarch64"))] #[inline] fn cast(self) -> Self::Cast { @@ -262,6 +263,33 @@ macro_rules! impl_trait { unsafe { core::intrinsics::simd::simd_as(self) } } + // https://github.com/llvm/llvm-project/issues/94694 + #[cfg(target_arch = "aarch64")] + #[inline] + fn cast(self) -> Self::Cast + { + const { assert!(N <= 64) }; + if N <= 2 || N == 4 || N == 8 || N == 16 || N == 32 || N == 64 { + // Safety: supported types are guaranteed by SimdCast + unsafe { core::intrinsics::simd::simd_as(self) } + } else if N < 4 { + let x = self.resize::<4>(Default::default()).cast(); + x.resize::(x[0]) + } else if N < 8 { + let x = self.resize::<8>(Default::default()).cast(); + x.resize::(x[0]) + } else if N < 16 { + let x = self.resize::<16>(Default::default()).cast(); + x.resize::(x[0]) + } else if N < 32 { + let x = self.resize::<32>(Default::default()).cast(); + x.resize::(x[0]) + } else { + let x = self.resize::<64>(Default::default()).cast(); + x.resize::(x[0]) + } + } + #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces unsafe fn to_int_unchecked(self) -> Self::Cast @@ -391,7 +419,7 @@ macro_rules! impl_trait { self.as_array().iter().sum() } else { // Safety: `self` is a float vector - unsafe { core::intrinsics::simd::simd_reduce_add_ordered(self, 0.) } + unsafe { core::intrinsics::simd::simd_reduce_add_ordered(self, -0.) } } } diff --git a/crates/core_simd/src/simd/num/int.rs b/crates/core_simd/src/simd/num/int.rs index d7598d9ceaf9..3a51235ff954 100644 --- a/crates/core_simd/src/simd/num/int.rs +++ b/crates/core_simd/src/simd/num/int.rs @@ -1,6 +1,6 @@ use super::sealed::Sealed; use crate::simd::{ - cmp::SimdPartialOrd, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement, + cmp::SimdOrd, cmp::SimdPartialOrd, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, }; @@ -70,11 +70,27 @@ pub trait SimdInt: Copy + Sealed { /// # #[cfg(not(feature = "as_crate"))] use core::simd; /// # use simd::prelude::*; /// use core::i32::{MIN, MAX}; - /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); + /// let xs = Simd::from_array([MIN, MIN + 1, -5, 0]); /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); /// ``` fn abs(self) -> Self; + /// Lanewise absolute difference. + /// Every element becomes the absolute difference of `self` and `second`. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::i32::{MIN, MAX}; + /// let a = Simd::from_array([MIN, MAX, 100, -100]); + /// let b = Simd::from_array([MAX, MIN, -80, -120]); + /// assert_eq!(a.abs_diff(b), Simd::from_array([u32::MAX, u32::MAX, 180, 20])); + /// ``` + fn abs_diff(self, second: Self) -> Self::Unsigned; + /// Lanewise saturating absolute value, implemented in Rust. /// As abs(), except the MIN value becomes MAX instead of itself. /// @@ -203,6 +219,12 @@ pub trait SimdInt: Copy + Sealed { /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. fn reverse_bits(self) -> Self; + /// Returns the number of ones in the binary representation of each element. + fn count_ones(self) -> Self::Unsigned; + + /// Returns the number of zeros in the binary representation of each element. + fn count_zeros(self) -> Self::Unsigned; + /// Returns the number of leading zeros in the binary representation of each element. fn leading_zeros(self) -> Self::Unsigned; @@ -259,6 +281,13 @@ macro_rules! impl_trait { (self^m) - m } + #[inline] + fn abs_diff(self, second: Self) -> Self::Unsigned { + let max = self.simd_max(second); + let min = self.simd_min(second); + (max - min).cast() + } + #[inline] fn saturating_abs(self) -> Self { // arith shift for -1 or 0 mask based on sign bit, giving 2s complement @@ -344,6 +373,16 @@ macro_rules! impl_trait { unsafe { core::intrinsics::simd::simd_bitreverse(self) } } + #[inline] + fn count_ones(self) -> Self::Unsigned { + self.cast::<$unsigned>().count_ones() + } + + #[inline] + fn count_zeros(self) -> Self::Unsigned { + self.cast::<$unsigned>().count_zeros() + } + #[inline] fn leading_zeros(self) -> Self::Unsigned { self.cast::<$unsigned>().leading_zeros() diff --git a/crates/core_simd/src/simd/num/uint.rs b/crates/core_simd/src/simd/num/uint.rs index 53dd97f501c6..1ab2d8c7b731 100644 --- a/crates/core_simd/src/simd/num/uint.rs +++ b/crates/core_simd/src/simd/num/uint.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount}; +use crate::simd::{cmp::SimdOrd, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount}; /// Operations on SIMD vectors of unsigned integers. pub trait SimdUint: Copy + Sealed { @@ -57,6 +57,22 @@ pub trait SimdUint: Copy + Sealed { /// assert_eq!(sat, Simd::splat(0)); fn saturating_sub(self, second: Self) -> Self; + /// Lanewise absolute difference. + /// Every element becomes the absolute difference of `self` and `second`. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::u32::MAX; + /// let a = Simd::from_array([0, MAX, 100, 20]); + /// let b = Simd::from_array([MAX, 0, 80, 200]); + /// assert_eq!(a.abs_diff(b), Simd::from_array([MAX, MAX, 20, 180])); + /// ``` + fn abs_diff(self, second: Self) -> Self; + /// Returns the sum of the elements of the vector, with wrapping addition. fn reduce_sum(self) -> Self::Scalar; @@ -85,6 +101,12 @@ pub trait SimdUint: Copy + Sealed { /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. fn reverse_bits(self) -> Self; + /// Returns the number of ones in the binary representation of each element. + fn count_ones(self) -> Self; + + /// Returns the number of zeros in the binary representation of each element. + fn count_zeros(self) -> Self; + /// Returns the number of leading zeros in the binary representation of each element. fn leading_zeros(self) -> Self; @@ -138,6 +160,13 @@ macro_rules! impl_trait { unsafe { core::intrinsics::simd::simd_saturating_sub(self, second) } } + #[inline] + fn abs_diff(self, second: Self) -> Self { + let max = self.simd_max(second); + let min = self.simd_min(second); + max - min + } + #[inline] fn reduce_sum(self) -> Self::Scalar { // Safety: `self` is an integer vector @@ -192,6 +221,17 @@ macro_rules! impl_trait { unsafe { core::intrinsics::simd::simd_bitreverse(self) } } + #[inline] + fn count_ones(self) -> Self { + // Safety: `self` is an integer vector + unsafe { core::intrinsics::simd::simd_ctpop(self) } + } + + #[inline] + fn count_zeros(self) -> Self { + (!self).count_ones() + } + #[inline] fn leading_zeros(self) -> Self { // Safety: `self` is an integer vector diff --git a/crates/core_simd/src/simd/ptr/const_ptr.rs b/crates/core_simd/src/simd/ptr/const_ptr.rs index be635ea640b8..47383809ffba 100644 --- a/crates/core_simd/src/simd/ptr/const_ptr.rs +++ b/crates/core_simd/src/simd/ptr/const_ptr.rs @@ -42,6 +42,19 @@ pub trait SimdConstPtr: Copy + Sealed { /// Equivalent to calling [`pointer::addr`] on each element. fn addr(self) -> Self::Usize; + /// Converts an address to a pointer without giving it any provenance. + /// + /// Without provenance, this pointer is not associated with any actual allocation. Such a + /// no-provenance pointer may be used for zero-sized memory accesses (if suitably aligned), but + /// non-zero-sized memory accesses with a no-provenance pointer are UB. No-provenance pointers + /// are little more than a usize address in disguise. + /// + /// This is different from [`Self::with_exposed_provenance`], which creates a pointer that picks up a + /// previously exposed provenance. + /// + /// Equivalent to calling [`core::ptr::without_provenance`] on each element. + fn without_provenance(addr: Self::Usize) -> Self; + /// Creates a new pointer with the given address. /// /// This performs the same operation as a cast, but copies the *address-space* and @@ -118,6 +131,14 @@ where unsafe { core::mem::transmute_copy(&self) } } + #[inline] + fn without_provenance(addr: Self::Usize) -> Self { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Integer-to-pointer transmutes are valid (if you are okay with not getting any + // provenance). + unsafe { core::mem::transmute_copy(&addr) } + } + #[inline] fn with_addr(self, addr: Self::Usize) -> Self { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. diff --git a/crates/core_simd/src/simd/ptr/mut_ptr.rs b/crates/core_simd/src/simd/ptr/mut_ptr.rs index f6823a949e32..3f20eef21a31 100644 --- a/crates/core_simd/src/simd/ptr/mut_ptr.rs +++ b/crates/core_simd/src/simd/ptr/mut_ptr.rs @@ -39,6 +39,19 @@ pub trait SimdMutPtr: Copy + Sealed { /// Equivalent to calling [`pointer::addr`] on each element. fn addr(self) -> Self::Usize; + /// Converts an address to a pointer without giving it any provenance. + /// + /// Without provenance, this pointer is not associated with any actual allocation. Such a + /// no-provenance pointer may be used for zero-sized memory accesses (if suitably aligned), but + /// non-zero-sized memory accesses with a no-provenance pointer are UB. No-provenance pointers + /// are little more than a usize address in disguise. + /// + /// This is different from [`Self::with_exposed_provenance`], which creates a pointer that picks up a + /// previously exposed provenance. + /// + /// Equivalent to calling [`core::ptr::without_provenance`] on each element. + fn without_provenance(addr: Self::Usize) -> Self; + /// Creates a new pointer with the given address. /// /// This performs the same operation as a cast, but copies the *address-space* and @@ -115,6 +128,14 @@ where unsafe { core::mem::transmute_copy(&self) } } + #[inline] + fn without_provenance(addr: Self::Usize) -> Self { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Integer-to-pointer transmutes are valid (if you are okay with not getting any + // provenance). + unsafe { core::mem::transmute_copy(&addr) } + } + #[inline] fn with_addr(self, addr: Self::Usize) -> Self { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index d62642fb9061..42425ef37e50 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -155,8 +155,7 @@ pub trait Swizzle { /// Creates a new mask from the elements of `mask`. /// - /// Element `i` of the output is `concat[Self::INDEX[i]]`, where `concat` is the concatenation of - /// `first` and `second`. + /// Element `i` of the output is `mask[Self::INDEX[i]]`. #[inline] #[must_use = "method returns a new mask and does not mutate the original inputs"] fn swizzle_mask(mask: Mask) -> Mask @@ -260,6 +259,50 @@ where Rotate::::swizzle(self) } + /// Shifts the vector elements to the left by `OFFSET`, filling in with + /// `padding` from the right. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn shift_elements_left(self, padding: T) -> Self { + struct Shift; + + impl Swizzle for Shift { + const INDEX: [usize; N] = const { + let mut index = [N; N]; + let mut i = 0; + while i + OFFSET < N { + index[i] = i + OFFSET; + i += 1; + } + index + }; + } + + Shift::::concat_swizzle(self, Simd::splat(padding)) + } + + /// Shifts the vector elements to the right by `OFFSET`, filling in with + /// `padding` from the left. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn shift_elements_right(self, padding: T) -> Self { + struct Shift; + + impl Swizzle for Shift { + const INDEX: [usize; N] = const { + let mut index = [N; N]; + let mut i = OFFSET; + while i < N { + index[i] = i - OFFSET; + i += 1; + } + index + }; + } + + Shift::::concat_swizzle(self, Simd::splat(padding)) + } + /// Interleave two vectors. /// /// The resulting vectors contain elements taken alternatively from `self` and `other`, first @@ -320,7 +363,9 @@ where /// /// ``` /// # #![feature(portable_simd)] - /// # use core::simd::Simd; + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::Simd; /// let a = Simd::from_array([0, 4, 1, 5]); /// let b = Simd::from_array([2, 6, 3, 7]); /// let (x, y) = a.deinterleave(b); @@ -391,4 +436,210 @@ where } Resize::::concat_swizzle(self, Simd::splat(value)) } + + /// Extract a vector from another vector. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::u32x4; + /// let x = u32x4::from_array([0, 1, 2, 3]); + /// assert_eq!(x.extract::<1, 2>().to_array(), [1, 2]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn extract(self) -> Simd + where + LaneCount: SupportedLaneCount, + { + struct Extract; + impl Swizzle for Extract { + const INDEX: [usize; LEN] = const { + assert!(START + LEN <= N, "index out of bounds"); + let mut index = [0; LEN]; + let mut i = 0; + while i < LEN { + index[i] = START + i; + i += 1; + } + index + }; + } + Extract::::swizzle(self) + } +} + +impl Mask +where + T: MaskElement, + LaneCount: SupportedLaneCount, +{ + /// Reverse the order of the elements in the mask. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn reverse(self) -> Self { + // Safety: swizzles are safe for masks + unsafe { Self::from_int_unchecked(self.to_int().reverse()) } + } + + /// Rotates the mask such that the first `OFFSET` elements of the slice move to the end + /// while the last `self.len() - OFFSET` elements move to the front. After calling `rotate_elements_left`, + /// the element previously at index `OFFSET` will become the first element in the slice. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn rotate_elements_left(self) -> Self { + // Safety: swizzles are safe for masks + unsafe { Self::from_int_unchecked(self.to_int().rotate_elements_left::()) } + } + + /// Rotates the mask such that the first `self.len() - OFFSET` elements of the mask move to + /// the end while the last `OFFSET` elements move to the front. After calling `rotate_elements_right`, + /// the element previously at index `self.len() - OFFSET` will become the first element in the slice. + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn rotate_elements_right(self) -> Self { + // Safety: swizzles are safe for masks + unsafe { Self::from_int_unchecked(self.to_int().rotate_elements_right::()) } + } + + /// Shifts the mask elements to the left by `OFFSET`, filling in with + /// `padding` from the right. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original inputs"] + pub fn shift_elements_left(self, padding: bool) -> Self { + // Safety: swizzles are safe for masks + unsafe { + Self::from_int_unchecked(self.to_int().shift_elements_left::(if padding { + T::TRUE + } else { + T::FALSE + })) + } + } + + /// Shifts the mask elements to the right by `OFFSET`, filling in with + /// `padding` from the left. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original inputs"] + pub fn shift_elements_right(self, padding: bool) -> Self { + // Safety: swizzles are safe for masks + unsafe { + Self::from_int_unchecked(self.to_int().shift_elements_right::(if padding { + T::TRUE + } else { + T::FALSE + })) + } + } + + /// Interleave two masks. + /// + /// The resulting masks contain elements taken alternatively from `self` and `other`, first + /// filling the first result, and then the second. + /// + /// The reverse of this operation is [`Mask::deinterleave`]. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::mask32x4; + /// let a = mask32x4::from_array([false, true, false, true]); + /// let b = mask32x4::from_array([false, false, true, true]); + /// let (x, y) = a.interleave(b); + /// assert_eq!(x.to_array(), [false, false, true, false]); + /// assert_eq!(y.to_array(), [false, true, true, true]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn interleave(self, other: Self) -> (Self, Self) { + let (lo, hi) = self.to_int().interleave(other.to_int()); + // Safety: swizzles are safe for masks + unsafe { (Self::from_int_unchecked(lo), Self::from_int_unchecked(hi)) } + } + + /// Deinterleave two masks. + /// + /// The first result takes every other element of `self` and then `other`, starting with + /// the first element. + /// + /// The second result takes every other element of `self` and then `other`, starting with + /// the second element. + /// + /// The reverse of this operation is [`Mask::interleave`]. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::mask32x4; + /// let a = mask32x4::from_array([false, true, false, true]); + /// let b = mask32x4::from_array([false, false, true, true]); + /// let (x, y) = a.deinterleave(b); + /// assert_eq!(x.to_array(), [false, false, false, true]); + /// assert_eq!(y.to_array(), [true, true, false, true]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn deinterleave(self, other: Self) -> (Self, Self) { + let (even, odd) = self.to_int().deinterleave(other.to_int()); + // Safety: swizzles are safe for masks + unsafe { + ( + Self::from_int_unchecked(even), + Self::from_int_unchecked(odd), + ) + } + } + + /// Resize a mask. + /// + /// If `M` > `N`, extends the length of a mask, setting the new elements to `value`. + /// If `M` < `N`, truncates the mask to the first `M` elements. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::mask32x4; + /// let x = mask32x4::from_array([false, true, true, false]); + /// assert_eq!(x.resize::<8>(true).to_array(), [false, true, true, false, true, true, true, true]); + /// assert_eq!(x.resize::<2>(true).to_array(), [false, true]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn resize(self, value: bool) -> Mask + where + LaneCount: SupportedLaneCount, + { + // Safety: swizzles are safe for masks + unsafe { + Mask::::from_int_unchecked(self.to_int().resize::(if value { + T::TRUE + } else { + T::FALSE + })) + } + } + + /// Extract a vector from another vector. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::mask32x4; + /// let x = mask32x4::from_array([false, true, true, false]); + /// assert_eq!(x.extract::<1, 2>().to_array(), [true, true]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn extract(self) -> Mask + where + LaneCount: SupportedLaneCount, + { + // Safety: swizzles are safe for masks + unsafe { Mask::::from_int_unchecked(self.to_int().extract::()) } + } } diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 3b6388d0f275..773bd028bae0 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -59,15 +59,40 @@ where target_endian = "little" ))] 16 => transize(vqtbl1q_u8, self, idxs), + #[cfg(all( + target_arch = "arm", + target_feature = "v7", + target_feature = "neon", + target_endian = "little" + ))] + 16 => transize(armv7_neon_swizzle_u8x16, self, idxs), #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] 32 => transize(avx2_pshufb, self, idxs), #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] - 32 => transize(x86::_mm256_permutexvar_epi8, zeroing_idxs(idxs), self), - // Notable absence: avx512bw shuffle - // If avx512bw is available, odds of avx512vbmi are good - // FIXME: initial AVX512VBMI variant didn't actually pass muster - // #[cfg(target_feature = "avx512vbmi")] - // 64 => transize(x86::_mm512_permutexvar_epi8, self, idxs), + 32 => { + // Unlike vpshufb, vpermb doesn't zero out values in the result based on the index high bit + let swizzler = |bytes, idxs| { + let mask = x86::_mm256_cmp_epu8_mask::<{ x86::_MM_CMPINT_LT }>( + idxs, + Simd::::splat(N as u8).into(), + ); + x86::_mm256_maskz_permutexvar_epi8(mask, idxs, bytes) + }; + transize(swizzler, self, idxs) + } + // Notable absence: avx512bw pshufb shuffle + #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))] + 64 => { + // Unlike vpshufb, vpermb doesn't zero out values in the result based on the index high bit + let swizzler = |bytes, idxs| { + let mask = x86::_mm512_cmp_epu8_mask::<{ x86::_MM_CMPINT_LT }>( + idxs, + Simd::::splat(N as u8).into(), + ); + x86::_mm512_maskz_permutexvar_epi8(mask, idxs, bytes) + }; + transize(swizzler, self, idxs) + } _ => { let mut array = [0; N]; for (i, k) in idxs.to_array().into_iter().enumerate() { @@ -82,6 +107,28 @@ where } } +/// armv7 neon supports swizzling `u8x16` by swizzling two u8x8 blocks +/// with a u8x8x2 lookup table. +/// +/// # Safety +/// This requires armv7 neon to work +#[cfg(all( + target_arch = "arm", + target_feature = "v7", + target_feature = "neon", + target_endian = "little" +))] +unsafe fn armv7_neon_swizzle_u8x16(bytes: Simd, idxs: Simd) -> Simd { + use core::arch::arm::{uint8x8x2_t, vcombine_u8, vget_high_u8, vget_low_u8, vtbl2_u8}; + // SAFETY: Caller promised arm neon support + unsafe { + let bytes = uint8x8x2_t(vget_low_u8(bytes.into()), vget_high_u8(bytes.into())); + let lo = vtbl2_u8(bytes, vget_low_u8(idxs.into())); + let hi = vtbl2_u8(bytes, vget_high_u8(idxs.into())); + vcombine_u8(lo, hi).into() + } +} + /// "vpshufb like it was meant to be" on AVX2 /// /// # Safety diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 3e2391691496..9c4dd36c24fe 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -99,7 +99,7 @@ use crate::simd::{ // directly constructing an instance of the type (i.e. `let vector = Simd(array)`) should be // avoided, as it will likely become illegal on `#[repr(simd)]` structs in the future. It also // causes rustc to emit illegal LLVM IR in some cases. -#[repr(simd)] +#[repr(simd, packed)] pub struct Simd([T; N]) where LaneCount: SupportedLaneCount, @@ -144,14 +144,32 @@ where /// assert_eq!(v.as_array(), &[8, 8, 8, 8]); /// ``` #[inline] - pub fn splat(value: T) -> Self { - // This is preferred over `[value; N]`, since it's explicitly a splat: - // https://github.com/rust-lang/rust/issues/97804 - struct Splat; - impl Swizzle for Splat { - const INDEX: [usize; N] = [0; N]; + #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] + pub const fn splat(value: T) -> Self { + const fn splat_const(value: T) -> Simd + where + T: SimdElement, + LaneCount: SupportedLaneCount, + { + Simd::from_array([value; N]) } - Splat::swizzle::(Simd::::from([value])) + + fn splat_rt(value: T) -> Simd + where + T: SimdElement, + LaneCount: SupportedLaneCount, + { + // This is preferred over `[value; N]`, since it's explicitly a splat: + // https://github.com/rust-lang/rust/issues/97804 + struct Splat; + impl Swizzle for Splat { + const INDEX: [usize; N] = [0; N]; + } + + Splat::swizzle::(Simd::::from([value])) + } + + core::intrinsics::const_eval_select((value,), splat_const, splat_rt) } /// Returns an array reference containing the entire SIMD vector. @@ -425,6 +443,9 @@ where /// /// When the element is disabled, that memory location is not accessed and the corresponding /// value from `or` is passed through. + /// + /// # Safety + /// Enabled loads must not exceed the length of `slice`. #[must_use] #[inline] pub unsafe fn load_select_unchecked( @@ -442,6 +463,9 @@ where /// /// When the element is disabled, that memory location is not accessed and the corresponding /// value from `or` is passed through. + /// + /// # Safety + /// Enabled `ptr` elements must be safe to read as if by `std::ptr::read`. #[must_use] #[inline] pub unsafe fn load_select_ptr( @@ -924,6 +948,7 @@ where } } +/// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead. impl PartialOrd for Simd where LaneCount: SupportedLaneCount, @@ -943,6 +968,7 @@ where { } +/// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead. impl Ord for Simd where LaneCount: SupportedLaneCount, @@ -1195,6 +1221,7 @@ fn lane_indices() -> Simd where LaneCount: SupportedLaneCount, { + #![allow(clippy::needless_range_loop)] let mut index = [0; N]; for i in 0..N { index[i] = i; diff --git a/crates/core_simd/src/vendor.rs b/crates/core_simd/src/vendor.rs index 1a34a3a8de5c..57536e4fc77d 100644 --- a/crates/core_simd/src/vendor.rs +++ b/crates/core_simd/src/vendor.rs @@ -29,3 +29,6 @@ mod arm; #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] mod powerpc; + +#[cfg(target_arch = "loongarch64")] +mod loongarch64; diff --git a/crates/core_simd/src/vendor/loongarch64.rs b/crates/core_simd/src/vendor/loongarch64.rs new file mode 100644 index 000000000000..1290bc166b2b --- /dev/null +++ b/crates/core_simd/src/vendor/loongarch64.rs @@ -0,0 +1,31 @@ +use crate::simd::*; +use core::arch::loongarch64::*; + +from_transmute! { unsafe u8x16 => v16u8 } +from_transmute! { unsafe u8x32 => v32u8 } +from_transmute! { unsafe i8x16 => v16i8 } +from_transmute! { unsafe i8x32 => v32i8 } + +from_transmute! { unsafe u16x8 => v8u16 } +from_transmute! { unsafe u16x16 => v16u16 } +from_transmute! { unsafe i16x8 => v8i16 } +from_transmute! { unsafe i16x16 => v16i16 } + +from_transmute! { unsafe u32x4 => v4u32 } +from_transmute! { unsafe u32x8 => v8u32 } +from_transmute! { unsafe i32x4 => v4i32 } +from_transmute! { unsafe i32x8 => v8i32 } +from_transmute! { unsafe f32x4 => v4f32 } +from_transmute! { unsafe f32x8 => v8f32 } + +from_transmute! { unsafe u64x2 => v2u64 } +from_transmute! { unsafe u64x4 => v4u64 } +from_transmute! { unsafe i64x2 => v2i64 } +from_transmute! { unsafe i64x4 => v4i64 } +from_transmute! { unsafe f64x2 => v2f64 } +from_transmute! { unsafe f64x4 => v4f64 } + +from_transmute! { unsafe usizex2 => v2u64 } +from_transmute! { unsafe usizex4 => v4u64 } +from_transmute! { unsafe isizex2 => v2i64 } +from_transmute! { unsafe isizex4 => v4i64 } diff --git a/crates/core_simd/tests/layout.rs b/crates/core_simd/tests/layout.rs new file mode 100644 index 000000000000..24114c2d261e --- /dev/null +++ b/crates/core_simd/tests/layout.rs @@ -0,0 +1,35 @@ +#![feature(portable_simd)] + +macro_rules! layout_tests { + { $($mod:ident, $ty:ty,)* } => { + $( + mod $mod { + test_helpers::test_lanes! { + fn no_padding() { + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::<[$ty; LANES]>(), + ); + } + } + } + )* + } +} + +layout_tests! { + i8, i8, + i16, i16, + i32, i32, + i64, i64, + isize, isize, + u8, u8, + u16, u16, + u32, u32, + u64, u64, + usize, usize, + f32, f32, + f64, f64, + mut_ptr, *mut (), + const_ptr, *const (), +} diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index fc6a3476b7c6..48786d02440b 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -99,7 +99,6 @@ macro_rules! test_mask_api { assert_eq!(Mask::<$type, 2>::from_bitmask(bitmask), mask); } - #[cfg(feature = "all_lane_counts")] #[test] fn roundtrip_bitmask_conversion_odd() { let values = [ @@ -134,48 +133,6 @@ macro_rules! test_mask_api { cast_impl::(); cast_impl::(); } - - #[test] - fn roundtrip_bitmask_vector_conversion() { - use core_simd::simd::ToBytes; - let values = [ - true, false, false, true, false, false, true, false, - true, true, false, false, false, false, false, true, - ]; - let mask = Mask::<$type, 16>::from_array(values); - let bitmask = mask.to_bitmask_vector(); - assert_eq!(bitmask.resize::<2>(0).to_ne_bytes()[..2], [0b01001001, 0b10000011]); - assert_eq!(Mask::<$type, 16>::from_bitmask_vector(bitmask), mask); - } - - // rust-lang/portable-simd#379 - #[test] - fn roundtrip_bitmask_vector_conversion_small() { - use core_simd::simd::ToBytes; - let values = [ - true, false, true, true - ]; - let mask = Mask::<$type, 4>::from_array(values); - let bitmask = mask.to_bitmask_vector(); - assert_eq!(bitmask.resize::<1>(0).to_ne_bytes()[0], 0b00001101); - assert_eq!(Mask::<$type, 4>::from_bitmask_vector(bitmask), mask); - } - - /* FIXME doesn't work with non-powers-of-two, yet - // rust-lang/portable-simd#379 - #[cfg(feature = "all_lane_counts")] - #[test] - fn roundtrip_bitmask_vector_conversion_odd() { - use core_simd::simd::ToBytes; - let values = [ - true, false, true, false, true, true, false, false, false, true, true, - ]; - let mask = Mask::<$type, 11>::from_array(values); - let bitmask = mask.to_bitmask_vector(); - assert_eq!(bitmask.resize::<2>(0).to_ne_bytes()[..2], [0b00110101, 0b00000110]); - assert_eq!(Mask::<$type, 11>::from_bitmask_vector(bitmask), mask); - } - */ } } } diff --git a/crates/core_simd/tests/ops_macros.rs b/crates/core_simd/tests/ops_macros.rs index aa565a137527..6de78f51e59d 100644 --- a/crates/core_simd/tests/ops_macros.rs +++ b/crates/core_simd/tests/ops_macros.rs @@ -216,6 +216,22 @@ macro_rules! impl_common_integer_tests { ) } + fn count_ones() { + test_helpers::test_unary_elementwise( + &$vector::::count_ones, + &|x| x.count_ones() as _, + &|_| true, + ) + } + + fn count_zeros() { + test_helpers::test_unary_elementwise( + &$vector::::count_zeros, + &|x| x.count_zeros() as _, + &|_| true, + ) + } + fn leading_zeros() { test_helpers::test_unary_elementwise( &$vector::::leading_zeros, @@ -307,6 +323,14 @@ macro_rules! impl_signed_tests { assert_eq!(a % b, Vector::::splat(0)); } + fn abs_diff() { + test_helpers::test_binary_elementwise( + &Vector::::abs_diff, + &Scalar::abs_diff, + &|_, _| true, + ) + } + fn simd_min() { use core_simd::simd::cmp::SimdOrd; let a = Vector::::splat(Scalar::MIN); @@ -419,6 +443,14 @@ macro_rules! impl_unsigned_tests { &|_| true, ); } + + fn abs_diff() { + test_helpers::test_binary_elementwise( + &Vector::::abs_diff, + &Scalar::abs_diff, + &|_, _| true, + ) + } } impl_binary_op_test!(Scalar, Add::add, AddAssign::add_assign, Scalar::wrapping_add); @@ -495,6 +527,9 @@ macro_rules! impl_float_tests { } fn is_normal() { + // Arm v7 Neon violates float opsem re: subnormals, see + // https://github.com/rust-lang/portable-simd/issues/439 + #[cfg(not(target_arch = "arm"))] test_helpers::test_unary_mask_elementwise( &Vector::::is_normal, &Scalar::is_normal, @@ -503,6 +538,9 @@ macro_rules! impl_float_tests { } fn is_subnormal() { + // Arm v7 Neon violates float opsem re: subnormals, see + // https://github.com/rust-lang/portable-simd/issues/439 + #[cfg(not(target_arch = "arm"))] test_helpers::test_unary_mask_elementwise( &Vector::::is_subnormal, &Scalar::is_subnormal, diff --git a/crates/core_simd/tests/swizzle.rs b/crates/core_simd/tests/swizzle.rs index 522d71439b77..7001e5f6bf87 100644 --- a/crates/core_simd/tests/swizzle.rs +++ b/crates/core_simd/tests/swizzle.rs @@ -48,6 +48,24 @@ fn rotate() { assert_eq!(a.rotate_elements_right::<5>().to_array(), [4, 1, 2, 3]); } +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn shift() { + let a = Simd::from_array([1, 2, 3, 4]); + assert_eq!(a.shift_elements_left::<0>(0).to_array(), [1, 2, 3, 4]); + assert_eq!(a.shift_elements_left::<1>(0).to_array(), [2, 3, 4, 0]); + assert_eq!(a.shift_elements_left::<2>(9).to_array(), [3, 4, 9, 9]); + assert_eq!(a.shift_elements_left::<3>(8).to_array(), [4, 8, 8, 8]); + assert_eq!(a.shift_elements_left::<4>(7).to_array(), [7, 7, 7, 7]); + assert_eq!(a.shift_elements_left::<5>(6).to_array(), [6, 6, 6, 6]); + assert_eq!(a.shift_elements_right::<0>(0).to_array(), [1, 2, 3, 4]); + assert_eq!(a.shift_elements_right::<1>(0).to_array(), [0, 1, 2, 3]); + assert_eq!(a.shift_elements_right::<2>(-1).to_array(), [-1, -1, 1, 2]); + assert_eq!(a.shift_elements_right::<3>(-2).to_array(), [-2, -2, -2, 1]); + assert_eq!(a.shift_elements_right::<4>(-3).to_array(), [-3, -3, -3, -3]); + assert_eq!(a.shift_elements_right::<5>(-4).to_array(), [-4, -4, -4, -4]); +} + #[test] #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn interleave() { diff --git a/crates/test_helpers/Cargo.toml b/crates/test_helpers/Cargo.toml index 23dae7c93381..a5359b9abc84 100644 --- a/crates/test_helpers/Cargo.toml +++ b/crates/test_helpers/Cargo.toml @@ -6,6 +6,3 @@ publish = false [dependencies] proptest = { version = "0.10", default-features = false, features = ["alloc"] } - -[features] -all_lane_counts = [] diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 51b860a86356..197c920e11ea 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -539,32 +539,22 @@ macro_rules! test_lanes { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; lanes_1 1; lanes_2 2; - lanes_4 4; - ); - - #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow - $crate::test_lanes_helper!( - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; - lanes_8 8; - lanes_16 16; - lanes_32 32; - lanes_64 64; - ); - - #[cfg(feature = "all_lane_counts")] - $crate::test_lanes_helper!( - // test some odd and even non-power-of-2 lengths on miri - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; + // Cover an odd and an even non-power-of-2 length in Miri. + // (Even non-power-of-2 vectors have alignment between element + // and vector size, so we want to cover that case as well.) lanes_3 3; - lanes_5 5; + lanes_6 6; ); - #[cfg(feature = "all_lane_counts")] #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow $crate::test_lanes_helper!( #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; + lanes_4 4; + lanes_5 5; + lanes_7 7; + lanes_8 8; lanes_9 9; lanes_10 10; lanes_11 11; @@ -572,52 +562,55 @@ macro_rules! test_lanes { lanes_13 13; lanes_14 14; lanes_15 15; + lanes_16 16; lanes_17 17; - lanes_18 18; - lanes_19 19; - lanes_20 20; - lanes_21 21; - lanes_22 22; - lanes_23 23; + //lanes_18 18; + //lanes_19 19; + //lanes_20 20; + //lanes_21 21; + //lanes_22 22; + //lanes_23 23; lanes_24 24; - lanes_25 25; - lanes_26 26; - lanes_27 27; - lanes_28 28; - lanes_29 29; - lanes_30 30; - lanes_31 31; - lanes_33 33; - lanes_34 34; - lanes_35 35; - lanes_36 36; - lanes_37 37; - lanes_38 38; - lanes_39 39; - lanes_40 40; - lanes_41 41; - lanes_42 42; - lanes_43 43; - lanes_44 44; - lanes_45 45; - lanes_46 46; + //lanes_25 25; + //lanes_26 26; + //lanes_27 27; + //lanes_28 28; + //lanes_29 29; + //lanes_30 30; + //lanes_31 31; + lanes_32 32; + //lanes_33 33; + //lanes_34 34; + //lanes_35 35; + //lanes_36 36; + //lanes_37 37; + //lanes_38 38; + //lanes_39 39; + //lanes_40 40; + //lanes_41 41; + //lanes_42 42; + //lanes_43 43; + //lanes_44 44; + //lanes_45 45; + //lanes_46 46; lanes_47 47; - lanes_48 48; - lanes_49 49; - lanes_50 50; - lanes_51 51; - lanes_52 52; - lanes_53 53; - lanes_54 54; - lanes_55 55; + //lanes_48 48; + //lanes_49 49; + //lanes_50 50; + //lanes_51 51; + //lanes_52 52; + //lanes_53 53; + //lanes_54 54; + //lanes_55 55; lanes_56 56; lanes_57 57; - lanes_58 58; - lanes_59 59; - lanes_60 60; - lanes_61 61; - lanes_62 62; + //lanes_58 58; + //lanes_59 59; + //lanes_60 60; + //lanes_61 61; + //lanes_62 62; lanes_63 63; + lanes_64 64; ); } )* @@ -639,36 +632,24 @@ macro_rules! test_lanes_panic { core_simd::simd::LaneCount<$lanes>: core_simd::simd::SupportedLaneCount, $body + // test some odd and even non-power-of-2 lengths on miri $crate::test_lanes_helper!( #[should_panic]; lanes_1 1; lanes_2 2; - lanes_4 4; - ); - - #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow - $crate::test_lanes_helper!( - #[should_panic]; - lanes_8 8; - lanes_16 16; - lanes_32 32; - lanes_64 64; - ); - - #[cfg(feature = "all_lane_counts")] - $crate::test_lanes_helper!( - // test some odd and even non-power-of-2 lengths on miri - #[should_panic]; lanes_3 3; - lanes_5 5; + lanes_6 6; ); - #[cfg(feature = "all_lane_counts")] #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow $crate::test_lanes_helper!( #[should_panic]; + lanes_4 4; + lanes_5 5; + lanes_7 7; + lanes_8 8; lanes_9 9; lanes_10 10; lanes_11 11; @@ -676,52 +657,55 @@ macro_rules! test_lanes_panic { lanes_13 13; lanes_14 14; lanes_15 15; + lanes_16 16; lanes_17 17; - lanes_18 18; - lanes_19 19; - lanes_20 20; - lanes_21 21; - lanes_22 22; - lanes_23 23; + //lanes_18 18; + //lanes_19 19; + //lanes_20 20; + //lanes_21 21; + //lanes_22 22; + //lanes_23 23; lanes_24 24; - lanes_25 25; - lanes_26 26; - lanes_27 27; - lanes_28 28; - lanes_29 29; - lanes_30 30; - lanes_31 31; - lanes_33 33; - lanes_34 34; - lanes_35 35; - lanes_36 36; - lanes_37 37; - lanes_38 38; - lanes_39 39; - lanes_40 40; - lanes_41 41; - lanes_42 42; - lanes_43 43; - lanes_44 44; - lanes_45 45; - lanes_46 46; + //lanes_25 25; + //lanes_26 26; + //lanes_27 27; + //lanes_28 28; + //lanes_29 29; + //lanes_30 30; + //lanes_31 31; + lanes_32 32; + //lanes_33 33; + //lanes_34 34; + //lanes_35 35; + //lanes_36 36; + //lanes_37 37; + //lanes_38 38; + //lanes_39 39; + //lanes_40 40; + //lanes_41 41; + //lanes_42 42; + //lanes_43 43; + //lanes_44 44; + //lanes_45 45; + //lanes_46 46; lanes_47 47; - lanes_48 48; - lanes_49 49; - lanes_50 50; - lanes_51 51; - lanes_52 52; - lanes_53 53; - lanes_54 54; - lanes_55 55; + //lanes_48 48; + //lanes_49 49; + //lanes_50 50; + //lanes_51 51; + //lanes_52 52; + //lanes_53 53; + //lanes_54 54; + //lanes_55 55; lanes_56 56; lanes_57 57; - lanes_58 58; - lanes_59 59; - lanes_60 60; - lanes_61 61; - lanes_62 62; + //lanes_58 58; + //lanes_59 59; + //lanes_60 60; + //lanes_61 61; + //lanes_62 62; lanes_63 63; + lanes_64 64; ); } )* diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000000..d17c6d2e8894 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2025-01-16" +components = ["rustfmt", "clippy", "miri", "rust-src"] diff --git a/subtree-sync.sh b/subtree-sync.sh new file mode 100755 index 000000000000..18360077623b --- /dev/null +++ b/subtree-sync.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -eou pipefail + +git fetch origin +pushd $2 +git fetch origin +popd + +if [ "$(git rev-parse --show-prefix)" != "" ]; then + echo "Run this script from the git root" >&2 + exit 1 +fi + +if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/master)" ]; then + echo "$(pwd) is not at origin/master" >&2 + exit 1 +fi + +if [ ! -f library/portable-simd/git-subtree.sh ]; then + curl -sS https://raw.githubusercontent.com/bjorn3/git/tqc-subtree-portable/contrib/subtree/git-subtree.sh -o library/portable-simd/git-subtree.sh + chmod +x library/portable-simd/git-subtree.sh +fi + +today=$(date +%Y-%m-%d) + +case $1 in + "push") + upstream=rust-upstream-$today + merge=sync-from-rust-$today + + pushd $2 + git checkout master + git pull + popd + + library/portable-simd/git-subtree.sh push -P library/portable-simd $2 $upstream + + pushd $2 + git checkout -B $merge origin/master + git merge $upstream + popd + echo "Branch \`$merge\` created in \`$2\`. You may need to resolve merge conflicts." + ;; + "pull") + branch=sync-from-portable-simd-$today + + git checkout -B $branch + echo "Creating branch \`$branch\`... You may need to resolve merge conflicts." + library/portable-simd/git-subtree.sh pull -P library/portable-simd $2 origin/master + ;; +esac From 02a28b29e2d1ab50b8d80c18018c33402cefc054 Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Wed, 12 Feb 2025 10:35:32 +0100 Subject: [PATCH 0002/1843] Remove ignored `#[must_use]` attributes from portable-simd The `#[must_use]` attribute has no effect when applied to methods in trait implementations. --- crates/core_simd/src/masks.rs | 13 ------------- crates/core_simd/src/masks/full_masks.rs | 5 ----- crates/core_simd/src/ops.rs | 1 - crates/core_simd/src/ops/deref.rs | 3 --- crates/core_simd/src/ops/unary.rs | 2 -- crates/core_simd/src/simd/num/float.rs | 1 - 6 files changed, 25 deletions(-) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index b763a7c75a5a..19d45f4d3b31 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -401,7 +401,6 @@ where LaneCount: SupportedLaneCount, { #[inline] - #[must_use = "method returns a defaulted mask with all elements set to false (0)"] fn default() -> Self { Self::splat(false) } @@ -413,7 +412,6 @@ where LaneCount: SupportedLaneCount, { #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] fn eq(&self, other: &Self) -> bool { self.0 == other.0 } @@ -425,7 +423,6 @@ where LaneCount: SupportedLaneCount, { #[inline] - #[must_use = "method returns a new Ordering and does not mutate the original value"] fn partial_cmp(&self, other: &Self) -> Option { self.0.partial_cmp(&other.0) } @@ -451,7 +448,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitand(self, rhs: Self) -> Self { Self(self.0 & rhs.0) } @@ -464,7 +460,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitand(self, rhs: bool) -> Self { self & Self::splat(rhs) } @@ -477,7 +472,6 @@ where { type Output = Mask; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitand(self, rhs: Mask) -> Mask { Mask::splat(self) & rhs } @@ -490,7 +484,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitor(self, rhs: Self) -> Self { Self(self.0 | rhs.0) } @@ -503,7 +496,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitor(self, rhs: bool) -> Self { self | Self::splat(rhs) } @@ -516,7 +508,6 @@ where { type Output = Mask; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitor(self, rhs: Mask) -> Mask { Mask::splat(self) | rhs } @@ -529,7 +520,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitxor(self, rhs: Self) -> Self::Output { Self(self.0 ^ rhs.0) } @@ -542,7 +532,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitxor(self, rhs: bool) -> Self::Output { self ^ Self::splat(rhs) } @@ -555,7 +544,6 @@ where { type Output = Mask; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitxor(self, rhs: Mask) -> Self::Output { Mask::splat(self) ^ rhs } @@ -568,7 +556,6 @@ where { type Output = Mask; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn not(self) -> Self::Output { Self(!self.0) } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 2d01946b5747..387b508c4b4e 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -21,7 +21,6 @@ where LaneCount: SupportedLaneCount, { #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn clone(&self) -> Self { *self } @@ -252,7 +251,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitand(self, rhs: Self) -> Self { // Safety: `self` is an integer vector unsafe { Self(core::intrinsics::simd::simd_and(self.0, rhs.0)) } @@ -266,7 +264,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitor(self, rhs: Self) -> Self { // Safety: `self` is an integer vector unsafe { Self(core::intrinsics::simd::simd_or(self.0, rhs.0)) } @@ -280,7 +277,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitxor(self, rhs: Self) -> Self { // Safety: `self` is an integer vector unsafe { Self(core::intrinsics::simd::simd_xor(self.0, rhs.0)) } @@ -294,7 +290,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn not(self) -> Self::Output { Self::splat(true) ^ self } diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index d3bd14a34027..4ac64a253a3b 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -135,7 +135,6 @@ macro_rules! for_base_types { type Output = $out; #[inline] - #[must_use = "operator returns a new vector without mutating the inputs"] // TODO: only useful for int Div::div, but we hope that this // will essentially always get inlined anyway. #[track_caller] diff --git a/crates/core_simd/src/ops/deref.rs b/crates/core_simd/src/ops/deref.rs index 0ff76cfba39b..913cbbe977c4 100644 --- a/crates/core_simd/src/ops/deref.rs +++ b/crates/core_simd/src/ops/deref.rs @@ -18,7 +18,6 @@ macro_rules! deref_lhs { type Output = Simd; #[inline] - #[must_use = "operator returns a new vector without mutating the inputs"] fn $call(self, rhs: $simd) -> Self::Output { (*self).$call(rhs) } @@ -39,7 +38,6 @@ macro_rules! deref_rhs { type Output = Simd; #[inline] - #[must_use = "operator returns a new vector without mutating the inputs"] fn $call(self, rhs: &$simd) -> Self::Output { self.$call(*rhs) } @@ -71,7 +69,6 @@ macro_rules! deref_ops { type Output = $simd; #[inline] - #[must_use = "operator returns a new vector without mutating the inputs"] fn $call(self, rhs: &'rhs $simd) -> Self::Output { (*self).$call(*rhs) } diff --git a/crates/core_simd/src/ops/unary.rs b/crates/core_simd/src/ops/unary.rs index bdae96332a3a..412a5b801171 100644 --- a/crates/core_simd/src/ops/unary.rs +++ b/crates/core_simd/src/ops/unary.rs @@ -11,7 +11,6 @@ macro_rules! neg { type Output = Self; #[inline] - #[must_use = "operator returns a new vector without mutating the input"] fn neg(self) -> Self::Output { // Safety: `self` is a signed vector unsafe { core::intrinsics::simd::simd_neg(self) } @@ -46,7 +45,6 @@ macro_rules! not { type Output = Self; #[inline] - #[must_use = "operator returns a new vector without mutating the input"] fn not(self) -> Self::Output { self ^ (Simd::splat(!(0 as $scalar))) } diff --git a/crates/core_simd/src/simd/num/float.rs b/crates/core_simd/src/simd/num/float.rs index 79954b937b39..db705dfe2022 100644 --- a/crates/core_simd/src/simd/num/float.rs +++ b/crates/core_simd/src/simd/num/float.rs @@ -371,7 +371,6 @@ macro_rules! impl_trait { } #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn is_normal(self) -> Self::Mask { !(self.abs().simd_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite()) } From aaf8ff1f9ec9b7049a7896a3c278f9e990306cb8 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Wed, 19 Mar 2025 00:58:47 -0400 Subject: [PATCH 0003/1843] Merge commit 'c14f2fc3eb69c164d8bf8d36d91ebd60bd5261e6' into sync-from-portable-simd-2025-03-19 --- beginners-guide.md | 4 +- crates/core_simd/Cargo.toml | 2 +- crates/core_simd/src/lib.rs | 6 ++- crates/core_simd/src/masks/bitmask.rs | 22 +++++------ crates/core_simd/src/masks/full_masks.rs | 20 +++++----- crates/core_simd/src/ops.rs | 2 +- crates/core_simd/src/simd/cmp/eq.rs | 2 +- crates/core_simd/src/simd/cmp/ord.rs | 2 +- crates/core_simd/src/simd/num/float.rs | 9 +++-- crates/core_simd/src/simd/num/int.rs | 4 +- crates/core_simd/src/simd/num/uint.rs | 2 +- crates/core_simd/src/simd/prelude.rs | 3 +- crates/core_simd/src/simd/ptr/const_ptr.rs | 2 +- crates/core_simd/src/simd/ptr/mut_ptr.rs | 2 +- crates/core_simd/src/swizzle.rs | 44 ++++++++++++++++++++++ crates/core_simd/src/to_bytes.rs | 2 +- crates/core_simd/src/vector.rs | 4 +- crates/core_simd/tests/layout.rs | 4 +- crates/core_simd/tests/pointers.rs | 2 +- crates/core_simd/tests/round.rs | 2 +- crates/test_helpers/src/subnormals.rs | 2 +- 21 files changed, 96 insertions(+), 46 deletions(-) diff --git a/beginners-guide.md b/beginners-guide.md index 17ade06ae80f..dc08d847ced5 100644 --- a/beginners-guide.md +++ b/beginners-guide.md @@ -80,12 +80,12 @@ Most of the portable SIMD API is designed to allow the user to gloss over the de Fortunately, most SIMD types have a fairly predictable size. `i32x4` is bit-equivalent to `[i32; 4]` and so can be bitcast to it, e.g. using [`mem::transmute`], though the API usually offers a safe cast you can use instead. -However, this is not the same as alignment. Computer architectures generally prefer aligned accesses, especially when moving data between memory and vector registers, and while some support specialized operations that can bend the rules to help with this, unaligned access is still typically slow, or even undefined behavior. In addition, different architectures can require different alignments when interacting with their native SIMD types. For this reason, any `#[repr(simd)]` type has a non-portable alignment. If it is necessary to directly interact with the alignment of these types, it should be via [`mem::align_of`]. +However, this is not the same as alignment. Computer architectures generally prefer aligned accesses, especially when moving data between memory and vector registers, and while some support specialized operations that can bend the rules to help with this, unaligned access is still typically slow, or even undefined behavior. In addition, different architectures can require different alignments when interacting with their native SIMD types. For this reason, any `#[repr(simd)]` type has a non-portable alignment. If it is necessary to directly interact with the alignment of these types, it should be via [`align_of`]. When working with slices, data correctly aligned for SIMD can be acquired using the [`as_simd`] and [`as_simd_mut`] methods of the slice primitive. [`mem::transmute`]: https://doc.rust-lang.org/core/mem/fn.transmute.html -[`mem::align_of`]: https://doc.rust-lang.org/core/mem/fn.align_of.html +[`align_of`]: https://doc.rust-lang.org/core/mem/fn.align_of.html [`as_simd`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.as_simd [`as_simd_mut`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.as_simd_mut diff --git a/crates/core_simd/Cargo.toml b/crates/core_simd/Cargo.toml index a7a6d43b11d3..537ce459c07c 100644 --- a/crates/core_simd/Cargo.toml +++ b/crates/core_simd/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "core_simd" version = "0.1.0" -edition = "2021" +edition = "2024" homepage = "https://github.com/rust-lang/portable-simd" repository = "https://github.com/rust-lang/portable-simd" keywords = ["core", "simd", "intrinsics"] diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 7f57847c9c23..717b882b64ba 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -35,7 +35,11 @@ feature(stdarch_x86_avx512) )] #![warn(missing_docs, clippy::missing_inline_in_public_items)] // basically all items, really -#![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)] +#![deny( + unsafe_op_in_unsafe_fn, + unreachable_pub, + clippy::undocumented_unsafe_blocks +)] #![doc(test(attr(deny(warnings))))] #![allow(internal_features)] #![unstable(feature = "portable_simd", issue = "86656")] diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index db4312d5bf88..8221d8f17e90 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -5,7 +5,7 @@ use core::marker::PhantomData; /// A mask where each lane is represented by a single bit. #[repr(transparent)] -pub struct Mask( +pub(crate) struct Mask( as SupportedLaneCount>::BitMask, PhantomData, ) @@ -78,7 +78,7 @@ where { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn splat(value: bool) -> Self { + pub(crate) fn splat(value: bool) -> Self { let mut mask = as SupportedLaneCount>::BitMask::default(); if value { mask.as_mut().fill(u8::MAX) @@ -93,12 +93,12 @@ where #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + pub(crate) unsafe fn test_unchecked(&self, lane: usize) -> bool { (self.0.as_ref()[lane / 8] >> (lane % 8)) & 0x1 > 0 } #[inline] - pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + pub(crate) unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { unsafe { self.0.as_mut()[lane / 8] ^= ((value ^ self.test_unchecked(lane)) as u8) << (lane % 8) } @@ -106,7 +106,7 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_int(self) -> Simd { + pub(crate) fn to_int(self) -> Simd { unsafe { core::intrinsics::simd::simd_select_bitmask( self.0, @@ -118,19 +118,19 @@ where #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub unsafe fn from_int_unchecked(value: Simd) -> Self { + pub(crate) unsafe fn from_int_unchecked(value: Simd) -> Self { unsafe { Self(core::intrinsics::simd::simd_bitmask(value), PhantomData) } } #[inline] - pub fn to_bitmask_integer(self) -> u64 { + pub(crate) fn to_bitmask_integer(self) -> u64 { let mut bitmask = [0u8; 8]; bitmask[..self.0.as_ref().len()].copy_from_slice(self.0.as_ref()); u64::from_ne_bytes(bitmask) } #[inline] - pub fn from_bitmask_integer(bitmask: u64) -> Self { + pub(crate) fn from_bitmask_integer(bitmask: u64) -> Self { let mut bytes = as SupportedLaneCount>::BitMask::default(); let len = bytes.as_mut().len(); bytes @@ -141,7 +141,7 @@ where #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn convert(self) -> Mask + pub(crate) fn convert(self) -> Mask where U: MaskElement, { @@ -151,13 +151,13 @@ where #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub fn any(self) -> bool { + pub(crate) fn any(self) -> bool { self != Self::splat(false) } #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub fn all(self) -> bool { + pub(crate) fn all(self) -> bool { self == Self::splat(true) } } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 387b508c4b4e..4e98db4070a9 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -3,7 +3,7 @@ use crate::simd::{LaneCount, MaskElement, Simd, SupportedLaneCount}; #[repr(transparent)] -pub struct Mask(Simd) +pub(crate) struct Mask(Simd) where T: MaskElement, LaneCount: SupportedLaneCount; @@ -80,7 +80,7 @@ macro_rules! impl_reverse_bits { #[inline(always)] fn reverse_bits(self, n: usize) -> Self { let rev = <$int>::reverse_bits(self); - let bitsize = core::mem::size_of::<$int>() * 8; + let bitsize = size_of::<$int>() * 8; if n < bitsize { // Shift things back to the right rev >> (bitsize - n) @@ -102,36 +102,36 @@ where { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn splat(value: bool) -> Self { + pub(crate) fn splat(value: bool) -> Self { Self(Simd::splat(if value { T::TRUE } else { T::FALSE })) } #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + pub(crate) unsafe fn test_unchecked(&self, lane: usize) -> bool { T::eq(self.0[lane], T::TRUE) } #[inline] - pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + pub(crate) unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { self.0[lane] = if value { T::TRUE } else { T::FALSE } } #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_int(self) -> Simd { + pub(crate) fn to_int(self) -> Simd { self.0 } #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub unsafe fn from_int_unchecked(value: Simd) -> Self { + pub(crate) unsafe fn from_int_unchecked(value: Simd) -> Self { Self(value) } #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn convert(self) -> Mask + pub(crate) fn convert(self) -> Mask where U: MaskElement, { @@ -220,14 +220,14 @@ where #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub fn any(self) -> bool { + pub(crate) fn any(self) -> bool { // Safety: use `self` as an integer vector unsafe { core::intrinsics::simd::simd_reduce_any(self.to_int()) } } #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub fn all(self) -> bool { + pub(crate) fn all(self) -> bool { // Safety: use `self` as an integer vector unsafe { core::intrinsics::simd::simd_reduce_all(self.to_int()) } } diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index 4ac64a253a3b..f36e8d01a73b 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -1,4 +1,4 @@ -use crate::simd::{cmp::SimdPartialEq, LaneCount, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount, cmp::SimdPartialEq}; use core::ops::{Add, Mul}; use core::ops::{BitAnd, BitOr, BitXor}; use core::ops::{Div, Rem, Sub}; diff --git a/crates/core_simd/src/simd/cmp/eq.rs b/crates/core_simd/src/simd/cmp/eq.rs index 93989ce91b89..2312ba401fa7 100644 --- a/crates/core_simd/src/simd/cmp/eq.rs +++ b/crates/core_simd/src/simd/cmp/eq.rs @@ -1,6 +1,6 @@ use crate::simd::{ - ptr::{SimdConstPtr, SimdMutPtr}, LaneCount, Mask, Simd, SimdElement, SupportedLaneCount, + ptr::{SimdConstPtr, SimdMutPtr}, }; /// Parallel `PartialEq`. diff --git a/crates/core_simd/src/simd/cmp/ord.rs b/crates/core_simd/src/simd/cmp/ord.rs index 899f00a83164..e813e7613032 100644 --- a/crates/core_simd/src/simd/cmp/ord.rs +++ b/crates/core_simd/src/simd/cmp/ord.rs @@ -1,7 +1,7 @@ use crate::simd::{ + LaneCount, Mask, Simd, SupportedLaneCount, cmp::SimdPartialEq, ptr::{SimdConstPtr, SimdMutPtr}, - LaneCount, Mask, Simd, SupportedLaneCount, }; /// Parallel `PartialOrd`. diff --git a/crates/core_simd/src/simd/num/float.rs b/crates/core_simd/src/simd/num/float.rs index db705dfe2022..b5972c47373b 100644 --- a/crates/core_simd/src/simd/num/float.rs +++ b/crates/core_simd/src/simd/num/float.rs @@ -1,7 +1,7 @@ use super::sealed::Sealed; use crate::simd::{ - cmp::{SimdPartialEq, SimdPartialOrd}, LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, + cmp::{SimdPartialEq, SimdPartialOrd}, }; /// Operations on SIMD vectors of floats. @@ -263,7 +263,8 @@ macro_rules! impl_trait { unsafe { core::intrinsics::simd::simd_as(self) } } - // https://github.com/llvm/llvm-project/issues/94694 + // workaround for https://github.com/llvm/llvm-project/issues/94694 (fixed in LLVM 20) + // tracked in: https://github.com/rust-lang/rust/issues/135982 #[cfg(target_arch = "aarch64")] #[inline] fn cast(self) -> Self::Cast @@ -302,14 +303,14 @@ macro_rules! impl_trait { #[inline] fn to_bits(self) -> Simd<$bits_ty, N> { - assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + assert_eq!(size_of::(), size_of::()); // Safety: transmuting between vector types is safe unsafe { core::mem::transmute_copy(&self) } } #[inline] fn from_bits(bits: Simd<$bits_ty, N>) -> Self { - assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + assert_eq!(size_of::(), size_of::()); // Safety: transmuting between vector types is safe unsafe { core::mem::transmute_copy(&bits) } } diff --git a/crates/core_simd/src/simd/num/int.rs b/crates/core_simd/src/simd/num/int.rs index 3a51235ff954..d25050c3e4b4 100644 --- a/crates/core_simd/src/simd/num/int.rs +++ b/crates/core_simd/src/simd/num/int.rs @@ -1,7 +1,7 @@ use super::sealed::Sealed; use crate::simd::{ - cmp::SimdOrd, cmp::SimdPartialOrd, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement, - SupportedLaneCount, + LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::SimdOrd, + cmp::SimdPartialOrd, num::SimdUint, }; /// Operations on SIMD vectors of signed integers. diff --git a/crates/core_simd/src/simd/num/uint.rs b/crates/core_simd/src/simd/num/uint.rs index 1ab2d8c7b731..45d978068b66 100644 --- a/crates/core_simd/src/simd/num/uint.rs +++ b/crates/core_simd/src/simd/num/uint.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{cmp::SimdOrd, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount}; +use crate::simd::{LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::SimdOrd}; /// Operations on SIMD vectors of unsigned integers. pub trait SimdUint: Copy + Sealed { diff --git a/crates/core_simd/src/simd/prelude.rs b/crates/core_simd/src/simd/prelude.rs index 4b7c744c0132..e5d7a2aeb73d 100644 --- a/crates/core_simd/src/simd/prelude.rs +++ b/crates/core_simd/src/simd/prelude.rs @@ -7,10 +7,11 @@ #[doc(no_inline)] pub use super::{ + Mask, Simd, cmp::{SimdOrd, SimdPartialEq, SimdPartialOrd}, num::{SimdFloat, SimdInt, SimdUint}, ptr::{SimdConstPtr, SimdMutPtr}, - simd_swizzle, Mask, Simd, + simd_swizzle, }; #[rustfmt::skip] diff --git a/crates/core_simd/src/simd/ptr/const_ptr.rs b/crates/core_simd/src/simd/ptr/const_ptr.rs index 47383809ffba..36452e7ae920 100644 --- a/crates/core_simd/src/simd/ptr/const_ptr.rs +++ b/crates/core_simd/src/simd/ptr/const_ptr.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{cmp::SimdPartialEq, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount}; +use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount, cmp::SimdPartialEq, num::SimdUint}; /// Operations on SIMD vectors of constant pointers. pub trait SimdConstPtr: Copy + Sealed { diff --git a/crates/core_simd/src/simd/ptr/mut_ptr.rs b/crates/core_simd/src/simd/ptr/mut_ptr.rs index 3f20eef21a31..c644f390c20a 100644 --- a/crates/core_simd/src/simd/ptr/mut_ptr.rs +++ b/crates/core_simd/src/simd/ptr/mut_ptr.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{cmp::SimdPartialEq, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount}; +use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount, cmp::SimdPartialEq, num::SimdUint}; /// Operations on SIMD vectors of mutable pointers. pub trait SimdMutPtr: Copy + Sealed { diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 42425ef37e50..dbdd6ef40eba 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -214,6 +214,17 @@ where /// Rotates the vector such that the first `OFFSET` elements of the slice move to the end /// while the last `self.len() - OFFSET` elements move to the front. After calling `rotate_elements_left`, /// the element previously at index `OFFSET` will become the first element in the slice. + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd::Simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd::Simd; + /// let a = Simd::from_array([0, 1, 2, 3]); + /// let x = a.rotate_elements_left::<3>(); + /// assert_eq!(x.to_array(), [3, 0, 1, 2]); + /// + /// let y = a.rotate_elements_left::<7>(); + /// assert_eq!(y.to_array(), [3, 0, 1, 2]); + /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn rotate_elements_left(self) -> Self { @@ -238,6 +249,17 @@ where /// Rotates the vector such that the first `self.len() - OFFSET` elements of the vector move to /// the end while the last `OFFSET` elements move to the front. After calling `rotate_elements_right`, /// the element previously at index `self.len() - OFFSET` will become the first element in the slice. + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd::Simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd::Simd; + /// let a = Simd::from_array([0, 1, 2, 3]); + /// let x = a.rotate_elements_right::<3>(); + /// assert_eq!(x.to_array(), [1, 2, 3, 0]); + /// + /// let y = a.rotate_elements_right::<7>(); + /// assert_eq!(y.to_array(), [1, 2, 3, 0]); + /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn rotate_elements_right(self) -> Self { @@ -261,6 +283,17 @@ where /// Shifts the vector elements to the left by `OFFSET`, filling in with /// `padding` from the right. + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd::Simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd::Simd; + /// let a = Simd::from_array([0, 1, 2, 3]); + /// let x = a.shift_elements_left::<3>(255); + /// assert_eq!(x.to_array(), [3, 255, 255, 255]); + /// + /// let y = a.shift_elements_left::<7>(255); + /// assert_eq!(y.to_array(), [255, 255, 255, 255]); + /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn shift_elements_left(self, padding: T) -> Self { @@ -283,6 +316,17 @@ where /// Shifts the vector elements to the right by `OFFSET`, filling in with /// `padding` from the left. + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd::Simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd::Simd; + /// let a = Simd::from_array([0, 1, 2, 3]); + /// let x = a.shift_elements_right::<3>(255); + /// assert_eq!(x.to_array(), [255, 255, 255, 0]); + /// + /// let y = a.shift_elements_right::<7>(255); + /// assert_eq!(y.to_array(), [255, 255, 255, 255]); + /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn shift_elements_right(self, padding: T) -> Self { diff --git a/crates/core_simd/src/to_bytes.rs b/crates/core_simd/src/to_bytes.rs index 4833ea9e1136..fee2cc06c5b0 100644 --- a/crates/core_simd/src/to_bytes.rs +++ b/crates/core_simd/src/to_bytes.rs @@ -1,6 +1,6 @@ use crate::simd::{ - num::{SimdFloat, SimdInt, SimdUint}, LaneCount, Simd, SimdElement, SupportedLaneCount, + num::{SimdFloat, SimdInt, SimdUint}, }; mod sealed { diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 9c4dd36c24fe..d76a6cd52bfc 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -1,8 +1,8 @@ use crate::simd::{ + LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle, cmp::SimdPartialOrd, num::SimdUint, ptr::{SimdConstPtr, SimdMutPtr}, - LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle, }; /// A SIMD vector with the shape of `[T; N]` but the operations of `T`. @@ -83,7 +83,7 @@ use crate::simd::{ /// converting `[T]` to `[Simd]`, and allows soundly operating on an aligned SIMD body, /// but it may cost more time when handling the scalar head and tail. /// If these are not enough, it is most ideal to design data structures to be already aligned -/// to `mem::align_of::>()` before using `unsafe` Rust to read or write. +/// to `align_of::>()` before using `unsafe` Rust to read or write. /// Other ways to compensate for these facts, like materializing `Simd` to or from an array first, /// are handled by safe methods like [`Simd::from_array`] and [`Simd::from_slice`]. /// diff --git a/crates/core_simd/tests/layout.rs b/crates/core_simd/tests/layout.rs index 24114c2d261e..3b4666249b0d 100644 --- a/crates/core_simd/tests/layout.rs +++ b/crates/core_simd/tests/layout.rs @@ -7,8 +7,8 @@ macro_rules! layout_tests { test_helpers::test_lanes! { fn no_padding() { assert_eq!( - core::mem::size_of::>(), - core::mem::size_of::<[$ty; LANES]>(), + size_of::>(), + size_of::<[$ty; LANES]>(), ); } } diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs index d7db4e82b3ca..6e74c2d18b1e 100644 --- a/crates/core_simd/tests/pointers.rs +++ b/crates/core_simd/tests/pointers.rs @@ -1,8 +1,8 @@ #![feature(portable_simd)] use core_simd::simd::{ - ptr::{SimdConstPtr, SimdMutPtr}, Simd, + ptr::{SimdConstPtr, SimdMutPtr}, }; macro_rules! common_tests { diff --git a/crates/core_simd/tests/round.rs b/crates/core_simd/tests/round.rs index 847766ec41ed..4c1ac3c36f89 100644 --- a/crates/core_simd/tests/round.rs +++ b/crates/core_simd/tests/round.rs @@ -58,7 +58,7 @@ macro_rules! float_rounding_test { // all of the mantissa digits set to 1, pushed up to the MSB. const ALL_MANTISSA_BITS: IntScalar = ((1 << ::MANTISSA_DIGITS) - 1); const MAX_REPRESENTABLE_VALUE: Scalar = - (ALL_MANTISSA_BITS << (core::mem::size_of::() * 8 - ::MANTISSA_DIGITS as usize - 1)) as Scalar; + (ALL_MANTISSA_BITS << (size_of::() * 8 - ::MANTISSA_DIGITS as usize - 1)) as Scalar; let mut runner = test_helpers::make_runner(); runner.run( diff --git a/crates/test_helpers/src/subnormals.rs b/crates/test_helpers/src/subnormals.rs index ec0f1fb24b93..b5f19ba47b81 100644 --- a/crates/test_helpers/src/subnormals.rs +++ b/crates/test_helpers/src/subnormals.rs @@ -12,7 +12,7 @@ macro_rules! impl_float { $( impl FlushSubnormals for $ty { fn flush(self) -> Self { - let is_f32 = core::mem::size_of::() == 4; + let is_f32 = size_of::() == 4; let ppc_flush = is_f32 && cfg!(all( any(target_arch = "powerpc", all(target_arch = "powerpc64", target_endian = "big")), target_feature = "altivec", From cdac23c86a06003d37ad102edbdc74c8d3d9f933 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 7 Apr 2025 18:13:32 +0200 Subject: [PATCH 0004/1843] Fix grammar in beginners-guide.md --- beginners-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginners-guide.md b/beginners-guide.md index dc08d847ced5..4250a18315a6 100644 --- a/beginners-guide.md +++ b/beginners-guide.md @@ -25,7 +25,7 @@ SIMD has a few special vocabulary terms you should know: * **Scalar:** "Scalar" in mathematical contexts refers to values that can be represented as a single element, mostly numbers like 6, 3.14, or -2. It can also be used to describe "scalar operations" that use strictly scalar values, like addition. This term is mostly used to differentiate between vectorized operations that use SIMD instructions and scalar operations that don't. -* **Lane:** A single element position within a vector is called a lane. If you have `N` lanes available then they're numbered from `0` to `N-1` when referring to them, again like an array. The biggest difference between an array element and a vector lane is that in general is *relatively costly* to access an individual lane value. On most architectures, the vector has to be pushed out of the SIMD register onto the stack, then an individual lane is accessed while it's on the stack (and possibly the stack value is read back into a register). For this reason, when working with SIMD you should avoid reading or writing the value of an individual lane during hot loops. +* **Lane:** A single element position within a vector is called a lane. If you have `N` lanes available then they're numbered from `0` to `N-1` when referring to them, again like an array. The biggest difference between an array element and a vector lane is that in general it is *relatively costly* to access an individual lane value. On most architectures, the vector has to be pushed out of the SIMD register onto the stack, then an individual lane is accessed while it's on the stack (and possibly the stack value is read back into a register). For this reason, when working with SIMD you should avoid reading or writing the value of an individual lane during hot loops. * **Bit Widths:** When talking about SIMD, the bit widths used are the bit size of the vectors involved, *not* the individual elements. So "128-bit SIMD" has 128-bit vectors, and that might be `f32x4`, `i32x4`, `i16x8`, or other variations. While 128-bit SIMD is the most common, there's also 64-bit, 256-bit, and even 512-bit on the newest CPUs. From 6951b685b8d8bd42b88765432e125959cb5fbdd8 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Wed, 11 Jun 2025 17:45:38 -0400 Subject: [PATCH 0005/1843] Remove usize/isize `From` impls for vendor vector types Remove usize/isize impls from - `loongarch64.rs` - `wasm32.rs` - `x86.rs` --- crates/core_simd/src/vendor/loongarch64.rs | 5 ----- crates/core_simd/src/vendor/wasm32.rs | 14 -------------- crates/core_simd/src/vendor/x86.rs | 22 ---------------------- 3 files changed, 41 deletions(-) diff --git a/crates/core_simd/src/vendor/loongarch64.rs b/crates/core_simd/src/vendor/loongarch64.rs index 1290bc166b2b..13dda4769aba 100644 --- a/crates/core_simd/src/vendor/loongarch64.rs +++ b/crates/core_simd/src/vendor/loongarch64.rs @@ -24,8 +24,3 @@ from_transmute! { unsafe i64x2 => v2i64 } from_transmute! { unsafe i64x4 => v4i64 } from_transmute! { unsafe f64x2 => v2f64 } from_transmute! { unsafe f64x4 => v4f64 } - -from_transmute! { unsafe usizex2 => v2u64 } -from_transmute! { unsafe usizex4 => v4u64 } -from_transmute! { unsafe isizex2 => v2i64 } -from_transmute! { unsafe isizex4 => v4i64 } diff --git a/crates/core_simd/src/vendor/wasm32.rs b/crates/core_simd/src/vendor/wasm32.rs index ef3baf885b0f..1fdb2bc86d34 100644 --- a/crates/core_simd/src/vendor/wasm32.rs +++ b/crates/core_simd/src/vendor/wasm32.rs @@ -14,17 +14,3 @@ from_transmute! { unsafe f32x4 => v128 } from_transmute! { unsafe u64x2 => v128 } from_transmute! { unsafe i64x2 => v128 } from_transmute! { unsafe f64x2 => v128 } - -#[cfg(target_pointer_width = "32")] -mod p32 { - use super::*; - from_transmute! { unsafe usizex4 => v128 } - from_transmute! { unsafe isizex4 => v128 } -} - -#[cfg(target_pointer_width = "64")] -mod p64 { - use super::*; - from_transmute! { unsafe usizex2 => v128 } - from_transmute! { unsafe isizex2 => v128 } -} diff --git a/crates/core_simd/src/vendor/x86.rs b/crates/core_simd/src/vendor/x86.rs index 66aaf90eef59..eae42e6fd0d0 100644 --- a/crates/core_simd/src/vendor/x86.rs +++ b/crates/core_simd/src/vendor/x86.rs @@ -39,25 +39,3 @@ from_transmute! { unsafe i64x8 => __m512i } from_transmute! { unsafe f64x2 => __m128d } from_transmute! { unsafe f64x4 => __m256d } from_transmute! { unsafe f64x8 => __m512d } - -#[cfg(target_pointer_width = "32")] -mod p32 { - use super::*; - from_transmute! { unsafe usizex4 => __m128i } - from_transmute! { unsafe usizex8 => __m256i } - from_transmute! { unsafe Simd => __m512i } - from_transmute! { unsafe isizex4 => __m128i } - from_transmute! { unsafe isizex8 => __m256i } - from_transmute! { unsafe Simd => __m512i } -} - -#[cfg(target_pointer_width = "64")] -mod p64 { - use super::*; - from_transmute! { unsafe usizex2 => __m128i } - from_transmute! { unsafe usizex4 => __m256i } - from_transmute! { unsafe usizex8 => __m512i } - from_transmute! { unsafe isizex2 => __m128i } - from_transmute! { unsafe isizex4 => __m256i } - from_transmute! { unsafe isizex8 => __m512i } -} From 97a3a5008470239510262f378350132da358edaf Mon Sep 17 00:00:00 2001 From: ltdk Date: Thu, 12 Jun 2025 12:03:40 -0400 Subject: [PATCH 0006/1843] Add const to as_mut_array, copy_to_slice --- crates/core_simd/src/vector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index d76a6cd52bfc..984a356c6552 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -195,7 +195,7 @@ where /// Returns a mutable array reference containing the entire SIMD vector. #[inline] - pub fn as_mut_array(&mut self) -> &mut [T; N] { + pub const fn as_mut_array(&mut self) -> &mut [T; N] { // SAFETY: `Simd` is just an overaligned `[T; N]` with // potential padding at the end, so pointer casting to a // `&mut [T; N]` is safe. @@ -324,7 +324,7 @@ where /// ``` #[inline] #[track_caller] - pub fn copy_to_slice(self, slice: &mut [T]) { + pub const fn copy_to_slice(self, slice: &mut [T]) { assert!( slice.len() >= Self::LEN, "slice length must be at least the number of elements" From a85790961605af9967fe5b76c45671c37c17f56b Mon Sep 17 00:00:00 2001 From: ltdk Date: Fri, 13 Jun 2025 00:38:09 -0400 Subject: [PATCH 0007/1843] Make Mask::splat const --- crates/core_simd/src/lane_count.rs | 15 ++++++++++++++- crates/core_simd/src/masks.rs | 3 ++- crates/core_simd/src/masks/bitmask.rs | 23 +++++++++++------------ crates/core_simd/src/masks/full_masks.rs | 3 ++- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/crates/core_simd/src/lane_count.rs b/crates/core_simd/src/lane_count.rs index 280b27bc9bc6..c62b9d3b7d9b 100644 --- a/crates/core_simd/src/lane_count.rs +++ b/crates/core_simd/src/lane_count.rs @@ -18,7 +18,11 @@ impl LaneCount { /// Only SIMD vectors with supported lane counts are constructable. pub trait SupportedLaneCount: Sealed { #[doc(hidden)] - type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>; + type BitMask: Copy + AsRef<[u8]> + AsMut<[u8]>; + #[doc(hidden)] + const EMPTY_BIT_MASK: Self::BitMask; + #[doc(hidden)] + const FULL_BIT_MASK: Self::BitMask; } impl Sealed for LaneCount {} @@ -28,6 +32,15 @@ macro_rules! supported_lane_count { $( impl SupportedLaneCount for LaneCount<$lanes> { type BitMask = [u8; ($lanes + 7) / 8]; + const EMPTY_BIT_MASK: Self::BitMask = [0; ($lanes + 7) / 8]; + const FULL_BIT_MASK: Self::BitMask = { + const LEN: usize = ($lanes + 7) / 8; + let mut array = [!0u8; LEN]; + if $lanes % 8 > 0 { + array[LEN - 1] = (!0) >> (8 - $lanes % 8); + } + array + }; } )+ }; diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 19d45f4d3b31..2b6be3676279 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -139,7 +139,8 @@ where { /// Constructs a mask by setting all elements to the given value. #[inline] - pub fn splat(value: bool) -> Self { + #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] + pub const fn splat(value: bool) -> Self { Self(mask_impl::Mask::splat(value)) } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 8221d8f17e90..ef03ec7f7159 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -78,17 +78,16 @@ where { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) fn splat(value: bool) -> Self { - let mut mask = as SupportedLaneCount>::BitMask::default(); - if value { - mask.as_mut().fill(u8::MAX) - } else { - mask.as_mut().fill(u8::MIN) - } - if N % 8 > 0 { - *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - N % 8); - } - Self(mask, PhantomData) + #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] + pub(crate) const fn splat(value: bool) -> Self { + Self( + if value { + as SupportedLaneCount>::FULL_BIT_MASK + } else { + as SupportedLaneCount>::EMPTY_BIT_MASK + }, + PhantomData, + ) } #[inline] @@ -131,7 +130,7 @@ where #[inline] pub(crate) fn from_bitmask_integer(bitmask: u64) -> Self { - let mut bytes = as SupportedLaneCount>::BitMask::default(); + let mut bytes = as SupportedLaneCount>::BitMask::EMPTY_BIT_MASK; let len = bytes.as_mut().len(); bytes .as_mut() diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 4e98db4070a9..ddd7fb69b189 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -102,7 +102,8 @@ where { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) fn splat(value: bool) -> Self { + #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] + pub(crate) const fn splat(value: bool) -> Self { Self(Simd::splat(if value { T::TRUE } else { T::FALSE })) } From b47a091b7df34d0a95913974419076c67a91643d Mon Sep 17 00:00:00 2001 From: Marijn Schouten Date: Fri, 4 Jul 2025 14:22:09 +0000 Subject: [PATCH 0008/1843] clippy fix: use div_ceil --- crates/core_simd/src/lane_count.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/core_simd/src/lane_count.rs b/crates/core_simd/src/lane_count.rs index c62b9d3b7d9b..839195c38eda 100644 --- a/crates/core_simd/src/lane_count.rs +++ b/crates/core_simd/src/lane_count.rs @@ -8,7 +8,7 @@ pub struct LaneCount; impl LaneCount { /// The number of bytes in a bitmask with this many lanes. - pub const BITMASK_LEN: usize = (N + 7) / 8; + pub const BITMASK_LEN: usize = N.div_ceil(8); } /// Statically guarantees that a lane count is marked as supported. @@ -31,13 +31,12 @@ macro_rules! supported_lane_count { ($($lanes:literal),+) => { $( impl SupportedLaneCount for LaneCount<$lanes> { - type BitMask = [u8; ($lanes + 7) / 8]; - const EMPTY_BIT_MASK: Self::BitMask = [0; ($lanes + 7) / 8]; + type BitMask = [u8; Self::BITMASK_LEN]; + const EMPTY_BIT_MASK: Self::BitMask = [0; Self::BITMASK_LEN]; const FULL_BIT_MASK: Self::BitMask = { - const LEN: usize = ($lanes + 7) / 8; - let mut array = [!0u8; LEN]; + let mut array = [!0u8; Self::BITMASK_LEN]; if $lanes % 8 > 0 { - array[LEN - 1] = (!0) >> (8 - $lanes % 8); + array[Self::BITMASK_LEN - 1] = (!0) >> (8 - $lanes % 8); } array }; From 5a26848a5aa9fbaf75eed785acb40fc0010a38ba Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 5 Jul 2025 08:36:27 +0200 Subject: [PATCH 0009/1843] use `div_ceil` instead of manual logic --- crates/core_simd/src/lane_count.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/lane_count.rs b/crates/core_simd/src/lane_count.rs index 280b27bc9bc6..bbdfd5f5f3ed 100644 --- a/crates/core_simd/src/lane_count.rs +++ b/crates/core_simd/src/lane_count.rs @@ -8,7 +8,7 @@ pub struct LaneCount; impl LaneCount { /// The number of bytes in a bitmask with this many lanes. - pub const BITMASK_LEN: usize = (N + 7) / 8; + pub const BITMASK_LEN: usize = N.div_ceil(8); } /// Statically guarantees that a lane count is marked as supported. From 3a0909c7677e0903145fb84a66683d5989eb7b69 Mon Sep 17 00:00:00 2001 From: burgerindividual Date: Mon, 4 Aug 2025 11:29:22 -0400 Subject: [PATCH 0010/1843] Fix incorrect reference to EMPTY_BIT_MASK --- crates/core_simd/src/masks/bitmask.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index ef03ec7f7159..780162c5b4bb 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -130,7 +130,7 @@ where #[inline] pub(crate) fn from_bitmask_integer(bitmask: u64) -> Self { - let mut bytes = as SupportedLaneCount>::BitMask::EMPTY_BIT_MASK; + let mut bytes = as SupportedLaneCount>::EMPTY_BIT_MASK; let len = bytes.as_mut().len(); bytes .as_mut() From b8955c5656e0805847549ac6043d6ae89f7fc0cc Mon Sep 17 00:00:00 2001 From: binarycat Date: Tue, 8 Jul 2025 14:09:34 -0500 Subject: [PATCH 0011/1843] core: add Option::get_or_try_insert_with Co-authored-by: kennytm --- library/core/src/option.rs | 45 +++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/library/core/src/option.rs b/library/core/src/option.rs index ed070fbd2274..da753f36b16a 100644 --- a/library/core/src/option.rs +++ b/library/core/src/option.rs @@ -578,7 +578,7 @@ use crate::iter::{self, FusedIterator, TrustedLen}; use crate::marker::Destruct; -use crate::ops::{self, ControlFlow, Deref, DerefMut}; +use crate::ops::{self, ControlFlow, Deref, DerefMut, Residual, Try}; use crate::panicking::{panic, panic_display}; use crate::pin::Pin; use crate::{cmp, convert, hint, mem, slice}; @@ -1807,6 +1807,49 @@ impl Option { unsafe { self.as_mut().unwrap_unchecked() } } + /// If the option is `None`, calls the closure and inserts its output if successful. + /// + /// If the closure returns a residual value such as `Err` or `None`, + /// that residual value is returned and nothing is inserted. + /// + /// If the option is `Some`, nothing is inserted. + /// + /// Unless a residual is returned, a mutable reference to the value + /// of the option will be output. + /// + /// # Examples + /// + /// ``` + /// #![feature(option_get_or_try_insert_with)] + /// let mut o1: Option = None; + /// let mut o2: Option = None; + /// + /// let number = "12345"; + /// + /// assert_eq!(o1.get_or_try_insert_with(|| number.parse()).copied(), Ok(12345)); + /// assert!(o2.get_or_try_insert_with(|| number.parse()).is_err()); + /// assert_eq!(o1, Some(12345)); + /// assert_eq!(o2, None); + /// ``` + #[inline] + #[unstable(feature = "option_get_or_try_insert_with", issue = "143648")] + pub fn get_or_try_insert_with<'a, R, F>( + &'a mut self, + f: F, + ) -> >::TryType + where + F: FnOnce() -> R, + R: Try>, + { + if let None = self { + *self = Some(f()?); + } + // SAFETY: a `None` variant for `self` would have been replaced by a `Some` + // variant in the code above. + + Try::from_output(unsafe { self.as_mut().unwrap_unchecked() }) + } + ///////////////////////////////////////////////////////////////////////// // Misc ///////////////////////////////////////////////////////////////////////// From 3f01c753fd24607e20de083a407a5e9275f1ec78 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Sun, 17 Aug 2025 20:36:53 -0400 Subject: [PATCH 0012/1843] Update nightly toolchain and fix broken examples Update examples to remove features that have been stabilized --- crates/core_simd/examples/dot_product.rs | 34 ++++++++++++------- crates/core_simd/examples/matrix_inversion.rs | 2 +- rust-toolchain.toml | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 75d152ae7f0e..4ef32bfa60b5 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -1,8 +1,6 @@ //! Code taken from the `packed_simd` crate. //! Run this code with `cargo test --example dot_product`. -#![feature(array_chunks)] -#![feature(slice_as_chunks)] // Add these imports to use the stdsimd library #![feature(portable_simd)] use core_simd::simd::prelude::*; @@ -33,7 +31,7 @@ pub fn dot_prod_scalar_1(a: &[f32], b: &[f32]) -> f32 { } // We now move on to the SIMD implementations: notice the following constructs: -// `array_chunks::<4>`: mapping this over the vector will let use construct SIMD vectors +// `as_chunks::<4>`: mapping this over the vector will let us construct SIMD vectors // `f32x4::from_array`: construct the SIMD vector from a slice // `(a * b).reduce_sum()`: Multiply both f32x4 vectors together, and then reduce them. // This approach essentially uses SIMD to produce a vector of length N/4 of all the products, @@ -42,9 +40,11 @@ pub fn dot_prod_scalar_1(a: &[f32], b: &[f32]) -> f32 { pub fn dot_prod_simd_0(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); // TODO handle remainder when a.len() % 4 != 0 - a.array_chunks::<4>() + a.as_chunks::<4>() + .0 + .iter() .map(|&a| f32x4::from_array(a)) - .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .zip(b.as_chunks::<4>().0.iter().map(|&b| f32x4::from_array(b))) .map(|(a, b)| (a * b).reduce_sum()) .sum() } @@ -60,9 +60,11 @@ pub fn dot_prod_simd_0(a: &[f32], b: &[f32]) -> f32 { pub fn dot_prod_simd_1(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); // TODO handle remainder when a.len() % 4 != 0 - a.array_chunks::<4>() + a.as_chunks::<4>() + .0 + .iter() .map(|&a| f32x4::from_array(a)) - .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .zip(b.as_chunks::<4>().0.iter().map(|&b| f32x4::from_array(b))) .fold(f32x4::splat(0.0), |acc, zipped| acc + zipped.0 * zipped.1) .reduce_sum() } @@ -74,9 +76,11 @@ pub fn dot_prod_simd_2(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); // TODO handle remainder when a.len() % 4 != 0 let mut res = f32x4::splat(0.0); - a.array_chunks::<4>() + a.as_chunks::<4>() + .0 + .iter() .map(|&a| f32x4::from_array(a)) - .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .zip(b.as_chunks::<4>().0.iter().map(|&b| f32x4::from_array(b))) .for_each(|(a, b)| { res = a.mul_add(b, res); }); @@ -113,9 +117,11 @@ pub fn dot_prod_simd_3(a: &[f32], b: &[f32]) -> f32 { // next example. pub fn dot_prod_simd_4(a: &[f32], b: &[f32]) -> f32 { let mut sum = a - .array_chunks::<4>() + .as_chunks::<4>() + .0 + .iter() .map(|&a| f32x4::from_array(a)) - .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .zip(b.as_chunks::<4>().0.iter().map(|&b| f32x4::from_array(b))) .map(|(a, b)| a * b) .fold(f32x4::splat(0.0), std::ops::Add::add) .reduce_sum(); @@ -131,9 +137,11 @@ pub fn dot_prod_simd_4(a: &[f32], b: &[f32]) -> f32 { // This version allocates a single `XMM` register for accumulation, and the folds don't allocate on top of that. // Notice the use of `mul_add`, which can do a multiply and an add operation ber iteration. pub fn dot_prod_simd_5(a: &[f32], b: &[f32]) -> f32 { - a.array_chunks::<4>() + a.as_chunks::<4>() + .0 + .iter() .map(|&a| f32x4::from_array(a)) - .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .zip(b.as_chunks::<4>().0.iter().map(|&b| f32x4::from_array(b))) .fold(f32x4::splat(0.), |acc, (a, b)| a.mul_add(b, acc)) .reduce_sum() } diff --git a/crates/core_simd/examples/matrix_inversion.rs b/crates/core_simd/examples/matrix_inversion.rs index bad86414401d..ad2eea9153e0 100644 --- a/crates/core_simd/examples/matrix_inversion.rs +++ b/crates/core_simd/examples/matrix_inversion.rs @@ -1,7 +1,7 @@ //! 4x4 matrix inverse // Code ported from the `packed_simd` crate // Run this code with `cargo test --example matrix_inversion` -#![feature(array_chunks, portable_simd)] +#![feature(portable_simd)] use core_simd::simd::prelude::*; // Gotta define our own 4x4 matrix since Rust doesn't ship multidim arrays yet :^) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index d17c6d2e8894..df4bd75ecfe2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2025-01-16" +channel = "nightly-2025-08-17" components = ["rustfmt", "clippy", "miri", "rust-src"] From 28e79b143328822ac43d9d72f5f0fbf60479c0a5 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Sun, 17 Aug 2025 21:42:22 -0400 Subject: [PATCH 0013/1843] Update Cargo.lock --- Cargo.lock | 267 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 206 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1584c704fb22..d7accf71ab69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,12 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "autocfg" -version = "1.1.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "bitflags" @@ -16,31 +16,30 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "console_error_panic_hook" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" -dependencies = [ - "cfg-if", - "wasm-bindgen", -] +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "core_simd" @@ -55,45 +54,59 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "log" -version = "0.4.20" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "minicov" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b" +dependencies = [ + "cc", + "walkdir", +] [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -114,9 +127,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -167,10 +180,25 @@ dependencies = [ ] [[package]] -name = "scoped-tls" -version = "1.0.1" +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "std_float" @@ -184,9 +212,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.29" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -202,29 +230,40 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -233,21 +272,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -255,9 +295,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -268,19 +308,21 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-bindgen-test" -version = "0.3.37" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6e302a7ea94f83a6d09e78e7dc7d9ca7b186bc2829c24a22d0753efd680671" +checksum = "66c8d5e33ca3b6d9fa3b4676d774c5778031d27a578c2b007f905acf816152c3" dependencies = [ - "console_error_panic_hook", "js-sys", - "scoped-tls", + "minicov", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", @@ -288,20 +330,123 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.37" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecb993dd8c836930ed130e020e77d9b2e65dd0fbab1b67c790b0f5d80b11a575" +checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", + "syn", ] [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", ] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] From 07e9de0f6d73d5c9ffc87dbdc496687fc2ad7200 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Sun, 17 Aug 2025 21:46:26 -0400 Subject: [PATCH 0014/1843] Remove `i586-pc-windows-msvc` from CI --- .github/workflows/ci.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3984d8f0d8d9..96881687af05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - target: [x86_64-pc-windows-msvc, i686-pc-windows-msvc, i586-pc-windows-msvc, x86_64-unknown-linux-gnu] + target: [x86_64-pc-windows-msvc, i686-pc-windows-msvc, x86_64-unknown-linux-gnu] # `default` means we use the default target config for the target, # `native` means we run with `-Ctarget-cpu=native`, and anything else is # an arg to `-Ctarget-feature` @@ -68,18 +68,12 @@ jobs: exclude: # -Ctarget-cpu=native sounds like bad-news if target != host - { target: i686-pc-windows-msvc, target_feature: native } - - { target: i586-pc-windows-msvc, target_feature: native } include: # Populate the `matrix.os` field - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest } - { target: x86_64-pc-windows-msvc, os: windows-latest } - { target: i686-pc-windows-msvc, os: windows-latest } - - { target: i586-pc-windows-msvc, os: windows-latest } - - # These are globally available on all the other targets. - - { target: i586-pc-windows-msvc, target_feature: +sse, os: windows-latest } - - { target: i586-pc-windows-msvc, target_feature: +sse2, os: windows-latest } # Annoyingly, the x86_64-unknown-linux-gnu runner *almost* always has # avx512vl, but occasionally doesn't. Maybe one day we can enable it. @@ -129,7 +123,7 @@ jobs: run: cargo doc --verbose --target=${{ matrix.target }} env: RUSTDOCFLAGS: -Dwarnings - + macos-tests: name: ${{ matrix.target }} runs-on: macos-latest From 323484c8278886ffe58e8d355107a5b59ffd85cd Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:42:20 -0400 Subject: [PATCH 0015/1843] Add no-extra-rounding-error flag to Miri CI config --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96881687af05..5344efe1db0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,5 +244,10 @@ jobs: PROPTEST_CASES: 16 steps: - uses: actions/checkout@v4 + - name: Download and install nightly + run: | + rustup toolchain install nightly --component miri,rust-src + rustup override set nightly + cargo miri setup - name: Test (Miri) - run: cargo miri test + run: MIRIFLAGS="-Zmiri-no-extra-rounding-error" cargo miri test From c43c8d25a8c4f7035d4265e672f41848effbe615 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 18 Aug 2025 01:31:25 -0400 Subject: [PATCH 0016/1843] Check some float ops approximately --- Cargo.lock | 10 +++ crates/std_float/tests/float.rs | 29 +++++++- crates/test_helpers/Cargo.toml | 1 + crates/test_helpers/src/approxeq.rs | 110 ++++++++++++++++++++++++++++ crates/test_helpers/src/lib.rs | 76 +++++++++++++++++++ 5 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 crates/test_helpers/src/approxeq.rs diff --git a/Cargo.lock b/Cargo.lock index d7accf71ab69..5a5f0d8907ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,15 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -225,6 +234,7 @@ dependencies = [ name = "test_helpers" version = "0.1.0" dependencies = [ + "float-cmp", "proptest", ] diff --git a/crates/std_float/tests/float.rs b/crates/std_float/tests/float.rs index c66c968f8c66..c608ba49564e 100644 --- a/crates/std_float/tests/float.rs +++ b/crates/std_float/tests/float.rs @@ -16,15 +16,33 @@ macro_rules! unary_test { } } -macro_rules! binary_test { +macro_rules! unary_approx_test { { $scalar:tt, $($func:tt),+ } => { test_helpers::test_lanes! { $( fn $func() { - test_helpers::test_binary_elementwise( + test_helpers::test_unary_elementwise_approx( + &core_simd::simd::Simd::<$scalar, LANES>::$func, + &$scalar::$func, + &|_| true, + 8, + ) + } + )* + } + } +} + +macro_rules! binary_approx_test { + { $scalar:tt, $($func:tt),+ } => { + test_helpers::test_lanes! { + $( + fn $func() { + test_helpers::test_binary_elementwise_approx( &core_simd::simd::Simd::<$scalar, LANES>::$func, &$scalar::$func, &|_, _| true, + 16, ) } )* @@ -53,10 +71,13 @@ macro_rules! impl_tests { mod $scalar { use std_float::StdFloat; - unary_test! { $scalar, sqrt, sin, cos, exp, exp2, ln, log2, log10, ceil, floor, round, trunc } - binary_test! { $scalar, log } + unary_test! { $scalar, sqrt, ceil, floor, round, trunc } ternary_test! { $scalar, mul_add } + // https://github.com/rust-lang/miri/issues/3555 + unary_approx_test! { $scalar, sin, cos, exp, exp2, ln, log2, log10 } + binary_approx_test! { $scalar, log } + test_helpers::test_lanes! { fn fract() { test_helpers::test_unary_elementwise_flush_subnormals( diff --git a/crates/test_helpers/Cargo.toml b/crates/test_helpers/Cargo.toml index a5359b9abc84..408bb04c7aa4 100644 --- a/crates/test_helpers/Cargo.toml +++ b/crates/test_helpers/Cargo.toml @@ -6,3 +6,4 @@ publish = false [dependencies] proptest = { version = "0.10", default-features = false, features = ["alloc"] } +float-cmp = "0.10" diff --git a/crates/test_helpers/src/approxeq.rs b/crates/test_helpers/src/approxeq.rs new file mode 100644 index 000000000000..57b43a16bc6f --- /dev/null +++ b/crates/test_helpers/src/approxeq.rs @@ -0,0 +1,110 @@ +//! Compare numeric types approximately. + +use float_cmp::Ulps; + +pub trait ApproxEq { + fn approxeq(&self, other: &Self, _ulps: i64) -> bool; + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result; +} + +impl ApproxEq for bool { + fn approxeq(&self, other: &Self, _ulps: i64) -> bool { + self == other + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} + +macro_rules! impl_integer_approxeq { + { $($type:ty),* } => { + $( + impl ApproxEq for $type { + fn approxeq(&self, other: &Self, _ulps: i64) -> bool { + self == other + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?} ({:x})", self, self) + } + } + )* + }; +} + +impl_integer_approxeq! { u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize } + +macro_rules! impl_float_approxeq { + { $($type:ty),* } => { + $( + impl ApproxEq for $type { + fn approxeq(&self, other: &Self, ulps: i64) -> bool { + if self.is_nan() && other.is_nan() { + true + } else { + (self.ulps(other) as i64).abs() <= ulps + } + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?} ({:x})", self, self.to_bits()) + } + } + )* + }; +} + +impl_float_approxeq! { f32, f64 } + +impl ApproxEq for [T; N] { + fn approxeq(&self, other: &Self, ulps: i64) -> bool { + self.iter() + .zip(other.iter()) + .fold(true, |value, (left, right)| { + value && left.approxeq(right, ulps) + }) + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + #[repr(transparent)] + struct Wrapper<'a, T: ApproxEq>(&'a T); + + impl core::fmt::Debug for Wrapper<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + self.0.fmt(f) + } + } + + f.debug_list() + .entries(self.iter().map(|x| Wrapper(x))) + .finish() + } +} + +#[doc(hidden)] +pub struct ApproxEqWrapper<'a, T>(pub &'a T, pub i64); + +impl PartialEq for ApproxEqWrapper<'_, T> { + fn eq(&self, other: &T) -> bool { + self.0.approxeq(other, self.1) + } +} + +impl core::fmt::Debug for ApproxEqWrapper<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + self.0.fmt(f) + } +} + +#[macro_export] +macro_rules! prop_assert_approxeq { + { $a:expr, $b:expr, $ulps:expr $(,)? } => { + { + use $crate::approxeq::ApproxEqWrapper; + let a = $a; + let b = $b; + proptest::prop_assert_eq!(ApproxEqWrapper(&a, $ulps), b); + } + }; +} diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 197c920e11ea..35401a9ddb40 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -12,6 +12,9 @@ pub mod wasm; #[macro_use] pub mod biteq; +#[macro_use] +pub mod approxeq; + pub mod subnormals; use subnormals::FlushSubnormals; @@ -185,6 +188,41 @@ pub fn test_unary_elementwise( + fv: &dyn Fn(Vector) -> VectorResult, + fs: &dyn Fn(Scalar) -> ScalarResult, + check: &dyn Fn([Scalar; LANES]) -> bool, + ulps: i64, +) where + Scalar: Copy + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + approxeq::ApproxEq + core::fmt::Debug + DefaultStrategy, + Vector: Into<[Scalar; LANES]> + From<[Scalar; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_1(&|x: [Scalar; LANES]| { + proptest::prop_assume!(check(x)); + let result_1: [ScalarResult; LANES] = fv(x.into()).into(); + let result_2: [ScalarResult; LANES] = x + .iter() + .copied() + .map(fs) + .collect::>() + .try_into() + .unwrap(); + crate::prop_assert_approxeq!(result_1, result_2, ulps); + Ok(()) + }); +} + /// Test a unary vector function against a unary scalar function, applied elementwise. /// /// Where subnormals are flushed, use approximate equality. @@ -290,6 +328,44 @@ pub fn test_binary_elementwise< }); } +/// Test a binary vector function against a binary scalar function, applied elementwise. +pub fn test_binary_elementwise_approx< + Scalar1, + Scalar2, + ScalarResult, + Vector1, + Vector2, + VectorResult, + const LANES: usize, +>( + fv: &dyn Fn(Vector1, Vector2) -> VectorResult, + fs: &dyn Fn(Scalar1, Scalar2) -> ScalarResult, + check: &dyn Fn([Scalar1; LANES], [Scalar2; LANES]) -> bool, + ulps: i64, +) where + Scalar1: Copy + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + approxeq::ApproxEq + core::fmt::Debug + DefaultStrategy, + Vector1: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, + Vector2: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + test_2(&|x: [Scalar1; LANES], y: [Scalar2; LANES]| { + proptest::prop_assume!(check(x, y)); + let result_1: [ScalarResult; LANES] = fv(x.into(), y.into()).into(); + let result_2: [ScalarResult; LANES] = x + .iter() + .copied() + .zip(y.iter().copied()) + .map(|(x, y)| fs(x, y)) + .collect::>() + .try_into() + .unwrap(); + crate::prop_assert_approxeq!(result_1, result_2, ulps); + Ok(()) + }); +} + /// Test a binary vector function against a binary scalar function, applied elementwise. /// /// Where subnormals are flushed, use approximate equality. From b902397bc20e1d0fa5ffe2939b3b9082f2f5b658 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 18 Aug 2025 01:33:28 -0400 Subject: [PATCH 0017/1843] Clippy isn't aware of repr(simd, packed) --- crates/core_simd/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 717b882b64ba..9d7dfd2ab351 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -41,7 +41,7 @@ clippy::undocumented_unsafe_blocks )] #![doc(test(attr(deny(warnings))))] -#![allow(internal_features)] +#![allow(internal_features, clippy::repr_packed_without_abi)] #![unstable(feature = "portable_simd", issue = "86656")] //! Portable SIMD module. From 1ce33dd37dd87dc566e0f9ea5b2e15e17c458ee5 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 18 Aug 2025 01:34:11 -0400 Subject: [PATCH 0018/1843] Revert "Add no-extra-rounding-error flag to Miri CI config" This reverts commit 323484c8278886ffe58e8d355107a5b59ffd85cd. --- .github/workflows/ci.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5344efe1db0e..96881687af05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,10 +244,5 @@ jobs: PROPTEST_CASES: 16 steps: - uses: actions/checkout@v4 - - name: Download and install nightly - run: | - rustup toolchain install nightly --component miri,rust-src - rustup override set nightly - cargo miri setup - name: Test (Miri) - run: MIRIFLAGS="-Zmiri-no-extra-rounding-error" cargo miri test + run: cargo miri test From 61c45c164a7a9ebf3107eb6d8735514ee3221053 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 8 Aug 2025 16:31:47 +0800 Subject: [PATCH 0019/1843] loongarch64: Use unified data types for SIMD intrinsics --- crates/core_simd/src/vendor/loongarch64.rs | 40 +++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/core_simd/src/vendor/loongarch64.rs b/crates/core_simd/src/vendor/loongarch64.rs index 13dda4769aba..1f84cdb971ec 100644 --- a/crates/core_simd/src/vendor/loongarch64.rs +++ b/crates/core_simd/src/vendor/loongarch64.rs @@ -1,26 +1,26 @@ use crate::simd::*; use core::arch::loongarch64::*; -from_transmute! { unsafe u8x16 => v16u8 } -from_transmute! { unsafe u8x32 => v32u8 } -from_transmute! { unsafe i8x16 => v16i8 } -from_transmute! { unsafe i8x32 => v32i8 } +from_transmute! { unsafe u8x16 => m128i } +from_transmute! { unsafe u8x32 => m256i } +from_transmute! { unsafe i8x16 => m128i } +from_transmute! { unsafe i8x32 => m256i } -from_transmute! { unsafe u16x8 => v8u16 } -from_transmute! { unsafe u16x16 => v16u16 } -from_transmute! { unsafe i16x8 => v8i16 } -from_transmute! { unsafe i16x16 => v16i16 } +from_transmute! { unsafe u16x8 => m128i } +from_transmute! { unsafe u16x16 => m256i } +from_transmute! { unsafe i16x8 => m128i } +from_transmute! { unsafe i16x16 => m256i } -from_transmute! { unsafe u32x4 => v4u32 } -from_transmute! { unsafe u32x8 => v8u32 } -from_transmute! { unsafe i32x4 => v4i32 } -from_transmute! { unsafe i32x8 => v8i32 } -from_transmute! { unsafe f32x4 => v4f32 } -from_transmute! { unsafe f32x8 => v8f32 } +from_transmute! { unsafe u32x4 => m128i } +from_transmute! { unsafe u32x8 => m256i } +from_transmute! { unsafe i32x4 => m128i } +from_transmute! { unsafe i32x8 => m256i } +from_transmute! { unsafe f32x4 => m128 } +from_transmute! { unsafe f32x8 => m256 } -from_transmute! { unsafe u64x2 => v2u64 } -from_transmute! { unsafe u64x4 => v4u64 } -from_transmute! { unsafe i64x2 => v2i64 } -from_transmute! { unsafe i64x4 => v4i64 } -from_transmute! { unsafe f64x2 => v2f64 } -from_transmute! { unsafe f64x4 => v4f64 } +from_transmute! { unsafe u64x2 => m128i } +from_transmute! { unsafe u64x4 => m256i } +from_transmute! { unsafe i64x2 => m128i } +from_transmute! { unsafe i64x4 => m256i } +from_transmute! { unsafe f64x2 => m128d } +from_transmute! { unsafe f64x4 => m256d } From 402c045f7a8f2085dd6552e63d4b6ae26049e9f9 Mon Sep 17 00:00:00 2001 From: AudaciousAxiom <179637270+AudaciousAxiom@users.noreply.github.com> Date: Tue, 2 Sep 2025 21:34:04 +0200 Subject: [PATCH 0020/1843] docs(std): add missing closing code block fences in doc comments --- crates/core_simd/src/simd/num/int.rs | 1 + crates/core_simd/src/simd/num/uint.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/core_simd/src/simd/num/int.rs b/crates/core_simd/src/simd/num/int.rs index d25050c3e4b4..e7253313f036 100644 --- a/crates/core_simd/src/simd/num/int.rs +++ b/crates/core_simd/src/simd/num/int.rs @@ -58,6 +58,7 @@ pub trait SimdInt: Copy + Sealed { /// let sat = x.saturating_sub(max); /// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0])); /// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0])); + /// ``` fn saturating_sub(self, second: Self) -> Self; /// Lanewise absolute value, implemented in Rust. diff --git a/crates/core_simd/src/simd/num/uint.rs b/crates/core_simd/src/simd/num/uint.rs index 45d978068b66..e3ba8658bd80 100644 --- a/crates/core_simd/src/simd/num/uint.rs +++ b/crates/core_simd/src/simd/num/uint.rs @@ -55,6 +55,7 @@ pub trait SimdUint: Copy + Sealed { /// let sat = x.saturating_sub(max); /// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0])); /// assert_eq!(sat, Simd::splat(0)); + /// ``` fn saturating_sub(self, second: Self) -> Self; /// Lanewise absolute difference. From 1e62d4c87cec532abf267cd5a7cb0342f8369ed1 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:29:26 -0400 Subject: [PATCH 0021/1843] Rename `Mask::to_int` to `Mask::to_simd` --- crates/core_simd/src/masks.rs | 8 ++++---- crates/core_simd/src/masks/bitmask.rs | 2 +- crates/core_simd/src/masks/full_masks.rs | 8 ++++---- crates/core_simd/src/select.rs | 2 +- crates/core_simd/src/simd/cmp/eq.rs | 4 ++-- crates/core_simd/src/simd/cmp/ord.rs | 8 ++++---- crates/core_simd/src/swizzle.rs | 22 +++++++++++----------- crates/core_simd/src/vector.rs | 8 ++++---- crates/core_simd/tests/masks.rs | 2 +- 9 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 2b6be3676279..b42cca7c977d 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -175,7 +175,7 @@ where // This would be hypothetically valid as an "in-place" transmute, // but these are "dependently-sized" types, so copy elision it is! unsafe { - let mut bytes: Simd = core::intrinsics::simd::simd_cast(self.to_int()); + let mut bytes: Simd = core::intrinsics::simd::simd_cast(self.to_simd()); bytes &= Simd::splat(1i8); mem::transmute_copy(&bytes) } @@ -214,8 +214,8 @@ where /// represents `true`. #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_int(self) -> Simd { - self.0.to_int() + pub fn to_simd(self) -> Simd { + self.0.to_simd() } /// Converts the mask to a mask of any other element size. @@ -352,7 +352,7 @@ where // Safety: the input and output are integer vectors let index: Simd = unsafe { core::intrinsics::simd::simd_cast(index) }; - let masked_index = self.select(index, Self::splat(true).to_int()); + let masked_index = self.select(index, Self::splat(true).to_simd()); // Safety: the input and output are integer vectors let masked_index: Simd = diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 780162c5b4bb..4211789da708 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -105,7 +105,7 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub(crate) fn to_int(self) -> Simd { + pub(crate) fn to_simd(self) -> Simd { unsafe { core::intrinsics::simd::simd_select_bitmask( self.0, diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index ddd7fb69b189..a5382f0ff739 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -120,7 +120,7 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub(crate) fn to_int(self) -> Simd { + pub(crate) fn to_simd(self) -> Simd { self.0 } @@ -145,7 +145,7 @@ where where LaneCount: SupportedLaneCount, { - let resized = self.to_int().resize::(T::FALSE); + let resized = self.to_simd().resize::(T::FALSE); // Safety: `resized` is an integer vector with length M, which must match T let bitmask: U = unsafe { core::intrinsics::simd::simd_bitmask(resized) }; @@ -223,14 +223,14 @@ where #[must_use = "method returns a new bool and does not mutate the original value"] pub(crate) fn any(self) -> bool { // Safety: use `self` as an integer vector - unsafe { core::intrinsics::simd::simd_reduce_any(self.to_int()) } + unsafe { core::intrinsics::simd::simd_reduce_any(self.to_simd()) } } #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] pub(crate) fn all(self) -> bool { // Safety: use `self` as an integer vector - unsafe { core::intrinsics::simd::simd_reduce_all(self.to_int()) } + unsafe { core::intrinsics::simd::simd_reduce_all(self.to_simd()) } } } diff --git a/crates/core_simd/src/select.rs b/crates/core_simd/src/select.rs index f33aa261a928..a2db455a5268 100644 --- a/crates/core_simd/src/select.rs +++ b/crates/core_simd/src/select.rs @@ -28,7 +28,7 @@ where { // Safety: The mask has been cast to a vector of integers, // and the operands to select between are vectors of the same type and length. - unsafe { core::intrinsics::simd::simd_select(self.to_int(), true_values, false_values) } + unsafe { core::intrinsics::simd::simd_select(self.to_simd(), true_values, false_values) } } /// Choose elements from two masks. diff --git a/crates/core_simd/src/simd/cmp/eq.rs b/crates/core_simd/src/simd/cmp/eq.rs index 2312ba401fa7..3c483f3f94a3 100644 --- a/crates/core_simd/src/simd/cmp/eq.rs +++ b/crates/core_simd/src/simd/cmp/eq.rs @@ -59,14 +59,14 @@ macro_rules! impl_mask { fn simd_eq(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_eq(self.to_int(), other.to_int())) } + unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_eq(self.to_simd(), other.to_simd())) } } #[inline] fn simd_ne(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ne(self.to_int(), other.to_int())) } + unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ne(self.to_simd(), other.to_simd())) } } } )* diff --git a/crates/core_simd/src/simd/cmp/ord.rs b/crates/core_simd/src/simd/cmp/ord.rs index e813e7613032..6eb3e360bdeb 100644 --- a/crates/core_simd/src/simd/cmp/ord.rs +++ b/crates/core_simd/src/simd/cmp/ord.rs @@ -163,28 +163,28 @@ macro_rules! impl_mask { fn simd_lt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_lt(self.to_int(), other.to_int())) } + unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_lt(self.to_simd(), other.to_simd())) } } #[inline] fn simd_le(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_le(self.to_int(), other.to_int())) } + unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_le(self.to_simd(), other.to_simd())) } } #[inline] fn simd_gt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_gt(self.to_int(), other.to_int())) } + unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_gt(self.to_simd(), other.to_simd())) } } #[inline] fn simd_ge(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ge(self.to_int(), other.to_int())) } + unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ge(self.to_simd(), other.to_simd())) } } } diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index dbdd6ef40eba..b6af2ded1cce 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -165,7 +165,7 @@ pub trait Swizzle { LaneCount: SupportedLaneCount, { // SAFETY: all elements of this mask come from another mask - unsafe { Mask::from_int_unchecked(Self::swizzle(mask.to_int())) } + unsafe { Mask::from_int_unchecked(Self::swizzle(mask.to_simd())) } } /// Creates a new mask from the elements of `first` and `second`. @@ -181,7 +181,7 @@ pub trait Swizzle { LaneCount: SupportedLaneCount, { // SAFETY: all elements of this mask come from another mask - unsafe { Mask::from_int_unchecked(Self::concat_swizzle(first.to_int(), second.to_int())) } + unsafe { Mask::from_int_unchecked(Self::concat_swizzle(first.to_simd(), second.to_simd())) } } } @@ -524,7 +524,7 @@ where #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn reverse(self) -> Self { // Safety: swizzles are safe for masks - unsafe { Self::from_int_unchecked(self.to_int().reverse()) } + unsafe { Self::from_int_unchecked(self.to_simd().reverse()) } } /// Rotates the mask such that the first `OFFSET` elements of the slice move to the end @@ -534,7 +534,7 @@ where #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn rotate_elements_left(self) -> Self { // Safety: swizzles are safe for masks - unsafe { Self::from_int_unchecked(self.to_int().rotate_elements_left::()) } + unsafe { Self::from_int_unchecked(self.to_simd().rotate_elements_left::()) } } /// Rotates the mask such that the first `self.len() - OFFSET` elements of the mask move to @@ -544,7 +544,7 @@ where #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn rotate_elements_right(self) -> Self { // Safety: swizzles are safe for masks - unsafe { Self::from_int_unchecked(self.to_int().rotate_elements_right::()) } + unsafe { Self::from_int_unchecked(self.to_simd().rotate_elements_right::()) } } /// Shifts the mask elements to the left by `OFFSET`, filling in with @@ -554,7 +554,7 @@ where pub fn shift_elements_left(self, padding: bool) -> Self { // Safety: swizzles are safe for masks unsafe { - Self::from_int_unchecked(self.to_int().shift_elements_left::(if padding { + Self::from_int_unchecked(self.to_simd().shift_elements_left::(if padding { T::TRUE } else { T::FALSE @@ -569,7 +569,7 @@ where pub fn shift_elements_right(self, padding: bool) -> Self { // Safety: swizzles are safe for masks unsafe { - Self::from_int_unchecked(self.to_int().shift_elements_right::(if padding { + Self::from_int_unchecked(self.to_simd().shift_elements_right::(if padding { T::TRUE } else { T::FALSE @@ -598,7 +598,7 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn interleave(self, other: Self) -> (Self, Self) { - let (lo, hi) = self.to_int().interleave(other.to_int()); + let (lo, hi) = self.to_simd().interleave(other.to_simd()); // Safety: swizzles are safe for masks unsafe { (Self::from_int_unchecked(lo), Self::from_int_unchecked(hi)) } } @@ -627,7 +627,7 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn deinterleave(self, other: Self) -> (Self, Self) { - let (even, odd) = self.to_int().deinterleave(other.to_int()); + let (even, odd) = self.to_simd().deinterleave(other.to_simd()); // Safety: swizzles are safe for masks unsafe { ( @@ -659,7 +659,7 @@ where { // Safety: swizzles are safe for masks unsafe { - Mask::::from_int_unchecked(self.to_int().resize::(if value { + Mask::::from_int_unchecked(self.to_simd().resize::(if value { T::TRUE } else { T::FALSE @@ -684,6 +684,6 @@ where LaneCount: SupportedLaneCount, { // Safety: swizzles are safe for masks - unsafe { Mask::::from_int_unchecked(self.to_int().extract::()) } + unsafe { Mask::::from_int_unchecked(self.to_simd().extract::()) } } } diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 984a356c6552..1f15c9f85c71 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -474,7 +474,7 @@ where or: Self, ) -> Self { // SAFETY: The safety of reading elements through `ptr` is ensured by the caller. - unsafe { core::intrinsics::simd::simd_masked_load(enable.to_int(), ptr, or) } + unsafe { core::intrinsics::simd::simd_masked_load(enable.to_simd(), ptr, or) } } /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. @@ -652,7 +652,7 @@ where or: Self, ) -> Self { // Safety: The caller is responsible for upholding all invariants - unsafe { core::intrinsics::simd::simd_gather(or, source, enable.to_int()) } + unsafe { core::intrinsics::simd::simd_gather(or, source, enable.to_simd()) } } /// Conditionally write contiguous elements to `slice`. The `enable` mask controls @@ -723,7 +723,7 @@ where #[inline] pub unsafe fn store_select_ptr(self, ptr: *mut T, enable: Mask<::Mask, N>) { // SAFETY: The safety of writing elements through `ptr` is ensured by the caller. - unsafe { core::intrinsics::simd::simd_masked_store(enable.to_int(), ptr, self) } + unsafe { core::intrinsics::simd::simd_masked_store(enable.to_simd(), ptr, self) } } /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`. @@ -882,7 +882,7 @@ where #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, N>, enable: Mask) { // Safety: The caller is responsible for upholding all invariants - unsafe { core::intrinsics::simd::simd_scatter(self, dest, enable.to_int()) } + unsafe { core::intrinsics::simd::simd_scatter(self, dest, enable.to_simd()) } } } diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 48786d02440b..be7c5a27e86f 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -65,7 +65,7 @@ macro_rules! test_mask_api { fn roundtrip_int_conversion() { let values = [true, false, false, true, false, false, true, false]; let mask = Mask::<$type, 8>::from_array(values); - let int = mask.to_int(); + let int = mask.to_simd(); assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]); assert_eq!(Mask::<$type, 8>::from_int(int), mask); } From 42409f05bf162f4fac8e7010892c08539ccfa034 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:30:44 -0400 Subject: [PATCH 0022/1843] Rename `Mask::from_int` to `Mask::from_simd` --- crates/core_simd/src/masks.rs | 2 +- crates/core_simd/tests/masks.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index b42cca7c977d..022003ef4dcc 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -204,7 +204,7 @@ where #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] #[track_caller] - pub fn from_int(value: Simd) -> Self { + pub fn from_simd(value: Simd) -> Self { assert!(T::valid(value), "all values must be either 0 or -1",); // Safety: the validity has been checked unsafe { Self::from_int_unchecked(value) } diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index be7c5a27e86f..53fb2367b605 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -67,7 +67,7 @@ macro_rules! test_mask_api { let mask = Mask::<$type, 8>::from_array(values); let int = mask.to_simd(); assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]); - assert_eq!(Mask::<$type, 8>::from_int(int), mask); + assert_eq!(Mask::<$type, 8>::from_simd(int), mask); } #[test] From 3a70dd6a69e0eb4b7d72e812cf32204ba2e7669c Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:32:40 -0400 Subject: [PATCH 0023/1843] Rename `Mask::from_int_unchecked` to `Mask::from_simd_unchecked` --- crates/core_simd/src/masks.rs | 8 ++++---- crates/core_simd/src/masks/bitmask.rs | 2 +- crates/core_simd/src/masks/full_masks.rs | 4 ++-- crates/core_simd/src/simd/cmp/eq.rs | 8 ++++---- crates/core_simd/src/simd/cmp/ord.rs | 24 +++++++++++----------- crates/core_simd/src/swizzle.rs | 26 +++++++++++++----------- crates/core_simd/src/vector.rs | 4 ++-- 7 files changed, 39 insertions(+), 37 deletions(-) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 022003ef4dcc..ca1e3db8b46f 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -157,7 +157,7 @@ where let bytes: [u8; N] = mem::transmute_copy(&array); let bools: Simd = core::intrinsics::simd::simd_ne(Simd::from_array(bytes), Simd::splat(0u8)); - Mask::from_int_unchecked(core::intrinsics::simd::simd_cast(bools)) + Mask::from_simd_unchecked(core::intrinsics::simd::simd_cast(bools)) } } @@ -188,11 +188,11 @@ where /// All elements must be either 0 or -1. #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub unsafe fn from_int_unchecked(value: Simd) -> Self { + pub unsafe fn from_simd_unchecked(value: Simd) -> Self { // Safety: the caller must confirm this invariant unsafe { core::intrinsics::assume(::valid(value)); - Self(mask_impl::Mask::from_int_unchecked(value)) + Self(mask_impl::Mask::from_simd_unchecked(value)) } } @@ -207,7 +207,7 @@ where pub fn from_simd(value: Simd) -> Self { assert!(T::valid(value), "all values must be either 0 or -1",); // Safety: the validity has been checked - unsafe { Self::from_int_unchecked(value) } + unsafe { Self::from_simd_unchecked(value) } } /// Converts the mask to a vector of integers, where 0 represents `false` and -1 diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 4211789da708..83ee88c372ab 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -117,7 +117,7 @@ where #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) unsafe fn from_int_unchecked(value: Simd) -> Self { + pub(crate) unsafe fn from_simd_unchecked(value: Simd) -> Self { unsafe { Self(core::intrinsics::simd::simd_bitmask(value), PhantomData) } } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index a5382f0ff739..5ad2c1d1eaf6 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -126,7 +126,7 @@ where #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) unsafe fn from_int_unchecked(value: Simd) -> Self { + pub(crate) unsafe fn from_simd_unchecked(value: Simd) -> Self { Self(value) } @@ -180,7 +180,7 @@ where }; // SAFETY: `mask` only contains `T::TRUE` or `T::FALSE` - unsafe { Self::from_int_unchecked(mask.resize::(T::FALSE)) } + unsafe { Self::from_simd_unchecked(mask.resize::(T::FALSE)) } } #[inline] diff --git a/crates/core_simd/src/simd/cmp/eq.rs b/crates/core_simd/src/simd/cmp/eq.rs index 3c483f3f94a3..789fc0bb9424 100644 --- a/crates/core_simd/src/simd/cmp/eq.rs +++ b/crates/core_simd/src/simd/cmp/eq.rs @@ -30,14 +30,14 @@ macro_rules! impl_number { fn simd_eq(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_eq(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_eq(self, other)) } } #[inline] fn simd_ne(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_ne(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_ne(self, other)) } } } )* @@ -59,14 +59,14 @@ macro_rules! impl_mask { fn simd_eq(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_eq(self.to_simd(), other.to_simd())) } + unsafe { Self::from_simd_unchecked(core::intrinsics::simd::simd_eq(self.to_simd(), other.to_simd())) } } #[inline] fn simd_ne(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ne(self.to_simd(), other.to_simd())) } + unsafe { Self::from_simd_unchecked(core::intrinsics::simd::simd_ne(self.to_simd(), other.to_simd())) } } } )* diff --git a/crates/core_simd/src/simd/cmp/ord.rs b/crates/core_simd/src/simd/cmp/ord.rs index 6eb3e360bdeb..4b2d0b55feba 100644 --- a/crates/core_simd/src/simd/cmp/ord.rs +++ b/crates/core_simd/src/simd/cmp/ord.rs @@ -56,28 +56,28 @@ macro_rules! impl_integer { fn simd_lt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_lt(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_lt(self, other)) } } #[inline] fn simd_le(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_le(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_le(self, other)) } } #[inline] fn simd_gt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_gt(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_gt(self, other)) } } #[inline] fn simd_ge(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_ge(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_ge(self, other)) } } } @@ -122,28 +122,28 @@ macro_rules! impl_float { fn simd_lt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_lt(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_lt(self, other)) } } #[inline] fn simd_le(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_le(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_le(self, other)) } } #[inline] fn simd_gt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_gt(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_gt(self, other)) } } #[inline] fn simd_ge(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Mask::from_int_unchecked(core::intrinsics::simd::simd_ge(self, other)) } + unsafe { Mask::from_simd_unchecked(core::intrinsics::simd::simd_ge(self, other)) } } } )* @@ -163,28 +163,28 @@ macro_rules! impl_mask { fn simd_lt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_lt(self.to_simd(), other.to_simd())) } + unsafe { Self::from_simd_unchecked(core::intrinsics::simd::simd_lt(self.to_simd(), other.to_simd())) } } #[inline] fn simd_le(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_le(self.to_simd(), other.to_simd())) } + unsafe { Self::from_simd_unchecked(core::intrinsics::simd::simd_le(self.to_simd(), other.to_simd())) } } #[inline] fn simd_gt(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_gt(self.to_simd(), other.to_simd())) } + unsafe { Self::from_simd_unchecked(core::intrinsics::simd::simd_gt(self.to_simd(), other.to_simd())) } } #[inline] fn simd_ge(self, other: Self) -> Self::Mask { // Safety: `self` is a vector, and the result of the comparison // is always a valid mask. - unsafe { Self::from_int_unchecked(core::intrinsics::simd::simd_ge(self.to_simd(), other.to_simd())) } + unsafe { Self::from_simd_unchecked(core::intrinsics::simd::simd_ge(self.to_simd(), other.to_simd())) } } } diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index b6af2ded1cce..81085a9ee4a3 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -165,7 +165,7 @@ pub trait Swizzle { LaneCount: SupportedLaneCount, { // SAFETY: all elements of this mask come from another mask - unsafe { Mask::from_int_unchecked(Self::swizzle(mask.to_simd())) } + unsafe { Mask::from_simd_unchecked(Self::swizzle(mask.to_simd())) } } /// Creates a new mask from the elements of `first` and `second`. @@ -181,7 +181,9 @@ pub trait Swizzle { LaneCount: SupportedLaneCount, { // SAFETY: all elements of this mask come from another mask - unsafe { Mask::from_int_unchecked(Self::concat_swizzle(first.to_simd(), second.to_simd())) } + unsafe { + Mask::from_simd_unchecked(Self::concat_swizzle(first.to_simd(), second.to_simd())) + } } } @@ -524,7 +526,7 @@ where #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn reverse(self) -> Self { // Safety: swizzles are safe for masks - unsafe { Self::from_int_unchecked(self.to_simd().reverse()) } + unsafe { Self::from_simd_unchecked(self.to_simd().reverse()) } } /// Rotates the mask such that the first `OFFSET` elements of the slice move to the end @@ -534,7 +536,7 @@ where #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn rotate_elements_left(self) -> Self { // Safety: swizzles are safe for masks - unsafe { Self::from_int_unchecked(self.to_simd().rotate_elements_left::()) } + unsafe { Self::from_simd_unchecked(self.to_simd().rotate_elements_left::()) } } /// Rotates the mask such that the first `self.len() - OFFSET` elements of the mask move to @@ -544,7 +546,7 @@ where #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn rotate_elements_right(self) -> Self { // Safety: swizzles are safe for masks - unsafe { Self::from_int_unchecked(self.to_simd().rotate_elements_right::()) } + unsafe { Self::from_simd_unchecked(self.to_simd().rotate_elements_right::()) } } /// Shifts the mask elements to the left by `OFFSET`, filling in with @@ -554,7 +556,7 @@ where pub fn shift_elements_left(self, padding: bool) -> Self { // Safety: swizzles are safe for masks unsafe { - Self::from_int_unchecked(self.to_simd().shift_elements_left::(if padding { + Self::from_simd_unchecked(self.to_simd().shift_elements_left::(if padding { T::TRUE } else { T::FALSE @@ -569,7 +571,7 @@ where pub fn shift_elements_right(self, padding: bool) -> Self { // Safety: swizzles are safe for masks unsafe { - Self::from_int_unchecked(self.to_simd().shift_elements_right::(if padding { + Self::from_simd_unchecked(self.to_simd().shift_elements_right::(if padding { T::TRUE } else { T::FALSE @@ -600,7 +602,7 @@ where pub fn interleave(self, other: Self) -> (Self, Self) { let (lo, hi) = self.to_simd().interleave(other.to_simd()); // Safety: swizzles are safe for masks - unsafe { (Self::from_int_unchecked(lo), Self::from_int_unchecked(hi)) } + unsafe { (Self::from_simd_unchecked(lo), Self::from_simd_unchecked(hi)) } } /// Deinterleave two masks. @@ -631,8 +633,8 @@ where // Safety: swizzles are safe for masks unsafe { ( - Self::from_int_unchecked(even), - Self::from_int_unchecked(odd), + Self::from_simd_unchecked(even), + Self::from_simd_unchecked(odd), ) } } @@ -659,7 +661,7 @@ where { // Safety: swizzles are safe for masks unsafe { - Mask::::from_int_unchecked(self.to_simd().resize::(if value { + Mask::::from_simd_unchecked(self.to_simd().resize::(if value { T::TRUE } else { T::FALSE @@ -684,6 +686,6 @@ where LaneCount: SupportedLaneCount, { // Safety: swizzles are safe for masks - unsafe { Mask::::from_int_unchecked(self.to_simd().extract::()) } + unsafe { Mask::::from_simd_unchecked(self.to_simd().extract::()) } } } diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 1f15c9f85c71..c00cfcdd41ff 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -926,7 +926,7 @@ where let mask = unsafe { let tfvec: Simd<::Mask, N> = core::intrinsics::simd::simd_eq(*self, *other); - Mask::from_int_unchecked(tfvec) + Mask::from_simd_unchecked(tfvec) }; // Two vectors are equal if all elements are equal when compared elementwise @@ -940,7 +940,7 @@ where let mask = unsafe { let tfvec: Simd<::Mask, N> = core::intrinsics::simd::simd_ne(*self, *other); - Mask::from_int_unchecked(tfvec) + Mask::from_simd_unchecked(tfvec) }; // Two vectors are non-equal if any elements are non-equal when compared elementwise From 32ba8edf03c233640569c1f9a0bf4529c3a6608c Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 12 Sep 2025 22:03:51 -0400 Subject: [PATCH 0024/1843] Remove poor-performing bitmasks, add Select trait, and enable select on integer bitmasks (#482) --- crates/core_simd/src/masks.rs | 123 +++++++--- crates/core_simd/src/masks/bitmask.rs | 231 ------------------ crates/core_simd/src/masks/full_masks.rs | 297 ----------------------- crates/core_simd/src/mod.rs | 1 + crates/core_simd/src/ops.rs | 2 +- crates/core_simd/src/select.rs | 199 +++++++++++---- crates/core_simd/src/simd/cmp/ord.rs | 6 +- crates/core_simd/src/simd/num/float.rs | 2 +- crates/core_simd/src/simd/num/int.rs | 2 +- crates/core_simd/src/swizzle_dyn.rs | 4 +- 10 files changed, 248 insertions(+), 619 deletions(-) delete mode 100644 crates/core_simd/src/masks/bitmask.rs delete mode 100644 crates/core_simd/src/masks/full_masks.rs diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index ca1e3db8b46f..7baa96475910 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -2,20 +2,33 @@ //! Types representing #![allow(non_camel_case_types)] -#[cfg_attr( - not(all(target_arch = "x86_64", target_feature = "avx512f")), - path = "masks/full_masks.rs" -)] -#[cfg_attr( - all(target_arch = "x86_64", target_feature = "avx512f"), - path = "masks/bitmask.rs" -)] -mod mask_impl; - -use crate::simd::{LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount}; +use crate::simd::{LaneCount, Select, Simd, SimdCast, SimdElement, SupportedLaneCount}; use core::cmp::Ordering; use core::{fmt, mem}; +pub(crate) trait FixEndianness { + fn fix_endianness(self) -> Self; +} + +macro_rules! impl_fix_endianness { + { $($int:ty),* } => { + $( + impl FixEndianness for $int { + #[inline(always)] + fn fix_endianness(self) -> Self { + if cfg!(target_endian = "big") { + <$int>::reverse_bits(self) + } else { + self + } + } + } + )* + } +} + +impl_fix_endianness! { u8, u16, u32, u64 } + mod sealed { use super::*; @@ -109,7 +122,7 @@ impl_element! { isize, usize } /// and/or Rust versions, and code should not assume that it is equivalent to /// `[T; N]`. #[repr(transparent)] -pub struct Mask(mask_impl::Mask) +pub struct Mask(Simd) where T: MaskElement, LaneCount: SupportedLaneCount; @@ -141,7 +154,7 @@ where #[inline] #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] pub const fn splat(value: bool) -> Self { - Self(mask_impl::Mask::splat(value)) + Self(Simd::splat(if value { T::TRUE } else { T::FALSE })) } /// Converts an array of bools to a SIMD mask. @@ -192,8 +205,8 @@ where // Safety: the caller must confirm this invariant unsafe { core::intrinsics::assume(::valid(value)); - Self(mask_impl::Mask::from_simd_unchecked(value)) } + Self(value) } /// Converts a vector of integers to a mask, where 0 represents `false` and -1 @@ -215,14 +228,15 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] pub fn to_simd(self) -> Simd { - self.0.to_simd() + self.0 } /// Converts the mask to a mask of any other element size. #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] pub fn cast(self) -> Mask { - Mask(self.0.convert()) + // Safety: mask elements are integers + unsafe { Mask(core::intrinsics::simd::simd_as(self.0)) } } /// Tests the value of the specified element. @@ -233,7 +247,7 @@ where #[must_use = "method returns a new bool and does not mutate the original value"] pub unsafe fn test_unchecked(&self, index: usize) -> bool { // Safety: the caller must confirm this invariant - unsafe { self.0.test_unchecked(index) } + unsafe { T::eq(*self.0.as_array().get_unchecked(index), T::TRUE) } } /// Tests the value of the specified element. @@ -244,9 +258,7 @@ where #[must_use = "method returns a new bool and does not mutate the original value"] #[track_caller] pub fn test(&self, index: usize) -> bool { - assert!(index < N, "element index out of range"); - // Safety: the element index has been checked - unsafe { self.test_unchecked(index) } + T::eq(self.0[index], T::TRUE) } /// Sets the value of the specified element. @@ -257,7 +269,7 @@ where pub unsafe fn set_unchecked(&mut self, index: usize, value: bool) { // Safety: the caller must confirm this invariant unsafe { - self.0.set_unchecked(index, value); + *self.0.as_mut_array().get_unchecked_mut(index) = if value { T::TRUE } else { T::FALSE } } } @@ -268,35 +280,67 @@ where #[inline] #[track_caller] pub fn set(&mut self, index: usize, value: bool) { - assert!(index < N, "element index out of range"); - // Safety: the element index has been checked - unsafe { - self.set_unchecked(index, value); - } + self.0[index] = if value { T::TRUE } else { T::FALSE } } /// Returns true if any element is set, or false otherwise. #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] pub fn any(self) -> bool { - self.0.any() + // Safety: `self` is a mask vector + unsafe { core::intrinsics::simd::simd_reduce_any(self.0) } } /// Returns true if all elements are set, or false otherwise. #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] pub fn all(self) -> bool { - self.0.all() + // Safety: `self` is a mask vector + unsafe { core::intrinsics::simd::simd_reduce_all(self.0) } } /// Creates a bitmask from a mask. /// /// Each bit is set if the corresponding element in the mask is `true`. - /// If the mask contains more than 64 elements, the bitmask is truncated to the first 64. #[inline] #[must_use = "method returns a new integer and does not mutate the original value"] pub fn to_bitmask(self) -> u64 { - self.0.to_bitmask_integer() + const { + assert!(N <= 64, "number of elements can't be greater than 64"); + } + + #[inline] + unsafe fn to_bitmask_impl( + mask: Mask, + ) -> U + where + T: MaskElement, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + { + let resized = mask.resize::(false); + + // Safety: `resized` is an integer vector with length M, which must match T + let bitmask: U = unsafe { core::intrinsics::simd::simd_bitmask(resized.0) }; + + // LLVM assumes bit order should match endianness + bitmask.fix_endianness() + } + + // TODO modify simd_bitmask to zero-extend output, making this unnecessary + if N <= 8 { + // Safety: bitmask matches length + unsafe { to_bitmask_impl::(self) as u64 } + } else if N <= 16 { + // Safety: bitmask matches length + unsafe { to_bitmask_impl::(self) as u64 } + } else if N <= 32 { + // Safety: bitmask matches length + unsafe { to_bitmask_impl::(self) as u64 } + } else { + // Safety: bitmask matches length + unsafe { to_bitmask_impl::(self) } + } } /// Creates a mask from a bitmask. @@ -306,7 +350,7 @@ where #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] pub fn from_bitmask(bitmask: u64) -> Self { - Self(mask_impl::Mask::from_bitmask_integer(bitmask)) + Self(bitmask.select(Simd::splat(T::TRUE), Simd::splat(T::FALSE))) } /// Finds the index of the first set element. @@ -450,7 +494,8 @@ where type Output = Self; #[inline] fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) + // Safety: `self` is an integer vector + unsafe { Self(core::intrinsics::simd::simd_and(self.0, rhs.0)) } } } @@ -486,7 +531,8 @@ where type Output = Self; #[inline] fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) + // Safety: `self` is an integer vector + unsafe { Self(core::intrinsics::simd::simd_or(self.0, rhs.0)) } } } @@ -522,7 +568,8 @@ where type Output = Self; #[inline] fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) + // Safety: `self` is an integer vector + unsafe { Self(core::intrinsics::simd::simd_xor(self.0, rhs.0)) } } } @@ -558,7 +605,7 @@ where type Output = Mask; #[inline] fn not(self) -> Self::Output { - Self(!self.0) + Self::splat(true) ^ self } } @@ -569,7 +616,7 @@ where { #[inline] fn bitand_assign(&mut self, rhs: Self) { - self.0 = self.0 & rhs.0; + *self = *self & rhs; } } @@ -591,7 +638,7 @@ where { #[inline] fn bitor_assign(&mut self, rhs: Self) { - self.0 = self.0 | rhs.0; + *self = *self | rhs; } } @@ -613,7 +660,7 @@ where { #[inline] fn bitxor_assign(&mut self, rhs: Self) { - self.0 = self.0 ^ rhs.0; + *self = *self ^ rhs; } } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs deleted file mode 100644 index 83ee88c372ab..000000000000 --- a/crates/core_simd/src/masks/bitmask.rs +++ /dev/null @@ -1,231 +0,0 @@ -#![allow(unused_imports)] -use super::MaskElement; -use crate::simd::{LaneCount, Simd, SupportedLaneCount}; -use core::marker::PhantomData; - -/// A mask where each lane is represented by a single bit. -#[repr(transparent)] -pub(crate) struct Mask( - as SupportedLaneCount>::BitMask, - PhantomData, -) -where - T: MaskElement, - LaneCount: SupportedLaneCount; - -impl Copy for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ -} - -impl Clone for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn clone(&self) -> Self { - *self - } -} - -impl PartialEq for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn eq(&self, other: &Self) -> bool { - self.0.as_ref() == other.0.as_ref() - } -} - -impl PartialOrd for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.0.as_ref().partial_cmp(other.0.as_ref()) - } -} - -impl Eq for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ -} - -impl Ord for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.0.as_ref().cmp(other.0.as_ref()) - } -} - -impl Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] - pub(crate) const fn splat(value: bool) -> Self { - Self( - if value { - as SupportedLaneCount>::FULL_BIT_MASK - } else { - as SupportedLaneCount>::EMPTY_BIT_MASK - }, - PhantomData, - ) - } - - #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] - pub(crate) unsafe fn test_unchecked(&self, lane: usize) -> bool { - (self.0.as_ref()[lane / 8] >> (lane % 8)) & 0x1 > 0 - } - - #[inline] - pub(crate) unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { - unsafe { - self.0.as_mut()[lane / 8] ^= ((value ^ self.test_unchecked(lane)) as u8) << (lane % 8) - } - } - - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub(crate) fn to_simd(self) -> Simd { - unsafe { - core::intrinsics::simd::simd_select_bitmask( - self.0, - Simd::splat(T::TRUE), - Simd::splat(T::FALSE), - ) - } - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) unsafe fn from_simd_unchecked(value: Simd) -> Self { - unsafe { Self(core::intrinsics::simd::simd_bitmask(value), PhantomData) } - } - - #[inline] - pub(crate) fn to_bitmask_integer(self) -> u64 { - let mut bitmask = [0u8; 8]; - bitmask[..self.0.as_ref().len()].copy_from_slice(self.0.as_ref()); - u64::from_ne_bytes(bitmask) - } - - #[inline] - pub(crate) fn from_bitmask_integer(bitmask: u64) -> Self { - let mut bytes = as SupportedLaneCount>::EMPTY_BIT_MASK; - let len = bytes.as_mut().len(); - bytes - .as_mut() - .copy_from_slice(&bitmask.to_ne_bytes()[..len]); - Self(bytes, PhantomData) - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) fn convert(self) -> Mask - where - U: MaskElement, - { - // Safety: bitmask layout does not depend on the element width - unsafe { core::mem::transmute_copy(&self) } - } - - #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] - pub(crate) fn any(self) -> bool { - self != Self::splat(false) - } - - #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] - pub(crate) fn all(self) -> bool { - self == Self::splat(true) - } -} - -impl core::ops::BitAnd for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, - as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, -{ - type Output = Self; - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - fn bitand(mut self, rhs: Self) -> Self { - for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { - *l &= r; - } - self - } -} - -impl core::ops::BitOr for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, - as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, -{ - type Output = Self; - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - fn bitor(mut self, rhs: Self) -> Self { - for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { - *l |= r; - } - self - } -} - -impl core::ops::BitXor for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - type Output = Self; - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - fn bitxor(mut self, rhs: Self) -> Self::Output { - for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { - *l ^= r; - } - self - } -} - -impl core::ops::Not for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - type Output = Self; - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - fn not(mut self) -> Self::Output { - for x in self.0.as_mut() { - *x = !*x; - } - if N % 8 > 0 { - *self.0.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - N % 8); - } - self - } -} diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs deleted file mode 100644 index 5ad2c1d1eaf6..000000000000 --- a/crates/core_simd/src/masks/full_masks.rs +++ /dev/null @@ -1,297 +0,0 @@ -//! Masks that take up full SIMD vector registers. - -use crate::simd::{LaneCount, MaskElement, Simd, SupportedLaneCount}; - -#[repr(transparent)] -pub(crate) struct Mask(Simd) -where - T: MaskElement, - LaneCount: SupportedLaneCount; - -impl Copy for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ -} - -impl Clone for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn clone(&self) -> Self { - *self - } -} - -impl PartialEq for Mask -where - T: MaskElement + PartialEq, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn eq(&self, other: &Self) -> bool { - self.0.eq(&other.0) - } -} - -impl PartialOrd for Mask -where - T: MaskElement + PartialOrd, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) - } -} - -impl Eq for Mask -where - T: MaskElement + Eq, - LaneCount: SupportedLaneCount, -{ -} - -impl Ord for Mask -where - T: MaskElement + Ord, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.0.cmp(&other.0) - } -} - -// Used for bitmask bit order workaround -pub(crate) trait ReverseBits { - // Reverse the least significant `n` bits of `self`. - // (Remaining bits must be 0.) - fn reverse_bits(self, n: usize) -> Self; -} - -macro_rules! impl_reverse_bits { - { $($int:ty),* } => { - $( - impl ReverseBits for $int { - #[inline(always)] - fn reverse_bits(self, n: usize) -> Self { - let rev = <$int>::reverse_bits(self); - let bitsize = size_of::<$int>() * 8; - if n < bitsize { - // Shift things back to the right - rev >> (bitsize - n) - } else { - rev - } - } - } - )* - } -} - -impl_reverse_bits! { u8, u16, u32, u64 } - -impl Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - #[rustc_const_unstable(feature = "portable_simd", issue = "86656")] - pub(crate) const fn splat(value: bool) -> Self { - Self(Simd::splat(if value { T::TRUE } else { T::FALSE })) - } - - #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] - pub(crate) unsafe fn test_unchecked(&self, lane: usize) -> bool { - T::eq(self.0[lane], T::TRUE) - } - - #[inline] - pub(crate) unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { - self.0[lane] = if value { T::TRUE } else { T::FALSE } - } - - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub(crate) fn to_simd(self) -> Simd { - self.0 - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) unsafe fn from_simd_unchecked(value: Simd) -> Self { - Self(value) - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub(crate) fn convert(self) -> Mask - where - U: MaskElement, - { - // Safety: masks are simply integer vectors of 0 and -1, and we can cast the element type. - unsafe { Mask(core::intrinsics::simd::simd_cast(self.0)) } - } - - #[inline] - unsafe fn to_bitmask_impl(self) -> U - where - LaneCount: SupportedLaneCount, - { - let resized = self.to_simd().resize::(T::FALSE); - - // Safety: `resized` is an integer vector with length M, which must match T - let bitmask: U = unsafe { core::intrinsics::simd::simd_bitmask(resized) }; - - // LLVM assumes bit order should match endianness - if cfg!(target_endian = "big") { - bitmask.reverse_bits(M) - } else { - bitmask - } - } - - #[inline] - unsafe fn from_bitmask_impl(bitmask: U) -> Self - where - LaneCount: SupportedLaneCount, - { - // LLVM assumes bit order should match endianness - let bitmask = if cfg!(target_endian = "big") { - bitmask.reverse_bits(M) - } else { - bitmask - }; - - // SAFETY: `mask` is the correct bitmask type for a u64 bitmask - let mask: Simd = unsafe { - core::intrinsics::simd::simd_select_bitmask( - bitmask, - Simd::::splat(T::TRUE), - Simd::::splat(T::FALSE), - ) - }; - - // SAFETY: `mask` only contains `T::TRUE` or `T::FALSE` - unsafe { Self::from_simd_unchecked(mask.resize::(T::FALSE)) } - } - - #[inline] - pub(crate) fn to_bitmask_integer(self) -> u64 { - // TODO modify simd_bitmask to zero-extend output, making this unnecessary - if N <= 8 { - // Safety: bitmask matches length - unsafe { self.to_bitmask_impl::() as u64 } - } else if N <= 16 { - // Safety: bitmask matches length - unsafe { self.to_bitmask_impl::() as u64 } - } else if N <= 32 { - // Safety: bitmask matches length - unsafe { self.to_bitmask_impl::() as u64 } - } else { - // Safety: bitmask matches length - unsafe { self.to_bitmask_impl::() } - } - } - - #[inline] - pub(crate) fn from_bitmask_integer(bitmask: u64) -> Self { - // TODO modify simd_bitmask_select to truncate input, making this unnecessary - if N <= 8 { - // Safety: bitmask matches length - unsafe { Self::from_bitmask_impl::(bitmask as u8) } - } else if N <= 16 { - // Safety: bitmask matches length - unsafe { Self::from_bitmask_impl::(bitmask as u16) } - } else if N <= 32 { - // Safety: bitmask matches length - unsafe { Self::from_bitmask_impl::(bitmask as u32) } - } else { - // Safety: bitmask matches length - unsafe { Self::from_bitmask_impl::(bitmask) } - } - } - - #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] - pub(crate) fn any(self) -> bool { - // Safety: use `self` as an integer vector - unsafe { core::intrinsics::simd::simd_reduce_any(self.to_simd()) } - } - - #[inline] - #[must_use = "method returns a new bool and does not mutate the original value"] - pub(crate) fn all(self) -> bool { - // Safety: use `self` as an integer vector - unsafe { core::intrinsics::simd::simd_reduce_all(self.to_simd()) } - } -} - -impl From> for Simd -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - #[inline] - fn from(value: Mask) -> Self { - value.0 - } -} - -impl core::ops::BitAnd for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - type Output = Self; - #[inline] - fn bitand(self, rhs: Self) -> Self { - // Safety: `self` is an integer vector - unsafe { Self(core::intrinsics::simd::simd_and(self.0, rhs.0)) } - } -} - -impl core::ops::BitOr for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - type Output = Self; - #[inline] - fn bitor(self, rhs: Self) -> Self { - // Safety: `self` is an integer vector - unsafe { Self(core::intrinsics::simd::simd_or(self.0, rhs.0)) } - } -} - -impl core::ops::BitXor for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - type Output = Self; - #[inline] - fn bitxor(self, rhs: Self) -> Self { - // Safety: `self` is an integer vector - unsafe { Self(core::intrinsics::simd::simd_xor(self.0, rhs.0)) } - } -} - -impl core::ops::Not for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ - type Output = Self; - #[inline] - fn not(self) -> Self::Output { - Self::splat(true) ^ self - } -} diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index 45b1a0f97514..14fe70df4ed1 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -29,6 +29,7 @@ pub mod simd { pub use crate::core_simd::cast::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; pub use crate::core_simd::masks::*; + pub use crate::core_simd::select::*; pub use crate::core_simd::swizzle::*; pub use crate::core_simd::to_bytes::ToBytes; pub use crate::core_simd::vector::*; diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index f36e8d01a73b..f36e360fadf9 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount, cmp::SimdPartialEq}; +use crate::simd::{LaneCount, Select, Simd, SimdElement, SupportedLaneCount, cmp::SimdPartialEq}; use core::ops::{Add, Mul}; use core::ops::{BitAnd, BitOr, BitXor}; use core::ops::{Div, Rem, Sub}; diff --git a/crates/core_simd/src/select.rs b/crates/core_simd/src/select.rs index a2db455a5268..5240b9b0c716 100644 --- a/crates/core_simd/src/select.rs +++ b/crates/core_simd/src/select.rs @@ -1,54 +1,163 @@ -use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{ + FixEndianness, LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount, +}; -impl Mask +/// Choose elements from two vectors using a mask. +/// +/// For each element in the mask, choose the corresponding element from `true_values` if +/// that element mask is true, and `false_values` if that element mask is false. +/// +/// If the mask is `u64`, it's treated as a bitmask with the least significant bit +/// corresponding to the first element. +/// +/// # Examples +/// +/// ## Selecting values from `Simd` +/// ``` +/// # #![feature(portable_simd)] +/// # #[cfg(feature = "as_crate")] use core_simd::simd; +/// # #[cfg(not(feature = "as_crate"))] use core::simd; +/// # use simd::{Simd, Mask, Select}; +/// let a = Simd::from_array([0, 1, 2, 3]); +/// let b = Simd::from_array([4, 5, 6, 7]); +/// let mask = Mask::::from_array([true, false, false, true]); +/// let c = mask.select(a, b); +/// assert_eq!(c.to_array(), [0, 5, 6, 3]); +/// ``` +/// +/// ## Selecting values from `Mask` +/// ``` +/// # #![feature(portable_simd)] +/// # #[cfg(feature = "as_crate")] use core_simd::simd; +/// # #[cfg(not(feature = "as_crate"))] use core::simd; +/// # use simd::{Mask, Select}; +/// let a = Mask::::from_array([true, true, false, false]); +/// let b = Mask::::from_array([false, false, true, true]); +/// let mask = Mask::::from_array([true, false, false, true]); +/// let c = mask.select(a, b); +/// assert_eq!(c.to_array(), [true, false, true, false]); +/// ``` +/// +/// ## Selecting with a bitmask +/// ``` +/// # #![feature(portable_simd)] +/// # #[cfg(feature = "as_crate")] use core_simd::simd; +/// # #[cfg(not(feature = "as_crate"))] use core::simd; +/// # use simd::{Mask, Select}; +/// let a = Mask::::from_array([true, true, false, false]); +/// let b = Mask::::from_array([false, false, true, true]); +/// let mask = 0b1001; +/// let c = mask.select(a, b); +/// assert_eq!(c.to_array(), [true, false, true, false]); +/// ``` +pub trait Select { + /// Choose elements + fn select(self, true_values: T, false_values: T) -> T; +} + +impl Select> for Mask +where + T: SimdElement, + U: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn select(self, true_values: Simd, false_values: Simd) -> Simd { + // Safety: + // simd_as between masks is always safe (they're vectors of ints). + // simd_select uses a mask that matches the width and number of elements + unsafe { + let mask: Simd = core::intrinsics::simd::simd_as(self.to_simd()); + core::intrinsics::simd::simd_select(mask, true_values, false_values) + } + } +} + +impl Select> for u64 +where + T: SimdElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn select(self, true_values: Simd, false_values: Simd) -> Simd { + const { + assert!(N <= 64, "number of elements can't be greater than 64"); + } + + #[inline] + unsafe fn select_impl( + bitmask: U, + true_values: Simd, + false_values: Simd, + ) -> Simd + where + T: SimdElement, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + { + let default = true_values[0]; + let true_values = true_values.resize::(default); + let false_values = false_values.resize::(default); + + // LLVM assumes bit order should match endianness + let bitmask = bitmask.fix_endianness(); + + // Safety: the caller guarantees that the size of U matches M + let selected = unsafe { + core::intrinsics::simd::simd_select_bitmask(bitmask, true_values, false_values) + }; + + selected.resize::(default) + } + + // TODO modify simd_bitmask_select to truncate input, making this unnecessary + if N <= 8 { + let bitmask = self as u8; + // Safety: bitmask matches length + unsafe { select_impl::(bitmask, true_values, false_values) } + } else if N <= 16 { + let bitmask = self as u16; + // Safety: bitmask matches length + unsafe { select_impl::(bitmask, true_values, false_values) } + } else if N <= 32 { + let bitmask = self as u32; + // Safety: bitmask matches length + unsafe { select_impl::(bitmask, true_values, false_values) } + } else { + let bitmask = self; + // Safety: bitmask matches length + unsafe { select_impl::(bitmask, true_values, false_values) } + } + } +} + +impl Select> for Mask +where + T: MaskElement, + U: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn select(self, true_values: Mask, false_values: Mask) -> Mask { + let selected: Simd = + Select::select(self, true_values.to_simd(), false_values.to_simd()); + + // Safety: all values come from masks + unsafe { Mask::from_simd_unchecked(selected) } + } +} + +impl Select> for u64 where T: MaskElement, LaneCount: SupportedLaneCount, { - /// Choose elements from two vectors. - /// - /// For each element in the mask, choose the corresponding element from `true_values` if - /// that element mask is true, and `false_values` if that element mask is false. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::{Simd, Mask}; - /// let a = Simd::from_array([0, 1, 2, 3]); - /// let b = Simd::from_array([4, 5, 6, 7]); - /// let mask = Mask::from_array([true, false, false, true]); - /// let c = mask.select(a, b); - /// assert_eq!(c.to_array(), [0, 5, 6, 3]); - /// ``` #[inline] - #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn select(self, true_values: Simd, false_values: Simd) -> Simd - where - U: SimdElement, - { - // Safety: The mask has been cast to a vector of integers, - // and the operands to select between are vectors of the same type and length. - unsafe { core::intrinsics::simd::simd_select(self.to_simd(), true_values, false_values) } - } + fn select(self, true_values: Mask, false_values: Mask) -> Mask { + let selected: Simd = + Select::select(self, true_values.to_simd(), false_values.to_simd()); - /// Choose elements from two masks. - /// - /// For each element in the mask, choose the corresponding element from `true_values` if - /// that element mask is true, and `false_values` if that element mask is false. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Mask; - /// let a = Mask::::from_array([true, true, false, false]); - /// let b = Mask::::from_array([false, false, true, true]); - /// let mask = Mask::::from_array([true, false, false, true]); - /// let c = mask.select_mask(a, b); - /// assert_eq!(c.to_array(), [true, false, true, false]); - /// ``` - #[inline] - #[must_use = "method returns a new mask and does not mutate the original inputs"] - pub fn select_mask(self, true_values: Self, false_values: Self) -> Self { - self & true_values | !self & false_values + // Safety: all values come from masks + unsafe { Mask::from_simd_unchecked(selected) } } } diff --git a/crates/core_simd/src/simd/cmp/ord.rs b/crates/core_simd/src/simd/cmp/ord.rs index 4b2d0b55feba..1b1c689ad458 100644 --- a/crates/core_simd/src/simd/cmp/ord.rs +++ b/crates/core_simd/src/simd/cmp/ord.rs @@ -1,5 +1,5 @@ use crate::simd::{ - LaneCount, Mask, Simd, SupportedLaneCount, + LaneCount, Mask, Select, Simd, SupportedLaneCount, cmp::SimdPartialEq, ptr::{SimdConstPtr, SimdMutPtr}, }; @@ -194,12 +194,12 @@ macro_rules! impl_mask { { #[inline] fn simd_max(self, other: Self) -> Self { - self.simd_gt(other).select_mask(other, self) + self.simd_gt(other).select(other, self) } #[inline] fn simd_min(self, other: Self) -> Self { - self.simd_lt(other).select_mask(other, self) + self.simd_lt(other).select(other, self) } #[inline] diff --git a/crates/core_simd/src/simd/num/float.rs b/crates/core_simd/src/simd/num/float.rs index b5972c47373b..76ab5748c638 100644 --- a/crates/core_simd/src/simd/num/float.rs +++ b/crates/core_simd/src/simd/num/float.rs @@ -1,6 +1,6 @@ use super::sealed::Sealed; use crate::simd::{ - LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, + LaneCount, Mask, Select, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::{SimdPartialEq, SimdPartialOrd}, }; diff --git a/crates/core_simd/src/simd/num/int.rs b/crates/core_simd/src/simd/num/int.rs index d25050c3e4b4..5a292407d050 100644 --- a/crates/core_simd/src/simd/num/int.rs +++ b/crates/core_simd/src/simd/num/int.rs @@ -1,6 +1,6 @@ use super::sealed::Sealed; use crate::simd::{ - LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::SimdOrd, + LaneCount, Mask, Select, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::SimdOrd, cmp::SimdPartialOrd, num::SimdUint, }; diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 773bd028bae0..73b18595d0a1 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -139,7 +139,7 @@ unsafe fn armv7_neon_swizzle_u8x16(bytes: Simd, idxs: Simd) -> S #[inline] #[allow(clippy::let_and_return)] unsafe fn avx2_pshufb(bytes: Simd, idxs: Simd) -> Simd { - use crate::simd::cmp::SimdPartialOrd; + use crate::simd::{Select, cmp::SimdPartialOrd}; #[cfg(target_arch = "x86")] use core::arch::x86; #[cfg(target_arch = "x86_64")] @@ -200,7 +200,7 @@ fn zeroing_idxs(idxs: Simd) -> Simd where LaneCount: SupportedLaneCount, { - use crate::simd::cmp::SimdPartialOrd; + use crate::simd::{Select, cmp::SimdPartialOrd}; idxs.simd_lt(Simd::splat(N as u8)) .select(idxs, Simd::splat(u8::MAX)) } From b031c516641c53577d021da1a89d0c953acdc23a Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Tue, 7 Oct 2025 05:52:37 -0400 Subject: [PATCH 0025/1843] slice/ascii: Optimize `eq_ignore_ascii_case` with auto-vectorization Refactor the current functionality into a helper function Use `as_chunks` to encourage auto-vectorization in the optimized chunk processing function Add a codegen test Add benches for `eq_ignore_ascii_case` The optimized function is initially only enabled for x86_64 which has `sse2` as part of its baseline, but none of the code is platform specific. Other platforms with SIMD instructions may also benefit from this implementation. Performance improvements only manifest for slices of 16 bytes or longer, so the optimized path is gated behind a length check for greater than or equal to 16. --- library/core/src/slice/ascii.rs | 43 ++++++++++++++ library/coretests/benches/ascii.rs | 1 + .../benches/ascii/eq_ignore_ascii_case.rs | 56 +++++++++++++++++++ .../lib-optimizations/eq_ignore_ascii_case.rs | 14 +++++ 4 files changed, 114 insertions(+) create mode 100644 library/coretests/benches/ascii/eq_ignore_ascii_case.rs create mode 100644 tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index e17a2e03d2dc..1f9ca4bc6698 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -60,6 +60,18 @@ impl [u8] { return false; } + #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] + if self.len() >= 16 { + return self.eq_ignore_ascii_case_chunks(other); + } + + self.eq_ignore_ascii_case_simple(other) + } + + /// ASCII case-insensitive equality check without chunk-at-a-time + /// optimization. + #[inline] + const fn eq_ignore_ascii_case_simple(&self, other: &[u8]) -> bool { // FIXME(const-hack): This implementation can be reverted when // `core::iter::zip` is allowed in const. The original implementation: // self.len() == other.len() && iter::zip(self, other).all(|(a, b)| a.eq_ignore_ascii_case(b)) @@ -78,6 +90,37 @@ impl [u8] { true } + /// Optimized version of `eq_ignore_ascii_case` which processes chunks at a + /// time. + /// + /// Platforms that have SIMD instructions may benefit from this + /// implementation over `eq_ignore_ascii_case_simple`. + #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] + #[inline] + const fn eq_ignore_ascii_case_chunks(&self, other: &[u8]) -> bool { + const N: usize = 16; + let (a, a_rem) = self.as_chunks::(); + let (b, b_rem) = other.as_chunks::(); + + let mut i = 0; + while i < a.len() && i < b.len() { + let mut equal_ascii = true; + let mut j = 0; + while j < N { + equal_ascii &= a[i][j].eq_ignore_ascii_case(&b[i][j]); + j += 1; + } + + if !equal_ascii { + return false; + } + + i += 1; + } + + a_rem.eq_ignore_ascii_case_simple(b_rem) + } + /// Converts this slice to its ASCII upper case equivalent in-place. /// /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', diff --git a/library/coretests/benches/ascii.rs b/library/coretests/benches/ascii.rs index 64bdc7fed118..17a520922bfa 100644 --- a/library/coretests/benches/ascii.rs +++ b/library/coretests/benches/ascii.rs @@ -1,3 +1,4 @@ +mod eq_ignore_ascii_case; mod is_ascii; // Lower-case ASCII 'a' is the first byte that has its highest bit set diff --git a/library/coretests/benches/ascii/eq_ignore_ascii_case.rs b/library/coretests/benches/ascii/eq_ignore_ascii_case.rs new file mode 100644 index 000000000000..a51acb1e8463 --- /dev/null +++ b/library/coretests/benches/ascii/eq_ignore_ascii_case.rs @@ -0,0 +1,56 @@ +use test::Bencher; + +#[bench] +fn bench_str_under_8_bytes_eq(b: &mut Bencher) { + let s = "foo"; + let other = "FOo"; + b.iter(|| { + assert!(s.eq_ignore_ascii_case(other)); + }) +} + +#[bench] +fn bench_str_of_8_bytes_eq(b: &mut Bencher) { + let s = "foobar78"; + let other = "FOObAr78"; + b.iter(|| { + assert!(s.eq_ignore_ascii_case(other)); + }) +} + +#[bench] +fn bench_str_17_bytes_eq(b: &mut Bencher) { + let s = "performance-criti"; + let other = "performANce-cRIti"; + b.iter(|| { + assert!(s.eq_ignore_ascii_case(other)); + }) +} + +#[bench] +fn bench_str_31_bytes_eq(b: &mut Bencher) { + let s = "foobarbazquux02foobarbazquux025"; + let other = "fooBARbazQuuX02fooBARbazQuuX025"; + b.iter(|| { + assert!(s.eq_ignore_ascii_case(other)); + }) +} + +#[bench] +fn bench_long_str_eq(b: &mut Bencher) { + let s = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor \ + incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud \ + exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute \ + irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla \ + pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui \ + officia deserunt mollit anim id est laborum."; + let other = "Lorem ipsum dolor sit amet, CONSECTETUR adipisicing elit, sed do eiusmod tempor \ + incididunt ut labore et dolore MAGNA aliqua. Ut enim ad MINIM veniam, quis nostrud \ + exercitation ullamco LABORIS nisi ut aliquip ex ea commodo consequat. Duis aute \ + irure dolor in reprehenderit in voluptate velit esse cillum DOLORE eu fugiat nulla \ + pariatur. Excepteur sint occaecat CUPIDATAT non proident, sunt in culpa qui \ + officia deserunt mollit anim id est laborum."; + b.iter(|| { + assert!(s.eq_ignore_ascii_case(other)); + }) +} diff --git a/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs b/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs new file mode 100644 index 000000000000..b733f1812c92 --- /dev/null +++ b/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs @@ -0,0 +1,14 @@ +//@ compile-flags: -Copt-level=3 +//@ only-x86_64 +#![crate_type = "lib"] + +// Ensure that the optimized variant of the function gets auto-vectorized. +// CHECK-LABEL: @eq_ignore_ascii_case_autovectorized +#[no_mangle] +pub fn eq_ignore_ascii_case_autovectorized(s: &str, other: &str) -> bool { + // CHECK: load <16 x i8> + // CHECK: load <16 x i8> + // CHECK: bitcast <16 x i1> + // CHECK-NOT: panic + s.eq_ignore_ascii_case(other) +} From 8ce88bcb509e1f7d7b6b76724a6fcb827d9a78ac Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Tue, 7 Oct 2025 10:16:49 -0400 Subject: [PATCH 0026/1843] Remove LaneCount in favor of #[rustc_simd_monomorphize_lane_limit] attribute (#485) * Remove LaneCount in favor of #[rustc_simd_monomorphize_lane_limit] attribute * Document allowed vector lengths --- crates/core_simd/src/fmt.rs | 3 +- crates/core_simd/src/iter.rs | 10 +---- crates/core_simd/src/lane_count.rs | 52 ---------------------- crates/core_simd/src/lib.rs | 3 +- crates/core_simd/src/masks.rs | 46 +++---------------- crates/core_simd/src/mod.rs | 2 - crates/core_simd/src/ops.rs | 5 +-- crates/core_simd/src/ops/assign.rs | 1 - crates/core_simd/src/ops/deref.rs | 3 -- crates/core_simd/src/ops/shift_scalar.rs | 10 +---- crates/core_simd/src/ops/unary.rs | 4 +- crates/core_simd/src/select.rs | 10 +---- crates/core_simd/src/simd/cmp/eq.rs | 16 ++----- crates/core_simd/src/simd/cmp/ord.rs | 32 +++---------- crates/core_simd/src/simd/num/float.rs | 10 +---- crates/core_simd/src/simd/num/int.rs | 14 ++---- crates/core_simd/src/simd/num/uint.rs | 10 +---- crates/core_simd/src/simd/ptr/const_ptr.rs | 9 ++-- crates/core_simd/src/simd/ptr/mut_ptr.rs | 9 ++-- crates/core_simd/src/swizzle.rs | 32 +++---------- crates/core_simd/src/swizzle_dyn.rs | 17 ++----- crates/core_simd/src/to_bytes.rs | 4 +- crates/core_simd/src/vector.rs | 43 +++--------------- crates/std_float/src/lib.rs | 10 ++--- crates/test_helpers/src/lib.rs | 4 -- rust-toolchain.toml | 2 +- 26 files changed, 57 insertions(+), 304 deletions(-) delete mode 100644 crates/core_simd/src/lane_count.rs diff --git a/crates/core_simd/src/fmt.rs b/crates/core_simd/src/fmt.rs index 3a540f5a0490..90c520e75bb3 100644 --- a/crates/core_simd/src/fmt.rs +++ b/crates/core_simd/src/fmt.rs @@ -1,9 +1,8 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{Simd, SimdElement}; use core::fmt; impl fmt::Debug for Simd where - LaneCount: SupportedLaneCount, T: SimdElement + fmt::Debug, { /// A `Simd` has a debug format like the one for `[T]`: diff --git a/crates/core_simd/src/iter.rs b/crates/core_simd/src/iter.rs index b3732fd74d5f..fdc458efeda4 100644 --- a/crates/core_simd/src/iter.rs +++ b/crates/core_simd/src/iter.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Simd, SupportedLaneCount}; +use crate::simd::Simd; use core::{ iter::{Product, Sum}, ops::{Add, Mul}, @@ -7,8 +7,6 @@ use core::{ macro_rules! impl_traits { { $type:ty } => { impl Sum for Simd<$type, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn sum>(iter: I) -> Self { @@ -17,8 +15,6 @@ macro_rules! impl_traits { } impl Product for Simd<$type, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn product>(iter: I) -> Self { @@ -27,8 +23,6 @@ macro_rules! impl_traits { } impl<'a, const N: usize> Sum<&'a Self> for Simd<$type, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn sum>(iter: I) -> Self { @@ -37,8 +31,6 @@ macro_rules! impl_traits { } impl<'a, const N: usize> Product<&'a Self> for Simd<$type, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn product>(iter: I) -> Self { diff --git a/crates/core_simd/src/lane_count.rs b/crates/core_simd/src/lane_count.rs deleted file mode 100644 index 839195c38eda..000000000000 --- a/crates/core_simd/src/lane_count.rs +++ /dev/null @@ -1,52 +0,0 @@ -mod sealed { - pub trait Sealed {} -} -use sealed::Sealed; - -/// Specifies the number of lanes in a SIMD vector as a type. -pub struct LaneCount; - -impl LaneCount { - /// The number of bytes in a bitmask with this many lanes. - pub const BITMASK_LEN: usize = N.div_ceil(8); -} - -/// Statically guarantees that a lane count is marked as supported. -/// -/// This trait is *sealed*: the list of implementors below is total. -/// Users do not have the ability to mark additional `LaneCount` values as supported. -/// Only SIMD vectors with supported lane counts are constructable. -pub trait SupportedLaneCount: Sealed { - #[doc(hidden)] - type BitMask: Copy + AsRef<[u8]> + AsMut<[u8]>; - #[doc(hidden)] - const EMPTY_BIT_MASK: Self::BitMask; - #[doc(hidden)] - const FULL_BIT_MASK: Self::BitMask; -} - -impl Sealed for LaneCount {} - -macro_rules! supported_lane_count { - ($($lanes:literal),+) => { - $( - impl SupportedLaneCount for LaneCount<$lanes> { - type BitMask = [u8; Self::BITMASK_LEN]; - const EMPTY_BIT_MASK: Self::BitMask = [0; Self::BITMASK_LEN]; - const FULL_BIT_MASK: Self::BitMask = { - let mut array = [!0u8; Self::BITMASK_LEN]; - if $lanes % 8 > 0 { - array[Self::BITMASK_LEN - 1] = (!0) >> (8 - $lanes % 8); - } - array - }; - } - )+ - }; -} - -supported_lane_count!( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 -); diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 9d7dfd2ab351..3e5ebe19e4db 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -9,7 +9,8 @@ simd_ffi, staged_api, prelude_import, - ptr_metadata + ptr_metadata, + rustc_attrs )] #![cfg_attr( all( diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 7baa96475910..3e2209556b66 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -2,7 +2,7 @@ //! Types representing #![allow(non_camel_case_types)] -use crate::simd::{LaneCount, Select, Simd, SimdCast, SimdElement, SupportedLaneCount}; +use crate::simd::{Select, Simd, SimdCast, SimdElement}; use core::cmp::Ordering; use core::{fmt, mem}; @@ -41,7 +41,6 @@ mod sealed { pub trait Sealed { fn valid(values: Simd) -> bool where - LaneCount: SupportedLaneCount, Self: SimdElement; fn eq(self, other: Self) -> bool; @@ -69,8 +68,6 @@ macro_rules! impl_element { impl Sealed for $ty { #[inline] fn valid(value: Simd) -> bool - where - LaneCount: SupportedLaneCount, { // We can't use `Simd` directly, because `Simd`'s functions call this function and // we will end up with an infinite loop. @@ -121,23 +118,19 @@ impl_element! { isize, usize } /// The layout of this type is unspecified, and may change between platforms /// and/or Rust versions, and code should not assume that it is equivalent to /// `[T; N]`. +/// +/// `N` cannot be 0 and may be at most 64. This limit may be increased in +/// the future. #[repr(transparent)] pub struct Mask(Simd) where - T: MaskElement, - LaneCount: SupportedLaneCount; + T: MaskElement; -impl Copy for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ -} +impl Copy for Mask where T: MaskElement {} impl Clone for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn clone(&self) -> Self { @@ -148,7 +141,6 @@ where impl Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { /// Constructs a mask by setting all elements to the given value. #[inline] @@ -315,8 +307,6 @@ where ) -> U where T: MaskElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, { let resized = mask.resize::(false); @@ -421,7 +411,6 @@ where impl From<[bool; N]> for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn from(array: [bool; N]) -> Self { @@ -432,7 +421,6 @@ where impl From> for [bool; N] where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn from(vector: Mask) -> Self { @@ -443,7 +431,6 @@ where impl Default for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn default() -> Self { @@ -454,7 +441,6 @@ where impl PartialEq for Mask where T: MaskElement + PartialEq, - LaneCount: SupportedLaneCount, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -465,7 +451,6 @@ where impl PartialOrd for Mask where T: MaskElement + PartialOrd, - LaneCount: SupportedLaneCount, { #[inline] fn partial_cmp(&self, other: &Self) -> Option { @@ -476,7 +461,6 @@ where impl fmt::Debug for Mask where T: MaskElement + fmt::Debug, - LaneCount: SupportedLaneCount, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -489,7 +473,6 @@ where impl core::ops::BitAnd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -502,7 +485,6 @@ where impl core::ops::BitAnd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -514,7 +496,6 @@ where impl core::ops::BitAnd> for bool where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Mask; #[inline] @@ -526,7 +507,6 @@ where impl core::ops::BitOr for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -539,7 +519,6 @@ where impl core::ops::BitOr for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -551,7 +530,6 @@ where impl core::ops::BitOr> for bool where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Mask; #[inline] @@ -563,7 +541,6 @@ where impl core::ops::BitXor for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -576,7 +553,6 @@ where impl core::ops::BitXor for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -588,7 +564,6 @@ where impl core::ops::BitXor> for bool where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Mask; #[inline] @@ -600,7 +575,6 @@ where impl core::ops::Not for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { type Output = Mask; #[inline] @@ -612,7 +586,6 @@ where impl core::ops::BitAndAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn bitand_assign(&mut self, rhs: Self) { @@ -623,7 +596,6 @@ where impl core::ops::BitAndAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn bitand_assign(&mut self, rhs: bool) { @@ -634,7 +606,6 @@ where impl core::ops::BitOrAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn bitor_assign(&mut self, rhs: Self) { @@ -645,7 +616,6 @@ where impl core::ops::BitOrAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn bitor_assign(&mut self, rhs: bool) { @@ -656,7 +626,6 @@ where impl core::ops::BitXorAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { @@ -667,7 +636,6 @@ where impl core::ops::BitXorAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn bitxor_assign(&mut self, rhs: bool) { @@ -679,8 +647,6 @@ macro_rules! impl_from { { $from:ty => $($to:ty),* } => { $( impl From> for Mask<$to, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn from(value: Mask<$from, N>) -> Self { diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index 14fe70df4ed1..5f635d80a178 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -5,7 +5,6 @@ mod alias; mod cast; mod fmt; mod iter; -mod lane_count; mod masks; mod ops; mod select; @@ -27,7 +26,6 @@ pub mod simd { pub use crate::core_simd::alias::*; pub use crate::core_simd::cast::*; - pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; pub use crate::core_simd::masks::*; pub use crate::core_simd::select::*; pub use crate::core_simd::swizzle::*; diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index f36e360fadf9..eb6601f73483 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Select, Simd, SimdElement, SupportedLaneCount, cmp::SimdPartialEq}; +use crate::simd::{Select, Simd, SimdElement, cmp::SimdPartialEq}; use core::ops::{Add, Mul}; use core::ops::{BitAnd, BitOr, BitXor}; use core::ops::{Div, Rem, Sub}; @@ -12,7 +12,6 @@ mod unary; impl core::ops::Index for Simd where T: SimdElement, - LaneCount: SupportedLaneCount, I: core::slice::SliceIndex<[T]>, { type Output = I::Output; @@ -25,7 +24,6 @@ where impl core::ops::IndexMut for Simd where T: SimdElement, - LaneCount: SupportedLaneCount, I: core::slice::SliceIndex<[T]>, { #[inline] @@ -130,7 +128,6 @@ macro_rules! for_base_types { impl $op for Simd<$scalar, N> where $scalar: SimdElement, - LaneCount: SupportedLaneCount, { type Output = $out; diff --git a/crates/core_simd/src/ops/assign.rs b/crates/core_simd/src/ops/assign.rs index d21d867de26d..c1830c35df77 100644 --- a/crates/core_simd/src/ops/assign.rs +++ b/crates/core_simd/src/ops/assign.rs @@ -21,7 +21,6 @@ macro_rules! assign_ops { where Self: $trait, T: SimdElement, - LaneCount: SupportedLaneCount, { #[inline] fn $assign_call(&mut self, rhs: U) { diff --git a/crates/core_simd/src/ops/deref.rs b/crates/core_simd/src/ops/deref.rs index 913cbbe977c4..360b83c40346 100644 --- a/crates/core_simd/src/ops/deref.rs +++ b/crates/core_simd/src/ops/deref.rs @@ -13,7 +13,6 @@ macro_rules! deref_lhs { where T: SimdElement, $simd: $trait<$simd, Output = $simd>, - LaneCount: SupportedLaneCount, { type Output = Simd; @@ -33,7 +32,6 @@ macro_rules! deref_rhs { where T: SimdElement, $simd: $trait<$simd, Output = $simd>, - LaneCount: SupportedLaneCount, { type Output = Simd; @@ -64,7 +62,6 @@ macro_rules! deref_ops { where T: SimdElement, $simd: $trait<$simd, Output = $simd>, - LaneCount: SupportedLaneCount, { type Output = $simd; diff --git a/crates/core_simd/src/ops/shift_scalar.rs b/crates/core_simd/src/ops/shift_scalar.rs index f5115a5a5e93..7ca83dc40f61 100644 --- a/crates/core_simd/src/ops/shift_scalar.rs +++ b/crates/core_simd/src/ops/shift_scalar.rs @@ -1,13 +1,11 @@ // Shift operations uniquely typically only have a scalar on the right-hand side. // Here, we implement shifts for scalar RHS arguments. -use crate::simd::{LaneCount, Simd, SupportedLaneCount}; +use crate::simd::Simd; macro_rules! impl_splatted_shifts { { impl $trait:ident :: $trait_fn:ident for $ty:ty } => { impl core::ops::$trait<$ty> for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -17,8 +15,6 @@ macro_rules! impl_splatted_shifts { } impl core::ops::$trait<&$ty> for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -28,8 +24,6 @@ macro_rules! impl_splatted_shifts { } impl<'lhs, const N: usize> core::ops::$trait<$ty> for &'lhs Simd<$ty, N> - where - LaneCount: SupportedLaneCount, { type Output = Simd<$ty, N>; #[inline] @@ -39,8 +33,6 @@ macro_rules! impl_splatted_shifts { } impl<'lhs, const N: usize> core::ops::$trait<&$ty> for &'lhs Simd<$ty, N> - where - LaneCount: SupportedLaneCount, { type Output = Simd<$ty, N>; #[inline] diff --git a/crates/core_simd/src/ops/unary.rs b/crates/core_simd/src/ops/unary.rs index 412a5b801171..e1c06167f979 100644 --- a/crates/core_simd/src/ops/unary.rs +++ b/crates/core_simd/src/ops/unary.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{Simd, SimdElement}; use core::ops::{Neg, Not}; // unary ops macro_rules! neg { @@ -6,7 +6,6 @@ macro_rules! neg { $(impl Neg for Simd<$scalar, N> where $scalar: SimdElement, - LaneCount: SupportedLaneCount, { type Output = Self; @@ -40,7 +39,6 @@ macro_rules! not { $(impl Not for Simd<$scalar, N> where $scalar: SimdElement, - LaneCount: SupportedLaneCount, { type Output = Self; diff --git a/crates/core_simd/src/select.rs b/crates/core_simd/src/select.rs index 5240b9b0c716..404f54d8f382 100644 --- a/crates/core_simd/src/select.rs +++ b/crates/core_simd/src/select.rs @@ -1,6 +1,4 @@ -use crate::simd::{ - FixEndianness, LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount, -}; +use crate::simd::{FixEndianness, Mask, MaskElement, Simd, SimdElement}; /// Choose elements from two vectors using a mask. /// @@ -59,7 +57,6 @@ impl Select> for Mask where T: SimdElement, U: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn select(self, true_values: Simd, false_values: Simd) -> Simd { @@ -76,7 +73,6 @@ where impl Select> for u64 where T: SimdElement, - LaneCount: SupportedLaneCount, { #[inline] fn select(self, true_values: Simd, false_values: Simd) -> Simd { @@ -92,8 +88,6 @@ where ) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, { let default = true_values[0]; let true_values = true_values.resize::(default); @@ -135,7 +129,6 @@ impl Select> for Mask where T: MaskElement, U: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn select(self, true_values: Mask, false_values: Mask) -> Mask { @@ -150,7 +143,6 @@ where impl Select> for u64 where T: MaskElement, - LaneCount: SupportedLaneCount, { #[inline] fn select(self, true_values: Mask, false_values: Mask) -> Mask { diff --git a/crates/core_simd/src/simd/cmp/eq.rs b/crates/core_simd/src/simd/cmp/eq.rs index 789fc0bb9424..d553d6c040c9 100644 --- a/crates/core_simd/src/simd/cmp/eq.rs +++ b/crates/core_simd/src/simd/cmp/eq.rs @@ -1,5 +1,5 @@ use crate::simd::{ - LaneCount, Mask, Simd, SimdElement, SupportedLaneCount, + Mask, Simd, SimdElement, ptr::{SimdConstPtr, SimdMutPtr}, }; @@ -21,8 +21,6 @@ macro_rules! impl_number { { $($number:ty),* } => { $( impl SimdPartialEq for Simd<$number, N> - where - LaneCount: SupportedLaneCount, { type Mask = Mask<<$number as SimdElement>::Mask, N>; @@ -50,8 +48,6 @@ macro_rules! impl_mask { { $($integer:ty),* } => { $( impl SimdPartialEq for Mask<$integer, N> - where - LaneCount: SupportedLaneCount, { type Mask = Self; @@ -75,10 +71,7 @@ macro_rules! impl_mask { impl_mask! { i8, i16, i32, i64, isize } -impl SimdPartialEq for Simd<*const T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdPartialEq for Simd<*const T, N> { type Mask = Mask; #[inline] @@ -92,10 +85,7 @@ where } } -impl SimdPartialEq for Simd<*mut T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdPartialEq for Simd<*mut T, N> { type Mask = Mask; #[inline] diff --git a/crates/core_simd/src/simd/cmp/ord.rs b/crates/core_simd/src/simd/cmp/ord.rs index 1b1c689ad458..5672fbbf54ca 100644 --- a/crates/core_simd/src/simd/cmp/ord.rs +++ b/crates/core_simd/src/simd/cmp/ord.rs @@ -1,5 +1,5 @@ use crate::simd::{ - LaneCount, Mask, Select, Simd, SupportedLaneCount, + Mask, Select, Simd, cmp::SimdPartialEq, ptr::{SimdConstPtr, SimdMutPtr}, }; @@ -49,8 +49,6 @@ macro_rules! impl_integer { { $($integer:ty),* } => { $( impl SimdPartialOrd for Simd<$integer, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -82,8 +80,6 @@ macro_rules! impl_integer { } impl SimdOrd for Simd<$integer, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn simd_max(self, other: Self) -> Self { @@ -115,8 +111,6 @@ macro_rules! impl_float { { $($float:ty),* } => { $( impl SimdPartialOrd for Simd<$float, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -156,8 +150,6 @@ macro_rules! impl_mask { { $($integer:ty),* } => { $( impl SimdPartialOrd for Mask<$integer, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -189,8 +181,6 @@ macro_rules! impl_mask { } impl SimdOrd for Mask<$integer, N> - where - LaneCount: SupportedLaneCount, { #[inline] fn simd_max(self, other: Self) -> Self { @@ -218,10 +208,7 @@ macro_rules! impl_mask { impl_mask! { i8, i16, i32, i64, isize } -impl SimdPartialOrd for Simd<*const T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdPartialOrd for Simd<*const T, N> { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { self.addr().simd_lt(other.addr()) @@ -243,10 +230,7 @@ where } } -impl SimdOrd for Simd<*const T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdOrd for Simd<*const T, N> { #[inline] fn simd_max(self, other: Self) -> Self { self.simd_lt(other).select(other, self) @@ -268,10 +252,7 @@ where } } -impl SimdPartialOrd for Simd<*mut T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdPartialOrd for Simd<*mut T, N> { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { self.addr().simd_lt(other.addr()) @@ -293,10 +274,7 @@ where } } -impl SimdOrd for Simd<*mut T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdOrd for Simd<*mut T, N> { #[inline] fn simd_max(self, other: Self) -> Self { self.simd_lt(other).select(other, self) diff --git a/crates/core_simd/src/simd/num/float.rs b/crates/core_simd/src/simd/num/float.rs index 76ab5748c638..efd7c2469512 100644 --- a/crates/core_simd/src/simd/num/float.rs +++ b/crates/core_simd/src/simd/num/float.rs @@ -1,6 +1,6 @@ use super::sealed::Sealed; use crate::simd::{ - LaneCount, Mask, Select, Simd, SimdCast, SimdElement, SupportedLaneCount, + Mask, Select, Simd, SimdCast, SimdElement, cmp::{SimdPartialEq, SimdPartialOrd}, }; @@ -240,15 +240,9 @@ pub trait SimdFloat: Copy + Sealed { macro_rules! impl_trait { { $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => { $( - impl Sealed for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, - { - } + impl Sealed for Simd<$ty, N> {} impl SimdFloat for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, { type Mask = Mask<<$mask_ty as SimdElement>::Mask, N>; type Scalar = $ty; diff --git a/crates/core_simd/src/simd/num/int.rs b/crates/core_simd/src/simd/num/int.rs index 5a292407d050..6ebf0ba397c6 100644 --- a/crates/core_simd/src/simd/num/int.rs +++ b/crates/core_simd/src/simd/num/int.rs @@ -1,7 +1,6 @@ use super::sealed::Sealed; use crate::simd::{ - LaneCount, Mask, Select, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::SimdOrd, - cmp::SimdPartialOrd, num::SimdUint, + Mask, Select, Simd, SimdCast, SimdElement, cmp::SimdOrd, cmp::SimdPartialOrd, num::SimdUint, }; /// Operations on SIMD vectors of signed integers. @@ -241,16 +240,9 @@ pub trait SimdInt: Copy + Sealed { macro_rules! impl_trait { { $($ty:ident ($unsigned:ident)),* } => { $( - impl Sealed for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, - { - } + impl Sealed for Simd<$ty, N> {} - impl SimdInt for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, - { + impl SimdInt for Simd<$ty, N> { type Mask = Mask<<$ty as SimdElement>::Mask, N>; type Scalar = $ty; type Unsigned = Simd<$unsigned, N>; diff --git a/crates/core_simd/src/simd/num/uint.rs b/crates/core_simd/src/simd/num/uint.rs index 45d978068b66..f8a40f8ec565 100644 --- a/crates/core_simd/src/simd/num/uint.rs +++ b/crates/core_simd/src/simd/num/uint.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount, cmp::SimdOrd}; +use crate::simd::{Simd, SimdCast, SimdElement, cmp::SimdOrd}; /// Operations on SIMD vectors of unsigned integers. pub trait SimdUint: Copy + Sealed { @@ -123,15 +123,9 @@ pub trait SimdUint: Copy + Sealed { macro_rules! impl_trait { { $($ty:ident ($signed:ident)),* } => { $( - impl Sealed for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, - { - } + impl Sealed for Simd<$ty, N> {} impl SimdUint for Simd<$ty, N> - where - LaneCount: SupportedLaneCount, { type Scalar = $ty; type Cast = Simd; diff --git a/crates/core_simd/src/simd/ptr/const_ptr.rs b/crates/core_simd/src/simd/ptr/const_ptr.rs index 36452e7ae920..7ef9dc21373e 100644 --- a/crates/core_simd/src/simd/ptr/const_ptr.rs +++ b/crates/core_simd/src/simd/ptr/const_ptr.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount, cmp::SimdPartialEq, num::SimdUint}; +use crate::simd::{Mask, Simd, cmp::SimdPartialEq, num::SimdUint}; /// Operations on SIMD vectors of constant pointers. pub trait SimdConstPtr: Copy + Sealed { @@ -88,12 +88,9 @@ pub trait SimdConstPtr: Copy + Sealed { fn wrapping_sub(self, count: Self::Usize) -> Self; } -impl Sealed for Simd<*const T, N> where LaneCount: SupportedLaneCount {} +impl Sealed for Simd<*const T, N> {} -impl SimdConstPtr for Simd<*const T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdConstPtr for Simd<*const T, N> { type Usize = Simd; type Isize = Simd; type CastPtr = Simd<*const U, N>; diff --git a/crates/core_simd/src/simd/ptr/mut_ptr.rs b/crates/core_simd/src/simd/ptr/mut_ptr.rs index c644f390c20a..3b9b75ddf566 100644 --- a/crates/core_simd/src/simd/ptr/mut_ptr.rs +++ b/crates/core_simd/src/simd/ptr/mut_ptr.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount, cmp::SimdPartialEq, num::SimdUint}; +use crate::simd::{Mask, Simd, cmp::SimdPartialEq, num::SimdUint}; /// Operations on SIMD vectors of mutable pointers. pub trait SimdMutPtr: Copy + Sealed { @@ -85,12 +85,9 @@ pub trait SimdMutPtr: Copy + Sealed { fn wrapping_sub(self, count: Self::Usize) -> Self; } -impl Sealed for Simd<*mut T, N> where LaneCount: SupportedLaneCount {} +impl Sealed for Simd<*mut T, N> {} -impl SimdMutPtr for Simd<*mut T, N> -where - LaneCount: SupportedLaneCount, -{ +impl SimdMutPtr for Simd<*mut T, N> { type Usize = Simd; type Isize = Simd; type CastPtr = Simd<*mut U, N>; diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 81085a9ee4a3..02dcd71356dd 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{Mask, MaskElement, Simd, SimdElement}; /// Constructs a new SIMD vector by copying elements from selected elements in other vectors. /// @@ -82,8 +82,6 @@ pub trait Swizzle { fn swizzle(vector: Simd) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, { // Safety: `vector` is a vector, and the index is a const vector of u32. unsafe { @@ -122,8 +120,6 @@ pub trait Swizzle { fn concat_swizzle(first: Simd, second: Simd) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, { // Safety: `first` and `second` are vectors, and the index is a const vector of u32. unsafe { @@ -161,8 +157,6 @@ pub trait Swizzle { fn swizzle_mask(mask: Mask) -> Mask where T: MaskElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, { // SAFETY: all elements of this mask come from another mask unsafe { Mask::from_simd_unchecked(Self::swizzle(mask.to_simd())) } @@ -177,8 +171,6 @@ pub trait Swizzle { fn concat_swizzle_mask(first: Mask, second: Mask) -> Mask where T: MaskElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, { // SAFETY: all elements of this mask come from another mask unsafe { @@ -190,7 +182,6 @@ pub trait Swizzle { impl Simd where T: SimdElement, - LaneCount: SupportedLaneCount, { /// Reverse the order of the elements in the vector. #[inline] @@ -464,10 +455,7 @@ where /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn resize(self, value: T) -> Simd - where - LaneCount: SupportedLaneCount, - { + pub fn resize(self, value: T) -> Simd { struct Resize; impl Swizzle for Resize { const INDEX: [usize; M] = const { @@ -495,10 +483,7 @@ where /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn extract(self) -> Simd - where - LaneCount: SupportedLaneCount, - { + pub fn extract(self) -> Simd { struct Extract; impl Swizzle for Extract { const INDEX: [usize; LEN] = const { @@ -519,7 +504,6 @@ where impl Mask where T: MaskElement, - LaneCount: SupportedLaneCount, { /// Reverse the order of the elements in the mask. #[inline] @@ -655,10 +639,7 @@ where /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn resize(self, value: bool) -> Mask - where - LaneCount: SupportedLaneCount, - { + pub fn resize(self, value: bool) -> Mask { // Safety: swizzles are safe for masks unsafe { Mask::::from_simd_unchecked(self.to_simd().resize::(if value { @@ -681,10 +662,7 @@ where /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn extract(self) -> Mask - where - LaneCount: SupportedLaneCount, - { + pub fn extract(self) -> Mask { // Safety: swizzles are safe for masks unsafe { Mask::::from_simd_unchecked(self.to_simd().extract::()) } } diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 73b18595d0a1..ae0b174973da 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -1,10 +1,7 @@ -use crate::simd::{LaneCount, Simd, SupportedLaneCount}; +use crate::simd::Simd; use core::mem; -impl Simd -where - LaneCount: SupportedLaneCount, -{ +impl Simd { /// Swizzle a vector of bytes according to the index vector. /// Indices within range select the appropriate byte. /// Indices "out of bounds" instead select 0. @@ -184,10 +181,7 @@ unsafe fn transize( f: unsafe fn(T, T) -> T, a: Simd, b: Simd, -) -> Simd -where - LaneCount: SupportedLaneCount, -{ +) -> Simd { // SAFETY: Same obligation to use this function as to use mem::transmute_copy. unsafe { mem::transmute_copy(&f(mem::transmute_copy(&a), mem::transmute_copy(&b))) } } @@ -196,10 +190,7 @@ where #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[allow(unused)] #[inline(always)] -fn zeroing_idxs(idxs: Simd) -> Simd -where - LaneCount: SupportedLaneCount, -{ +fn zeroing_idxs(idxs: Simd) -> Simd { use crate::simd::{Select, cmp::SimdPartialOrd}; idxs.simd_lt(Simd::splat(N as u8)) .select(idxs, Simd::splat(u8::MAX)) diff --git a/crates/core_simd/src/to_bytes.rs b/crates/core_simd/src/to_bytes.rs index fee2cc06c5b0..1fd285e457db 100644 --- a/crates/core_simd/src/to_bytes.rs +++ b/crates/core_simd/src/to_bytes.rs @@ -1,12 +1,12 @@ use crate::simd::{ - LaneCount, Simd, SimdElement, SupportedLaneCount, + Simd, SimdElement, num::{SimdFloat, SimdInt, SimdUint}, }; mod sealed { use super::*; pub trait Sealed {} - impl Sealed for Simd where LaneCount: SupportedLaneCount {} + impl Sealed for Simd {} } use sealed::Sealed; diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index c00cfcdd41ff..2dba5c83e112 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -1,5 +1,5 @@ use crate::simd::{ - LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle, + Mask, MaskElement, Swizzle, cmp::SimdPartialOrd, num::SimdUint, ptr::{SimdConstPtr, SimdMutPtr}, @@ -51,6 +51,8 @@ use crate::simd::{ /// Thus it is sound to [`transmute`] `Simd` to `[T; N]` and should optimize to "zero cost", /// but the reverse transmutation may require a copy the compiler cannot simply elide. /// +/// `N` cannot be 0 and may be at most 64. This limit may be increased in the future. +/// /// # ABI "Features" /// Due to Rust's safety guarantees, `Simd` is currently passed and returned via memory, /// not SIMD registers, except as an optimization. Using `#[inline]` on functions that accept @@ -100,14 +102,13 @@ use crate::simd::{ // avoided, as it will likely become illegal on `#[repr(simd)]` structs in the future. It also // causes rustc to emit illegal LLVM IR in some cases. #[repr(simd, packed)] +#[rustc_simd_monomorphize_lane_limit = "64"] pub struct Simd([T; N]) where - LaneCount: SupportedLaneCount, T: SimdElement; impl Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { /// Number of elements in this vector. @@ -149,7 +150,6 @@ where const fn splat_const(value: T) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, { Simd::from_array([value; N]) } @@ -157,7 +157,6 @@ where fn splat_rt(value: T) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, { // This is preferred over `[value; N]`, since it's explicitly a splat: // https://github.com/rust-lang/rust/issues/97804 @@ -886,16 +885,10 @@ where } } -impl Copy for Simd -where - LaneCount: SupportedLaneCount, - T: SimdElement, -{ -} +impl Copy for Simd where T: SimdElement {} impl Clone for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -906,7 +899,6 @@ where impl Default for Simd where - LaneCount: SupportedLaneCount, T: SimdElement + Default, { #[inline] @@ -917,7 +909,6 @@ where impl PartialEq for Simd where - LaneCount: SupportedLaneCount, T: SimdElement + PartialEq, { #[inline] @@ -951,7 +942,6 @@ where /// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead. impl PartialOrd for Simd where - LaneCount: SupportedLaneCount, T: SimdElement + PartialOrd, { #[inline] @@ -961,17 +951,11 @@ where } } -impl Eq for Simd -where - LaneCount: SupportedLaneCount, - T: SimdElement + Eq, -{ -} +impl Eq for Simd where T: SimdElement + Eq {} /// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead. impl Ord for Simd where - LaneCount: SupportedLaneCount, T: SimdElement + Ord, { #[inline] @@ -983,7 +967,6 @@ where impl core::hash::Hash for Simd where - LaneCount: SupportedLaneCount, T: SimdElement + core::hash::Hash, { #[inline] @@ -998,7 +981,6 @@ where // array references impl AsRef<[T; N]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -1009,7 +991,6 @@ where impl AsMut<[T; N]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -1021,7 +1002,6 @@ where // slice references impl AsRef<[T]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -1032,7 +1012,6 @@ where impl AsMut<[T]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -1044,7 +1023,6 @@ where // vector/array conversion impl From<[T; N]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -1055,7 +1033,6 @@ where impl From> for [T; N] where - LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -1066,7 +1043,6 @@ where impl TryFrom<&[T]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { type Error = core::array::TryFromSliceError; @@ -1079,7 +1055,6 @@ where impl TryFrom<&mut [T]> for Simd where - LaneCount: SupportedLaneCount, T: SimdElement, { type Error = core::array::TryFromSliceError; @@ -1217,10 +1192,7 @@ where } #[inline] -fn lane_indices() -> Simd -where - LaneCount: SupportedLaneCount, -{ +fn lane_indices() -> Simd { #![allow(clippy::needless_range_loop)] let mut index = [0; N]; for i in 0..N { @@ -1232,7 +1204,6 @@ where #[inline] fn mask_up_to(len: usize) -> Mask where - LaneCount: SupportedLaneCount, M: MaskElement, { let index = lane_indices::(); diff --git a/crates/std_float/src/lib.rs b/crates/std_float/src/lib.rs index 148aa5f9f177..c3c9b76e50b8 100644 --- a/crates/std_float/src/lib.rs +++ b/crates/std_float/src/lib.rs @@ -11,7 +11,7 @@ use core_simd::simd; use core::intrinsics::simd as intrinsics; -use simd::{LaneCount, Simd, SupportedLaneCount}; +use simd::Simd; #[cfg(feature = "as_crate")] mod experimental { @@ -140,16 +140,14 @@ pub trait StdFloat: Sealed + Sized { fn fract(self) -> Self; } -impl Sealed for Simd where LaneCount: SupportedLaneCount {} -impl Sealed for Simd where LaneCount: SupportedLaneCount {} +impl Sealed for Simd {} +impl Sealed for Simd {} macro_rules! impl_float { { $($fn:ident: $intrinsic:ident,)* } => { impl StdFloat for Simd - where - LaneCount: SupportedLaneCount, { #[inline] fn fract(self) -> Self { @@ -165,8 +163,6 @@ macro_rules! impl_float { } impl StdFloat for Simd - where - LaneCount: SupportedLaneCount, { #[inline] fn fract(self) -> Self { diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 35401a9ddb40..eb3d3f68bc2e 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -604,8 +604,6 @@ macro_rules! test_lanes { use super::*; fn implementation() - where - core_simd::simd::LaneCount<$lanes>: core_simd::simd::SupportedLaneCount, $body #[cfg(target_arch = "wasm32")] @@ -704,8 +702,6 @@ macro_rules! test_lanes_panic { use super::*; fn implementation() - where - core_simd::simd::LaneCount<$lanes>: core_simd::simd::SupportedLaneCount, $body // test some odd and even non-power-of-2 lengths on miri diff --git a/rust-toolchain.toml b/rust-toolchain.toml index df4bd75ecfe2..ed4d7e8a801e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2025-08-17" +channel = "nightly-2025-09-30" components = ["rustfmt", "clippy", "miri", "rust-src"] From 7b88f48b530f6610581a0d023199c9563e635730 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Tue, 7 Oct 2025 15:35:02 -0400 Subject: [PATCH 0027/1843] Avoid scalar fallback in chunk remainder check Refactor the eq check into an inner function for reuse in tail checking Rather than fall back to the simple implementation for tail handling, load the last 16 bytes to take advantage of vectorization. This doesn't seem to negatively impact check time even when the remainder count is low. --- library/core/src/slice/ascii.rs | 35 +++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index 1f9ca4bc6698..3f3f5a8c441d 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -90,8 +90,8 @@ impl [u8] { true } - /// Optimized version of `eq_ignore_ascii_case` which processes chunks at a - /// time. + /// Optimized version of `eq_ignore_ascii_case` for byte lengths of at least + /// 16 bytes, which processes chunks at a time. /// /// Platforms that have SIMD instructions may benefit from this /// implementation over `eq_ignore_ascii_case_simple`. @@ -99,26 +99,41 @@ impl [u8] { #[inline] const fn eq_ignore_ascii_case_chunks(&self, other: &[u8]) -> bool { const N: usize = 16; - let (a, a_rem) = self.as_chunks::(); - let (b, b_rem) = other.as_chunks::(); + let (self_chunks, self_rem) = self.as_chunks::(); + let (other_chunks, _) = other.as_chunks::(); - let mut i = 0; - while i < a.len() && i < b.len() { + // Branchless check to encourage auto-vectorization + const fn eq_ignore_ascii_inner(lhs: &[u8; N], rhs: &[u8; N]) -> bool { let mut equal_ascii = true; let mut j = 0; while j < N { - equal_ascii &= a[i][j].eq_ignore_ascii_case(&b[i][j]); + equal_ascii &= lhs[j].eq_ignore_ascii_case(&rhs[j]); j += 1; } - if !equal_ascii { + equal_ascii + } + + // Process the chunks, returning early if an inequality is found + let mut i = 0; + while i < self_chunks.len() && i < other_chunks.len() { + if !eq_ignore_ascii_inner(&self_chunks[i], &other_chunks[i]) { return false; } - i += 1; } - a_rem.eq_ignore_ascii_case_simple(b_rem) + // If there are remaining tails, load the last N bytes in the slices to + // avoid falling back to per-byte checking. + if !self_rem.is_empty() { + if let (Some(a_rem), Some(b_rem)) = (self.last_chunk::(), other.last_chunk::()) { + if !eq_ignore_ascii_inner(a_rem, b_rem) { + return false; + } + } + } + + true } /// Converts this slice to its ASCII upper case equivalent in-place. From a5ba24843d6e4ceda580e49344bef73baec14204 Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Tue, 7 Oct 2025 19:04:32 -0400 Subject: [PATCH 0028/1843] Relocate bench and use str corpora for data Add #[inline(always)] to inner function and check not for filecheck test --- library/core/src/slice/ascii.rs | 1 + library/coretests/benches/ascii.rs | 1 - .../benches/ascii/eq_ignore_ascii_case.rs | 56 ------------------- library/coretests/benches/str.rs | 1 + .../benches/str/eq_ignore_ascii_case.rs | 45 +++++++++++++++ .../lib-optimizations/eq_ignore_ascii_case.rs | 4 +- 6 files changed, 50 insertions(+), 58 deletions(-) delete mode 100644 library/coretests/benches/ascii/eq_ignore_ascii_case.rs create mode 100644 library/coretests/benches/str/eq_ignore_ascii_case.rs diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index 3f3f5a8c441d..8b713947cd9c 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -103,6 +103,7 @@ impl [u8] { let (other_chunks, _) = other.as_chunks::(); // Branchless check to encourage auto-vectorization + #[inline(always)] const fn eq_ignore_ascii_inner(lhs: &[u8; N], rhs: &[u8; N]) -> bool { let mut equal_ascii = true; let mut j = 0; diff --git a/library/coretests/benches/ascii.rs b/library/coretests/benches/ascii.rs index 17a520922bfa..64bdc7fed118 100644 --- a/library/coretests/benches/ascii.rs +++ b/library/coretests/benches/ascii.rs @@ -1,4 +1,3 @@ -mod eq_ignore_ascii_case; mod is_ascii; // Lower-case ASCII 'a' is the first byte that has its highest bit set diff --git a/library/coretests/benches/ascii/eq_ignore_ascii_case.rs b/library/coretests/benches/ascii/eq_ignore_ascii_case.rs deleted file mode 100644 index a51acb1e8463..000000000000 --- a/library/coretests/benches/ascii/eq_ignore_ascii_case.rs +++ /dev/null @@ -1,56 +0,0 @@ -use test::Bencher; - -#[bench] -fn bench_str_under_8_bytes_eq(b: &mut Bencher) { - let s = "foo"; - let other = "FOo"; - b.iter(|| { - assert!(s.eq_ignore_ascii_case(other)); - }) -} - -#[bench] -fn bench_str_of_8_bytes_eq(b: &mut Bencher) { - let s = "foobar78"; - let other = "FOObAr78"; - b.iter(|| { - assert!(s.eq_ignore_ascii_case(other)); - }) -} - -#[bench] -fn bench_str_17_bytes_eq(b: &mut Bencher) { - let s = "performance-criti"; - let other = "performANce-cRIti"; - b.iter(|| { - assert!(s.eq_ignore_ascii_case(other)); - }) -} - -#[bench] -fn bench_str_31_bytes_eq(b: &mut Bencher) { - let s = "foobarbazquux02foobarbazquux025"; - let other = "fooBARbazQuuX02fooBARbazQuuX025"; - b.iter(|| { - assert!(s.eq_ignore_ascii_case(other)); - }) -} - -#[bench] -fn bench_long_str_eq(b: &mut Bencher) { - let s = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor \ - incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud \ - exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute \ - irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla \ - pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui \ - officia deserunt mollit anim id est laborum."; - let other = "Lorem ipsum dolor sit amet, CONSECTETUR adipisicing elit, sed do eiusmod tempor \ - incididunt ut labore et dolore MAGNA aliqua. Ut enim ad MINIM veniam, quis nostrud \ - exercitation ullamco LABORIS nisi ut aliquip ex ea commodo consequat. Duis aute \ - irure dolor in reprehenderit in voluptate velit esse cillum DOLORE eu fugiat nulla \ - pariatur. Excepteur sint occaecat CUPIDATAT non proident, sunt in culpa qui \ - officia deserunt mollit anim id est laborum."; - b.iter(|| { - assert!(s.eq_ignore_ascii_case(other)); - }) -} diff --git a/library/coretests/benches/str.rs b/library/coretests/benches/str.rs index 2f7d9d56a70b..bf45a8f0a79b 100644 --- a/library/coretests/benches/str.rs +++ b/library/coretests/benches/str.rs @@ -5,6 +5,7 @@ use test::{Bencher, black_box}; mod char_count; mod corpora; mod debug; +mod eq_ignore_ascii_case; mod iter; #[bench] diff --git a/library/coretests/benches/str/eq_ignore_ascii_case.rs b/library/coretests/benches/str/eq_ignore_ascii_case.rs new file mode 100644 index 000000000000..29129b933bc4 --- /dev/null +++ b/library/coretests/benches/str/eq_ignore_ascii_case.rs @@ -0,0 +1,45 @@ +use test::{Bencher, black_box}; + +use super::corpora::*; + +#[bench] +fn bench_str_under_8_bytes_eq(b: &mut Bencher) { + let s = black_box("foo"); + let other = black_box("foo"); + b.iter(|| assert!(s.eq_ignore_ascii_case(other))) +} + +#[bench] +fn bench_str_of_8_bytes_eq(b: &mut Bencher) { + let s = black_box(en::TINY); + let other = black_box(en::TINY); + b.iter(|| assert!(s.eq_ignore_ascii_case(other))) +} + +#[bench] +fn bench_str_17_bytes_eq(b: &mut Bencher) { + let s = black_box(&en::SMALL[..17]); + let other = black_box(&en::SMALL[..17]); + b.iter(|| assert!(s.eq_ignore_ascii_case(other))) +} + +#[bench] +fn bench_str_31_bytes_eq(b: &mut Bencher) { + let s = black_box(&en::SMALL[..31]); + let other = black_box(&en::SMALL[..31]); + b.iter(|| assert!(s.eq_ignore_ascii_case(other))) +} + +#[bench] +fn bench_medium_str_eq(b: &mut Bencher) { + let s = black_box(en::MEDIUM); + let other = black_box(en::MEDIUM); + b.iter(|| assert!(s.eq_ignore_ascii_case(other))) +} + +#[bench] +fn bench_large_str_eq(b: &mut Bencher) { + let s = black_box(en::LARGE); + let other = black_box(en::LARGE); + b.iter(|| assert!(s.eq_ignore_ascii_case(other))) +} diff --git a/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs b/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs index b733f1812c92..d4ac5d64585d 100644 --- a/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs +++ b/tests/codegen-llvm/lib-optimizations/eq_ignore_ascii_case.rs @@ -2,13 +2,15 @@ //@ only-x86_64 #![crate_type = "lib"] -// Ensure that the optimized variant of the function gets auto-vectorized. +// Ensure that the optimized variant of the function gets auto-vectorized and +// that the inner helper function is inlined. // CHECK-LABEL: @eq_ignore_ascii_case_autovectorized #[no_mangle] pub fn eq_ignore_ascii_case_autovectorized(s: &str, other: &str) -> bool { // CHECK: load <16 x i8> // CHECK: load <16 x i8> // CHECK: bitcast <16 x i1> + // CHECK-NOT: call {{.*}}eq_ignore_ascii_inner // CHECK-NOT: panic s.eq_ignore_ascii_case(other) } From 1566879798185efdcfba62d2944a2e60dc10d814 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Mon, 13 Oct 2025 19:53:57 +0200 Subject: [PATCH 0029/1843] clean-up - update file name to match lint name - use `Symbol`s instead of `&str`s - get rid of needless lifetimes --- ...alls.rs => collapsible_span_lint_calls.rs} | 47 +++++++++---------- clippy_lints_internal/src/lib.rs | 6 +-- clippy_utils/src/sym.rs | 4 ++ 3 files changed, 29 insertions(+), 28 deletions(-) rename clippy_lints_internal/src/{collapsible_calls.rs => collapsible_span_lint_calls.rs} (88%) diff --git a/clippy_lints_internal/src/collapsible_calls.rs b/clippy_lints_internal/src/collapsible_span_lint_calls.rs similarity index 88% rename from clippy_lints_internal/src/collapsible_calls.rs rename to clippy_lints_internal/src/collapsible_span_lint_calls.rs index 7c9e7286925e..e83508f6ff4a 100644 --- a/clippy_lints_internal/src/collapsible_calls.rs +++ b/clippy_lints_internal/src/collapsible_span_lint_calls.rs @@ -1,6 +1,6 @@ use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::source::snippet; -use clippy_utils::{SpanlessEq, is_lint_allowed, peel_blocks_with_stmt}; +use clippy_utils::{SpanlessEq, is_lint_allowed, peel_blocks_with_stmt, sym}; use rustc_errors::Applicability; use rustc_hir::{Closure, Expr, ExprKind}; use rustc_lint::{LateContext, LateLintPass}; @@ -91,8 +91,8 @@ impl<'tcx> LateLintPass<'tcx> for CollapsibleCalls { let and_then_snippets = get_and_then_snippets(cx, call_cx.span, call_lint.span, call_sp.span, call_msg.span); let mut sle = SpanlessEq::new(cx).deny_side_effects(); - match ps.ident.as_str() { - "span_suggestion" if sle.eq_expr(call_sp, &span_call_args[0]) => { + match ps.ident.name { + sym::span_suggestion if sle.eq_expr(call_sp, &span_call_args[0]) => { suggest_suggestion( cx, expr, @@ -100,19 +100,19 @@ impl<'tcx> LateLintPass<'tcx> for CollapsibleCalls { &span_suggestion_snippets(cx, span_call_args), ); }, - "span_help" if sle.eq_expr(call_sp, &span_call_args[0]) => { + sym::span_help if sle.eq_expr(call_sp, &span_call_args[0]) => { let help_snippet = snippet(cx, span_call_args[1].span, r#""...""#); suggest_help(cx, expr, &and_then_snippets, help_snippet.borrow(), true); }, - "span_note" if sle.eq_expr(call_sp, &span_call_args[0]) => { + sym::span_note if sle.eq_expr(call_sp, &span_call_args[0]) => { let note_snippet = snippet(cx, span_call_args[1].span, r#""...""#); suggest_note(cx, expr, &and_then_snippets, note_snippet.borrow(), true); }, - "help" => { + sym::help => { let help_snippet = snippet(cx, span_call_args[0].span, r#""...""#); suggest_help(cx, expr, &and_then_snippets, help_snippet.borrow(), false); }, - "note" => { + sym::note => { let note_snippet = snippet(cx, span_call_args[0].span, r#""...""#); suggest_note(cx, expr, &and_then_snippets, note_snippet.borrow(), false); }, @@ -122,11 +122,11 @@ impl<'tcx> LateLintPass<'tcx> for CollapsibleCalls { } } -struct AndThenSnippets<'a> { - cx: Cow<'a, str>, - lint: Cow<'a, str>, - span: Cow<'a, str>, - msg: Cow<'a, str>, +struct AndThenSnippets { + cx: Cow<'static, str>, + lint: Cow<'static, str>, + span: Cow<'static, str>, + msg: Cow<'static, str>, } fn get_and_then_snippets( @@ -135,7 +135,7 @@ fn get_and_then_snippets( lint_span: Span, span_span: Span, msg_span: Span, -) -> AndThenSnippets<'static> { +) -> AndThenSnippets { let cx_snippet = snippet(cx, cx_span, "cx"); let lint_snippet = snippet(cx, lint_span, ".."); let span_snippet = snippet(cx, span_span, "span"); @@ -149,16 +149,13 @@ fn get_and_then_snippets( } } -struct SpanSuggestionSnippets<'a> { - help: Cow<'a, str>, - sugg: Cow<'a, str>, - applicability: Cow<'a, str>, +struct SpanSuggestionSnippets { + help: Cow<'static, str>, + sugg: Cow<'static, str>, + applicability: Cow<'static, str>, } -fn span_suggestion_snippets<'a, 'hir>( - cx: &LateContext<'_>, - span_call_args: &'hir [Expr<'hir>], -) -> SpanSuggestionSnippets<'a> { +fn span_suggestion_snippets<'hir>(cx: &LateContext<'_>, span_call_args: &'hir [Expr<'hir>]) -> SpanSuggestionSnippets { let help_snippet = snippet(cx, span_call_args[1].span, r#""...""#); let sugg_snippet = snippet(cx, span_call_args[2].span, ".."); let applicability_snippet = snippet(cx, span_call_args[3].span, "Applicability::MachineApplicable"); @@ -173,8 +170,8 @@ fn span_suggestion_snippets<'a, 'hir>( fn suggest_suggestion( cx: &LateContext<'_>, expr: &Expr<'_>, - and_then_snippets: &AndThenSnippets<'_>, - span_suggestion_snippets: &SpanSuggestionSnippets<'_>, + and_then_snippets: &AndThenSnippets, + span_suggestion_snippets: &SpanSuggestionSnippets, ) { span_lint_and_sugg( cx, @@ -199,7 +196,7 @@ fn suggest_suggestion( fn suggest_help( cx: &LateContext<'_>, expr: &Expr<'_>, - and_then_snippets: &AndThenSnippets<'_>, + and_then_snippets: &AndThenSnippets, help: &str, with_span: bool, ) { @@ -226,7 +223,7 @@ fn suggest_help( fn suggest_note( cx: &LateContext<'_>, expr: &Expr<'_>, - and_then_snippets: &AndThenSnippets<'_>, + and_then_snippets: &AndThenSnippets, note: &str, with_span: bool, ) { diff --git a/clippy_lints_internal/src/lib.rs b/clippy_lints_internal/src/lib.rs index 43cde86504f5..2fc172f64725 100644 --- a/clippy_lints_internal/src/lib.rs +++ b/clippy_lints_internal/src/lib.rs @@ -31,7 +31,7 @@ extern crate rustc_session; extern crate rustc_span; mod almost_standard_lint_formulation; -mod collapsible_calls; +mod collapsible_span_lint_calls; mod derive_deserialize_allowing_unknown; mod internal_paths; mod lint_without_lint_pass; @@ -46,7 +46,7 @@ use rustc_lint::{Lint, LintStore}; static LINTS: &[&Lint] = &[ almost_standard_lint_formulation::ALMOST_STANDARD_LINT_FORMULATION, - collapsible_calls::COLLAPSIBLE_SPAN_LINT_CALLS, + collapsible_span_lint_calls::COLLAPSIBLE_SPAN_LINT_CALLS, derive_deserialize_allowing_unknown::DERIVE_DESERIALIZE_ALLOWING_UNKNOWN, lint_without_lint_pass::DEFAULT_LINT, lint_without_lint_pass::INVALID_CLIPPY_VERSION_ATTRIBUTE, @@ -66,7 +66,7 @@ pub fn register_lints(store: &mut LintStore) { store.register_early_pass(|| Box::new(unsorted_clippy_utils_paths::UnsortedClippyUtilsPaths)); store.register_early_pass(|| Box::new(produce_ice::ProduceIce)); - store.register_late_pass(|_| Box::new(collapsible_calls::CollapsibleCalls)); + store.register_late_pass(|_| Box::new(collapsible_span_lint_calls::CollapsibleCalls)); store.register_late_pass(|_| Box::new(derive_deserialize_allowing_unknown::DeriveDeserializeAllowingUnknown)); store.register_late_pass(|_| Box::::default()); store.register_late_pass(|_| Box::::default()); diff --git a/clippy_utils/src/sym.rs b/clippy_utils/src/sym.rs index 2b22f344e8c0..e3d733cada71 100644 --- a/clippy_utils/src/sym.rs +++ b/clippy_utils/src/sym.rs @@ -173,6 +173,7 @@ generate! { get_unchecked, get_unchecked_mut, has_significant_drop, + help, hidden_glob_reexports, hygiene, insert, @@ -310,7 +311,10 @@ generate! { sort, sort_by, sort_unstable_by, + span_help, span_lint_and_then, + span_note, + span_suggestion, split, split_at, split_at_checked, From 4d795229058b568570cf68a1320cd5bd95eab377 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Mon, 13 Oct 2025 19:32:34 +0200 Subject: [PATCH 0030/1843] fix: use `snippet_with_context` for spans that are likely to contain macro expns --- .../src/collapsible_span_lint_calls.rs | 69 ++++++++++++------- .../collapsible_span_lint_calls.fixed | 9 +++ .../collapsible_span_lint_calls.rs | 29 ++++++++ .../collapsible_span_lint_calls.stderr | 50 +++++++++++++- 4 files changed, 131 insertions(+), 26 deletions(-) diff --git a/clippy_lints_internal/src/collapsible_span_lint_calls.rs b/clippy_lints_internal/src/collapsible_span_lint_calls.rs index e83508f6ff4a..f7adb782929c 100644 --- a/clippy_lints_internal/src/collapsible_span_lint_calls.rs +++ b/clippy_lints_internal/src/collapsible_span_lint_calls.rs @@ -1,11 +1,11 @@ use clippy_utils::diagnostics::span_lint_and_sugg; -use clippy_utils::source::snippet; +use clippy_utils::source::{snippet, snippet_with_context}; use clippy_utils::{SpanlessEq, is_lint_allowed, peel_blocks_with_stmt, sym}; use rustc_errors::Applicability; use rustc_hir::{Closure, Expr, ExprKind}; use rustc_lint::{LateContext, LateLintPass}; use rustc_session::{declare_lint_pass, declare_tool_lint}; -use rustc_span::Span; +use rustc_span::{Span, SyntaxContext}; use std::borrow::{Borrow, Cow}; @@ -88,33 +88,42 @@ impl<'tcx> LateLintPass<'tcx> for CollapsibleCalls { && let ExprKind::MethodCall(ps, recv, span_call_args, _) = &only_expr.kind && let ExprKind::Path(..) = recv.kind { - let and_then_snippets = - get_and_then_snippets(cx, call_cx.span, call_lint.span, call_sp.span, call_msg.span); + let mut app = Applicability::MachineApplicable; + let expr_ctxt = expr.span.ctxt(); + let and_then_snippets = get_and_then_snippets( + cx, + expr_ctxt, + call_cx.span, + call_lint.span, + call_sp.span, + call_msg.span, + &mut app, + ); let mut sle = SpanlessEq::new(cx).deny_side_effects(); match ps.ident.name { sym::span_suggestion if sle.eq_expr(call_sp, &span_call_args[0]) => { - suggest_suggestion( - cx, - expr, - &and_then_snippets, - &span_suggestion_snippets(cx, span_call_args), - ); + let snippets = span_suggestion_snippets(cx, expr_ctxt, span_call_args, &mut app); + suggest_suggestion(cx, expr, &and_then_snippets, &snippets, app); }, sym::span_help if sle.eq_expr(call_sp, &span_call_args[0]) => { - let help_snippet = snippet(cx, span_call_args[1].span, r#""...""#); - suggest_help(cx, expr, &and_then_snippets, help_snippet.borrow(), true); + let help_snippet = + snippet_with_context(cx, span_call_args[1].span, expr_ctxt, r#""...""#, &mut app).0; + suggest_help(cx, expr, &and_then_snippets, help_snippet.borrow(), true, app); }, sym::span_note if sle.eq_expr(call_sp, &span_call_args[0]) => { - let note_snippet = snippet(cx, span_call_args[1].span, r#""...""#); - suggest_note(cx, expr, &and_then_snippets, note_snippet.borrow(), true); + let note_snippet = + snippet_with_context(cx, span_call_args[1].span, expr_ctxt, r#""...""#, &mut app).0; + suggest_note(cx, expr, &and_then_snippets, note_snippet.borrow(), true, app); }, sym::help => { - let help_snippet = snippet(cx, span_call_args[0].span, r#""...""#); - suggest_help(cx, expr, &and_then_snippets, help_snippet.borrow(), false); + let help_snippet = + snippet_with_context(cx, span_call_args[0].span, expr_ctxt, r#""...""#, &mut app).0; + suggest_help(cx, expr, &and_then_snippets, help_snippet.borrow(), false, app); }, sym::note => { - let note_snippet = snippet(cx, span_call_args[0].span, r#""...""#); - suggest_note(cx, expr, &and_then_snippets, note_snippet.borrow(), false); + let note_snippet = + snippet_with_context(cx, span_call_args[0].span, expr_ctxt, r#""...""#, &mut app).0; + suggest_note(cx, expr, &and_then_snippets, note_snippet.borrow(), false, app); }, _ => (), } @@ -131,15 +140,17 @@ struct AndThenSnippets { fn get_and_then_snippets( cx: &LateContext<'_>, + expr_ctxt: SyntaxContext, cx_span: Span, lint_span: Span, span_span: Span, msg_span: Span, + app: &mut Applicability, ) -> AndThenSnippets { let cx_snippet = snippet(cx, cx_span, "cx"); let lint_snippet = snippet(cx, lint_span, ".."); let span_snippet = snippet(cx, span_span, "span"); - let msg_snippet = snippet(cx, msg_span, r#""...""#); + let msg_snippet = snippet_with_context(cx, msg_span, expr_ctxt, r#""...""#, app).0; AndThenSnippets { cx: cx_snippet, @@ -155,9 +166,14 @@ struct SpanSuggestionSnippets { applicability: Cow<'static, str>, } -fn span_suggestion_snippets<'hir>(cx: &LateContext<'_>, span_call_args: &'hir [Expr<'hir>]) -> SpanSuggestionSnippets { - let help_snippet = snippet(cx, span_call_args[1].span, r#""...""#); - let sugg_snippet = snippet(cx, span_call_args[2].span, ".."); +fn span_suggestion_snippets<'hir>( + cx: &LateContext<'_>, + expr_ctxt: SyntaxContext, + span_call_args: &'hir [Expr<'hir>], + app: &mut Applicability, +) -> SpanSuggestionSnippets { + let help_snippet = snippet_with_context(cx, span_call_args[1].span, expr_ctxt, r#""...""#, app).0; + let sugg_snippet = snippet_with_context(cx, span_call_args[2].span, expr_ctxt, "..", app).0; let applicability_snippet = snippet(cx, span_call_args[3].span, "Applicability::MachineApplicable"); SpanSuggestionSnippets { @@ -172,6 +188,7 @@ fn suggest_suggestion( expr: &Expr<'_>, and_then_snippets: &AndThenSnippets, span_suggestion_snippets: &SpanSuggestionSnippets, + app: Applicability, ) { span_lint_and_sugg( cx, @@ -189,7 +206,7 @@ fn suggest_suggestion( span_suggestion_snippets.sugg, span_suggestion_snippets.applicability ), - Applicability::MachineApplicable, + app, ); } @@ -199,6 +216,7 @@ fn suggest_help( and_then_snippets: &AndThenSnippets, help: &str, with_span: bool, + app: Applicability, ) { let option_span = if with_span { format!("Some({})", and_then_snippets.span) @@ -216,7 +234,7 @@ fn suggest_help( "span_lint_and_help({}, {}, {}, {}, {}, {help})", and_then_snippets.cx, and_then_snippets.lint, and_then_snippets.span, and_then_snippets.msg, &option_span, ), - Applicability::MachineApplicable, + app, ); } @@ -226,6 +244,7 @@ fn suggest_note( and_then_snippets: &AndThenSnippets, note: &str, with_span: bool, + app: Applicability, ) { let note_span = if with_span { format!("Some({})", and_then_snippets.span) @@ -243,6 +262,6 @@ fn suggest_note( "span_lint_and_note({}, {}, {}, {}, {note_span}, {note})", and_then_snippets.cx, and_then_snippets.lint, and_then_snippets.span, and_then_snippets.msg, ), - Applicability::MachineApplicable, + app, ); } diff --git a/tests/ui-internal/collapsible_span_lint_calls.fixed b/tests/ui-internal/collapsible_span_lint_calls.fixed index 76f68686ee2a..2b646a38b534 100644 --- a/tests/ui-internal/collapsible_span_lint_calls.fixed +++ b/tests/ui-internal/collapsible_span_lint_calls.fixed @@ -50,6 +50,15 @@ impl EarlyLintPass for Pass { span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { db.help(help_msg).help(help_msg); }); + + // Issue #15880 + #[expect(clippy::disallowed_names)] + let foo = "foo"; + span_lint_and_sugg(cx, TEST_LINT, expr.span, lint_msg, format!("try using {foo}"), format!("{foo}.use"), Applicability::MachineApplicable); + span_lint_and_help(cx, TEST_LINT, expr.span, lint_msg, Some(expr.span), format!("try using {foo}")); + span_lint_and_help(cx, TEST_LINT, expr.span, lint_msg, None, format!("try using {foo}")); + span_lint_and_note(cx, TEST_LINT, expr.span, lint_msg, Some(expr.span), format!("required because of {foo}")); + span_lint_and_note(cx, TEST_LINT, expr.span, lint_msg, None, format!("required because of {foo}")); } } diff --git a/tests/ui-internal/collapsible_span_lint_calls.rs b/tests/ui-internal/collapsible_span_lint_calls.rs index 214c8783a669..500552370053 100644 --- a/tests/ui-internal/collapsible_span_lint_calls.rs +++ b/tests/ui-internal/collapsible_span_lint_calls.rs @@ -65,6 +65,35 @@ impl EarlyLintPass for Pass { span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { db.help(help_msg).help(help_msg); }); + + // Issue #15880 + #[expect(clippy::disallowed_names)] + let foo = "foo"; + span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { + //~^ collapsible_span_lint_calls + db.span_suggestion( + expr.span, + format!("try using {foo}"), + format!("{foo}.use"), + Applicability::MachineApplicable, + ); + }); + span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { + //~^ collapsible_span_lint_calls + db.span_help(expr.span, format!("try using {foo}")); + }); + span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { + //~^ collapsible_span_lint_calls + db.help(format!("try using {foo}")); + }); + span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { + //~^ collapsible_span_lint_calls + db.span_note(expr.span, format!("required because of {foo}")); + }); + span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { + //~^ collapsible_span_lint_calls + db.note(format!("required because of {foo}")); + }); } } diff --git a/tests/ui-internal/collapsible_span_lint_calls.stderr b/tests/ui-internal/collapsible_span_lint_calls.stderr index 9c83538947ca..76b453019270 100644 --- a/tests/ui-internal/collapsible_span_lint_calls.stderr +++ b/tests/ui-internal/collapsible_span_lint_calls.stderr @@ -49,5 +49,53 @@ LL | | db.note(note_msg); LL | | }); | |__________^ help: collapse into: `span_lint_and_note(cx, TEST_LINT, expr.span, lint_msg, None, note_msg)` -error: aborting due to 5 previous errors +error: this call is collapsible + --> tests/ui-internal/collapsible_span_lint_calls.rs:72:9 + | +LL | / span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { +LL | | +LL | | db.span_suggestion( +LL | | expr.span, +... | +LL | | ); +LL | | }); + | |__________^ help: collapse into: `span_lint_and_sugg(cx, TEST_LINT, expr.span, lint_msg, format!("try using {foo}"), format!("{foo}.use"), Applicability::MachineApplicable)` + +error: this call is collapsible + --> tests/ui-internal/collapsible_span_lint_calls.rs:81:9 + | +LL | / span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { +LL | | +LL | | db.span_help(expr.span, format!("try using {foo}")); +LL | | }); + | |__________^ help: collapse into: `span_lint_and_help(cx, TEST_LINT, expr.span, lint_msg, Some(expr.span), format!("try using {foo}"))` + +error: this call is collapsible + --> tests/ui-internal/collapsible_span_lint_calls.rs:85:9 + | +LL | / span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { +LL | | +LL | | db.help(format!("try using {foo}")); +LL | | }); + | |__________^ help: collapse into: `span_lint_and_help(cx, TEST_LINT, expr.span, lint_msg, None, format!("try using {foo}"))` + +error: this call is collapsible + --> tests/ui-internal/collapsible_span_lint_calls.rs:89:9 + | +LL | / span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { +LL | | +LL | | db.span_note(expr.span, format!("required because of {foo}")); +LL | | }); + | |__________^ help: collapse into: `span_lint_and_note(cx, TEST_LINT, expr.span, lint_msg, Some(expr.span), format!("required because of {foo}"))` + +error: this call is collapsible + --> tests/ui-internal/collapsible_span_lint_calls.rs:93:9 + | +LL | / span_lint_and_then(cx, TEST_LINT, expr.span, lint_msg, |db| { +LL | | +LL | | db.note(format!("required because of {foo}")); +LL | | }); + | |__________^ help: collapse into: `span_lint_and_note(cx, TEST_LINT, expr.span, lint_msg, None, format!("required because of {foo}"))` + +error: aborting due to 10 previous errors From b270954300ec06f3478e3758d0c8d3e9f38efebf Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Mon, 13 Oct 2025 19:50:47 +0200 Subject: [PATCH 0031/1843] s/snippet/snippet_with_applicability while we're at it --- .../src/collapsible_span_lint_calls.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/clippy_lints_internal/src/collapsible_span_lint_calls.rs b/clippy_lints_internal/src/collapsible_span_lint_calls.rs index f7adb782929c..b048a1004b0d 100644 --- a/clippy_lints_internal/src/collapsible_span_lint_calls.rs +++ b/clippy_lints_internal/src/collapsible_span_lint_calls.rs @@ -1,5 +1,5 @@ use clippy_utils::diagnostics::span_lint_and_sugg; -use clippy_utils::source::{snippet, snippet_with_context}; +use clippy_utils::source::{snippet_with_applicability, snippet_with_context}; use clippy_utils::{SpanlessEq, is_lint_allowed, peel_blocks_with_stmt, sym}; use rustc_errors::Applicability; use rustc_hir::{Closure, Expr, ExprKind}; @@ -147,9 +147,9 @@ fn get_and_then_snippets( msg_span: Span, app: &mut Applicability, ) -> AndThenSnippets { - let cx_snippet = snippet(cx, cx_span, "cx"); - let lint_snippet = snippet(cx, lint_span, ".."); - let span_snippet = snippet(cx, span_span, "span"); + let cx_snippet = snippet_with_applicability(cx, cx_span, "cx", app); + let lint_snippet = snippet_with_applicability(cx, lint_span, "..", app); + let span_snippet = snippet_with_applicability(cx, span_span, "span", app); let msg_snippet = snippet_with_context(cx, msg_span, expr_ctxt, r#""...""#, app).0; AndThenSnippets { @@ -174,7 +174,8 @@ fn span_suggestion_snippets<'hir>( ) -> SpanSuggestionSnippets { let help_snippet = snippet_with_context(cx, span_call_args[1].span, expr_ctxt, r#""...""#, app).0; let sugg_snippet = snippet_with_context(cx, span_call_args[2].span, expr_ctxt, "..", app).0; - let applicability_snippet = snippet(cx, span_call_args[3].span, "Applicability::MachineApplicable"); + let applicability_snippet = + snippet_with_applicability(cx, span_call_args[3].span, "Applicability::MachineApplicable", app); SpanSuggestionSnippets { help: help_snippet, From 31bf836c74e5b53e654bed70158cabfdfdca8acf Mon Sep 17 00:00:00 2001 From: okaneco <47607823+okaneco@users.noreply.github.com> Date: Tue, 14 Oct 2025 02:23:44 -0400 Subject: [PATCH 0032/1843] Use a const generic parameter for the function chunk size Add comments for the optimized function invariants to the caller Add const-hack fixme for using while-loops Document the invariant for the `_chunks` function Add a debug assert for the tail handling invariant --- library/core/src/slice/ascii.rs | 35 +++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index 8b713947cd9c..0f3cf243f164 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -61,8 +61,15 @@ impl [u8] { } #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] - if self.len() >= 16 { - return self.eq_ignore_ascii_case_chunks(other); + { + const CHUNK_SIZE: usize = 16; + // The following function has two invariants: + // 1. The slice lengths must be equal, which we checked above. + // 2. The slice lengths must greater than or equal to N, which this + // if-statement is checking. + if self.len() >= CHUNK_SIZE { + return self.eq_ignore_ascii_case_chunks::(other); + } } self.eq_ignore_ascii_case_simple(other) @@ -90,24 +97,30 @@ impl [u8] { true } - /// Optimized version of `eq_ignore_ascii_case` for byte lengths of at least - /// 16 bytes, which processes chunks at a time. + /// Optimized version of `eq_ignore_ascii_case` to process chunks at a time. /// /// Platforms that have SIMD instructions may benefit from this /// implementation over `eq_ignore_ascii_case_simple`. + /// + /// # Invariants + /// + /// The caller must guarantee that the slices are equal in length, and the + /// slice lengths are greater than or equal to `N` bytes. #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] #[inline] - const fn eq_ignore_ascii_case_chunks(&self, other: &[u8]) -> bool { - const N: usize = 16; + const fn eq_ignore_ascii_case_chunks(&self, other: &[u8]) -> bool { + // FIXME(const-hack): The while-loops that follow should be replaced by + // for-loops when available in const. + let (self_chunks, self_rem) = self.as_chunks::(); let (other_chunks, _) = other.as_chunks::(); // Branchless check to encourage auto-vectorization #[inline(always)] - const fn eq_ignore_ascii_inner(lhs: &[u8; N], rhs: &[u8; N]) -> bool { + const fn eq_ignore_ascii_inner(lhs: &[u8; L], rhs: &[u8; L]) -> bool { let mut equal_ascii = true; let mut j = 0; - while j < N { + while j < L { equal_ascii &= lhs[j].eq_ignore_ascii_case(&rhs[j]); j += 1; } @@ -124,6 +137,12 @@ impl [u8] { i += 1; } + // Check the length invariant which is necessary for the tail-handling + // logic to be correct. This should have been upheld by the caller, + // otherwise lengths less than N will compare as true without any + // checking. + debug_assert!(self.len() >= N); + // If there are remaining tails, load the last N bytes in the slices to // avoid falling back to per-byte checking. if !self_rem.is_empty() { From 061617bdbcaa49bb5c4456c0c5264d0849afe4b1 Mon Sep 17 00:00:00 2001 From: sayantn Date: Wed, 8 Oct 2025 08:12:09 +0530 Subject: [PATCH 0033/1843] Add alignment parameter to `simd_masked_{load,store}` --- crates/core_simd/src/vector.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index d76a6cd52bfc..f40031f8c4da 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -474,7 +474,14 @@ where or: Self, ) -> Self { // SAFETY: The safety of reading elements through `ptr` is ensured by the caller. - unsafe { core::intrinsics::simd::simd_masked_load(enable.to_int(), ptr, or) } + unsafe { + core::intrinsics::simd::simd_masked_load::< + _, + _, + _, + { core::intrinsics::simd::SimdAlign::Element }, + >(enable.to_int(), ptr, or) + } } /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. @@ -723,7 +730,14 @@ where #[inline] pub unsafe fn store_select_ptr(self, ptr: *mut T, enable: Mask<::Mask, N>) { // SAFETY: The safety of writing elements through `ptr` is ensured by the caller. - unsafe { core::intrinsics::simd::simd_masked_store(enable.to_int(), ptr, self) } + unsafe { + core::intrinsics::simd::simd_masked_store::< + _, + _, + _, + { core::intrinsics::simd::SimdAlign::Element }, + >(enable.to_int(), ptr, self) + } } /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`. From d160cb7313fb07d85ad57b99a98106211684c270 Mon Sep 17 00:00:00 2001 From: "Ian D. Bollinger" Date: Tue, 4 Nov 2025 13:00:02 -0500 Subject: [PATCH 0034/1843] `double_comparison`: add missing cases Add checks for expressions such as `x != y && x >= y` and `x != y && x <= y`. --- .../src/operators/double_comparison.rs | 12 +++++++++ tests/ui/double_comparison.fixed | 16 ++++++++++++ tests/ui/double_comparison.rs | 16 ++++++++++++ tests/ui/double_comparison.stderr | 26 ++++++++++++++++++- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/clippy_lints/src/operators/double_comparison.rs b/clippy_lints/src/operators/double_comparison.rs index 71982023779e..a40a724d2da5 100644 --- a/clippy_lints/src/operators/double_comparison.rs +++ b/clippy_lints/src/operators/double_comparison.rs @@ -39,6 +39,18 @@ pub(super) fn check(cx: &LateContext<'_>, op: BinOpKind, lhs: &Expr<'_>, rhs: &E | (BinOpKind::And, BinOpKind::Ge, BinOpKind::Le) => { "==" }, + // x != y && x >= y => x > y + (BinOpKind::And, BinOpKind::Ne, BinOpKind::Ge) + // x >= y && x != y => x > y + | (BinOpKind::And, BinOpKind::Ge, BinOpKind::Ne) => { + ">" + }, + // x != y && x <= y => x < y + (BinOpKind::And, BinOpKind::Ne, BinOpKind::Le) + // x <= y && x != y => x < y + | (BinOpKind::And, BinOpKind::Le, BinOpKind::Ne) => { + "<" + }, _ => return, }; diff --git a/tests/ui/double_comparison.fixed b/tests/ui/double_comparison.fixed index 0680eb35ef97..29047b8a31cb 100644 --- a/tests/ui/double_comparison.fixed +++ b/tests/ui/double_comparison.fixed @@ -35,4 +35,20 @@ fn main() { //~^ double_comparisons // do something } + if x < y { + //~^ double_comparisons + // do something + } + if x < y { + //~^ double_comparisons + // do something + } + if x > y { + //~^ double_comparisons + // do something + } + if x > y { + //~^ double_comparisons + // do something + } } diff --git a/tests/ui/double_comparison.rs b/tests/ui/double_comparison.rs index 18ab7d2c4254..13edb2a996a1 100644 --- a/tests/ui/double_comparison.rs +++ b/tests/ui/double_comparison.rs @@ -35,4 +35,20 @@ fn main() { //~^ double_comparisons // do something } + if x != y && x <= y { + //~^ double_comparisons + // do something + } + if x <= y && x != y { + //~^ double_comparisons + // do something + } + if x != y && x >= y { + //~^ double_comparisons + // do something + } + if x >= y && x != y { + //~^ double_comparisons + // do something + } } diff --git a/tests/ui/double_comparison.stderr b/tests/ui/double_comparison.stderr index 984614c203eb..be7eba611cb0 100644 --- a/tests/ui/double_comparison.stderr +++ b/tests/ui/double_comparison.stderr @@ -49,5 +49,29 @@ error: this binary expression can be simplified LL | if x >= y && x <= y { | ^^^^^^^^^^^^^^^^ help: try: `x == y` -error: aborting due to 8 previous errors +error: this binary expression can be simplified + --> tests/ui/double_comparison.rs:38:8 + | +LL | if x != y && x <= y { + | ^^^^^^^^^^^^^^^^ help: try: `x < y` + +error: this binary expression can be simplified + --> tests/ui/double_comparison.rs:42:8 + | +LL | if x <= y && x != y { + | ^^^^^^^^^^^^^^^^ help: try: `x < y` + +error: this binary expression can be simplified + --> tests/ui/double_comparison.rs:46:8 + | +LL | if x != y && x >= y { + | ^^^^^^^^^^^^^^^^ help: try: `x > y` + +error: this binary expression can be simplified + --> tests/ui/double_comparison.rs:50:8 + | +LL | if x >= y && x != y { + | ^^^^^^^^^^^^^^^^ help: try: `x > y` + +error: aborting due to 12 previous errors From 242b4b5dc96472858438338642635091dfd3ec99 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Tue, 11 Nov 2025 13:25:22 -0800 Subject: [PATCH 0035/1843] Remove more #[must_use] from portable-simd warning: `#[must_use]` attribute cannot be used on trait methods in impl blocks --> crates/core_simd/src/masks/bitmask.rs:173:5 | 173 | #[must_use = "method returns a new mask and does not mutate the original value"] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! = help: `#[must_use]` can be applied to data types, foreign functions, functions, inherent methods, provided trait methods, required trait methods, traits, and unions = note: `#[warn(unused_attributes)]` (part of `#[warn(unused)]`) on by default warning: `#[must_use]` attribute cannot be used on trait methods in impl blocks --> crates/core_simd/src/masks/bitmask.rs:190:5 | 190 | #[must_use = "method returns a new mask and does not mutate the original value"] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! = help: `#[must_use]` can be applied to data types, foreign functions, functions, inherent methods, provided trait methods, required trait methods, traits, and unions warning: `#[must_use]` attribute cannot be used on trait methods in impl blocks --> crates/core_simd/src/masks/bitmask.rs:206:5 | 206 | #[must_use = "method returns a new mask and does not mutate the original value"] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! = help: `#[must_use]` can be applied to data types, foreign functions, functions, inherent methods, provided trait methods, required trait methods, traits, and unions warning: `#[must_use]` attribute cannot be used on trait methods in impl blocks --> crates/core_simd/src/masks/bitmask.rs:222:5 | 222 | #[must_use = "method returns a new mask and does not mutate the original value"] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! = help: `#[must_use]` can be applied to data types, foreign functions, functions, inherent methods, provided trait methods, required trait methods, traits, and unions --- crates/core_simd/src/masks/bitmask.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 8221d8f17e90..32d37b553392 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -170,7 +170,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitand(mut self, rhs: Self) -> Self { for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { *l &= r; @@ -187,7 +186,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitor(mut self, rhs: Self) -> Self { for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { *l |= r; @@ -203,7 +201,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn bitxor(mut self, rhs: Self) -> Self::Output { for (l, r) in self.0.as_mut().iter_mut().zip(rhs.0.as_ref().iter()) { *l ^= r; @@ -219,7 +216,6 @@ where { type Output = Self; #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] fn not(mut self) -> Self::Output { for x in self.0.as_mut() { *x = !*x; From 055cd2717fe298a567c130204743cc0cc06a08e0 Mon Sep 17 00:00:00 2001 From: Voxell Paladynee Date: Wed, 19 Nov 2025 21:17:40 +0100 Subject: [PATCH 0036/1843] ptr_cast_slice: add new methods to raw pointers --- library/core/src/ptr/const_ptr.rs | 37 +++++++++++++++++++++++++ library/core/src/ptr/mut_ptr.rs | 45 +++++++++++++++++++++++++++++++ library/core/src/ptr/non_null.rs | 29 ++++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 84a6982d5680..e4d2f7436f13 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1386,6 +1386,43 @@ impl *const T { pub const fn cast_uninit(self) -> *const MaybeUninit { self as _ } + + /// Forms a raw slice from a pointer and a length. + /// + /// The `len` argument is the number of **elements**, not the number of bytes. + /// + /// This function is safe, but actually using the return value is unsafe. + /// See the documentation of [`slice::from_raw_parts`] for slice safety requirements. + /// + /// [`slice::from_raw_parts`]: crate::slice::from_raw_parts + /// + /// # Examples + /// + /// ```rust + /// #![feature(ptr_cast_slice)] + /// // create a slice pointer when starting out with a pointer to the first element + /// let x = [5, 6, 7]; + /// let raw_pointer = x.as_ptr(); + /// let slice = raw_pointer.cast_slice(3); + /// assert_eq!(unsafe { &*slice }[2], 7); + /// ``` + /// + /// You must ensure that the pointer is valid and not null before dereferencing + /// the raw slice. A slice reference must never have a null pointer, even if it's empty. + /// + /// ```rust,should_panic + /// #![feature(ptr_cast_slice)] + /// use std::ptr; + /// let danger: *const [u8] = ptr::null::().cast_slice(0); + /// unsafe { + /// danger.as_ref().expect("references must not be null"); + /// } + /// ``` + #[inline] + #[unstable(feature = "ptr_cast_slice", issue = "149103")] + pub const fn cast_slice(self, len: usize) -> *const [T] { + slice_from_raw_parts(self, len) + } } impl *const MaybeUninit { /// Casts from a maybe-uninitialized type to its initialized version. diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 85d54b4d3b9b..edcd0cd2d932 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1655,6 +1655,51 @@ impl *mut T { pub const fn cast_uninit(self) -> *mut MaybeUninit { self as _ } + + /// Forms a raw mutable slice from a pointer and a length. + /// + /// The `len` argument is the number of **elements**, not the number of bytes. + /// + /// Performs the same functionality as [`cast_slice`] on a `*const T`, except that a + /// raw mutable slice is returned, as opposed to a raw immutable slice. + /// + /// This function is safe, but actually using the return value is unsafe. + /// See the documentation of [`slice::from_raw_parts_mut`] for slice safety requirements. + /// + /// [`slice::from_raw_parts_mut`]: crate::slice::from_raw_parts_mut + /// [`cast_slice`]: pointer::cast_slice + /// + /// # Examples + /// + /// ```rust + /// #![feature(ptr_cast_slice)] + /// + /// let x = &mut [5, 6, 7]; + /// let slice = x.as_mut_ptr().cast_slice(3); + /// + /// unsafe { + /// (*slice)[2] = 99; // assign a value at an index in the slice + /// }; + /// + /// assert_eq!(unsafe { &*slice }[2], 99); + /// ``` + /// + /// You must ensure that the pointer is valid and not null before dereferencing + /// the raw slice. A slice reference must never have a null pointer, even if it's empty. + /// + /// ```rust,should_panic + /// #![feature(ptr_cast_slice)] + /// use std::ptr; + /// let danger: *mut [u8] = ptr::null_mut::().cast_slice(0); + /// unsafe { + /// danger.as_mut().expect("references must not be null"); + /// } + /// ``` + #[inline] + #[unstable(feature = "ptr_cast_slice", issue = "149103")] + pub const fn cast_slice(self, len: usize) -> *mut [T] { + slice_from_raw_parts_mut(self, len) + } } impl *mut MaybeUninit { /// Casts from a maybe-uninitialized type to its initialized version. diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index aa3af2f18528..2cefc41017f4 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -1377,6 +1377,35 @@ impl NonNull { pub const fn cast_uninit(self) -> NonNull> { self.cast() } + + /// Creates a non-null raw slice from a thin pointer and a length. + /// + /// The `len` argument is the number of **elements**, not the number of bytes. + /// + /// This function is safe, but dereferencing the return value is unsafe. + /// See the documentation of [`slice::from_raw_parts`] for slice safety requirements. + /// + /// # Examples + /// + /// ```rust + /// #![feature(ptr_cast_slice)] + /// use std::ptr::NonNull; + /// + /// // create a slice pointer when starting out with a pointer to the first element + /// let mut x = [5, 6, 7]; + /// let nonnull_pointer = NonNull::new(x.as_mut_ptr()).unwrap(); + /// let slice = nonnull_pointer.cast_slice(3); + /// assert_eq!(unsafe { slice.as_ref()[2] }, 7); + /// ``` + /// + /// (Note that this example artificially demonstrates a use of this method, + /// but `let slice = NonNull::from(&x[..]);` would be a better way to write code like this.) + #[inline] + #[must_use] + #[unstable(feature = "ptr_cast_slice", issue = "149103")] + pub const fn cast_slice(self, len: usize) -> NonNull<[T]> { + NonNull::slice_from_raw_parts(self, len) + } } impl NonNull> { /// Casts from a maybe-uninitialized type to its initialized version. From 12f41b519435e3952f2bf8e2c829e19c982ace04 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 17 Oct 2025 12:52:31 +0000 Subject: [PATCH 0037/1843] More explicit handling of the allocator shim around LTO --- compiler/rustc_codegen_ssa/src/back/write.rs | 56 +++++++++----------- compiler/rustc_codegen_ssa/src/base.rs | 11 +--- 2 files changed, 27 insertions(+), 40 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 3e36bd8552b1..885af95b1f2c 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -334,7 +334,6 @@ pub struct CodegenContext { pub output_filenames: Arc, pub invocation_temp: Option, pub module_config: Arc, - pub allocator_config: Arc, pub tm_factory: TargetMachineFactoryFn, pub msvc_imps_needed: bool, pub is_pe_coff: bool, @@ -799,19 +798,12 @@ pub(crate) fn compute_per_cgu_lto_type( sess_lto: &Lto, opts: &config::Options, sess_crate_types: &[CrateType], - module_kind: ModuleKind, ) -> ComputedLtoType { // If the linker does LTO, we don't have to do it. Note that we // keep doing full LTO, if it is requested, as not to break the // assumption that the output will be a single module. let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator module if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - let is_allocator = module_kind == ModuleKind::Allocator; - // We ignore a request for full crate graph LTO if the crate type // is only an rlib, as there is no full crate graph to process, // that'll happen later. @@ -823,7 +815,7 @@ pub(crate) fn compute_per_cgu_lto_type( let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]); match sess_lto { - Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin, + Lto::ThinLocal if !linker_does_lto => ComputedLtoType::Thin, Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin, Lto::Fat if !is_rlib => ComputedLtoType::Fat, _ => ComputedLtoType::No, @@ -839,23 +831,18 @@ fn execute_optimize_work_item( let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let module_config = match module.kind { - ModuleKind::Regular => &cgcx.module_config, - ModuleKind::Allocator => &cgcx.allocator_config, - }; - - B::optimize(cgcx, dcx, &mut module, module_config); + B::optimize(cgcx, dcx, &mut module, &cgcx.module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it // back to the coordinator thread for further LTO processing (which // has to wait for all the initial modules to be optimized). - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind); + let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types); // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. - let bitcode = if module_config.emit_pre_lto_bc { + let bitcode = if cgcx.module_config.emit_pre_lto_bc { let filename = pre_lto_bitcode_filename(&module.name); cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) } else { @@ -864,7 +851,7 @@ fn execute_optimize_work_item( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } ComputedLtoType::Thin => { @@ -1245,7 +1232,7 @@ fn start_executing_work( coordinator_receive: Receiver>, regular_config: Arc, allocator_config: Arc, - allocator_module: Option>, + mut allocator_module: Option>, coordinator_send: Sender>, ) -> thread::JoinHandle> { let sess = tcx.sess; @@ -1303,7 +1290,6 @@ fn start_executing_work( diag_emitter: shared_emitter.clone(), output_filenames: Arc::clone(tcx.output_filenames(())), module_config: regular_config, - allocator_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), msvc_imps_needed: msvc_imps_needed(tcx), is_pe_coff: tcx.sess.target.is_like_windows, @@ -1497,16 +1483,12 @@ fn start_executing_work( let mut llvm_start_time: Option> = None; - let compiled_allocator_module = allocator_module.and_then(|allocator_module| { - match execute_optimize_work_item(&cgcx, allocator_module) { - WorkItemResult::Finished(compiled_module) => return Some(compiled_module), - WorkItemResult::NeedsFatLto(fat_lto_input) => needs_fat_lto.push(fat_lto_input), - WorkItemResult::NeedsThinLto(name, thin_buffer) => { - needs_thin_lto.push((name, thin_buffer)) - } - } - None - }); + if let Some(allocator_module) = &mut allocator_module { + let dcx = cgcx.create_dcx(); + let dcx = dcx.handle(); + + B::optimize(&cgcx, dcx, allocator_module, &allocator_config); + } // Run the message loop while there's still anything that needs message // processing. Note that as soon as codegen is aborted we simply want to @@ -1733,6 +1715,10 @@ fn start_executing_work( assert!(compiled_modules.is_empty()); assert!(needs_thin_lto.is_empty()); + if let Some(allocator_module) = allocator_module.take() { + needs_fat_lto.push(FatLtoInput::InMemory(allocator_module)); + } + // This uses the implicit token let module = do_fat_lto( &cgcx, @@ -1746,6 +1732,13 @@ fn start_executing_work( assert!(compiled_modules.is_empty()); assert!(needs_fat_lto.is_empty()); + if cgcx.lto != Lto::ThinLocal { + if let Some(allocator_module) = allocator_module.take() { + let (name, thin_buffer) = B::prepare_thin(allocator_module); + needs_thin_lto.push((name, thin_buffer)); + } + } + compiled_modules.extend(do_thin_lto( &cgcx, exported_symbols_for_lto, @@ -1762,7 +1755,8 @@ fn start_executing_work( Ok(CompiledModules { modules: compiled_modules, - allocator_module: compiled_allocator_module, + allocator_module: allocator_module + .map(|allocator_module| B::codegen(&cgcx, allocator_module, &allocator_config)), }) }) .expect("failed to spawn coordinator thread"); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 414e9ce1c821..815a3ee706f8 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -49,9 +49,7 @@ use crate::meth::load_vtable; use crate::mir::operand::OperandValue; use crate::mir::place::PlaceRef; use crate::traits::*; -use crate::{ - CachedModuleCodegen, CodegenLintLevels, CrateInfo, ModuleCodegen, ModuleKind, errors, meth, mir, -}; +use crate::{CachedModuleCodegen, CodegenLintLevels, CrateInfo, ModuleCodegen, errors, meth, mir}; pub(crate) fn bin_op_to_icmp_predicate(op: BinOp, signed: bool) -> IntPredicate { match (op, signed) { @@ -1132,12 +1130,7 @@ pub fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> // We can re-use either the pre- or the post-thinlto state. If no LTO is // being performed then we can use post-LTO artifacts, otherwise we must // reuse pre-LTO artifacts - match compute_per_cgu_lto_type( - &tcx.sess.lto(), - &tcx.sess.opts, - tcx.crate_types(), - ModuleKind::Regular, - ) { + match compute_per_cgu_lto_type(&tcx.sess.lto(), &tcx.sess.opts, tcx.crate_types()) { ComputedLtoType::No => CguReuse::PostLto, _ => CguReuse::PreLto, } From b93b4b003eb1b1dea681d65a5ac9c222b854046d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 23 Oct 2025 09:39:13 +0000 Subject: [PATCH 0038/1843] Remove opts field from CodegenContext --- compiler/rustc_codegen_gcc/src/back/lto.rs | 2 +- compiler/rustc_codegen_llvm/src/back/lto.rs | 2 +- compiler/rustc_codegen_llvm/src/back/write.rs | 6 +++--- compiler/rustc_codegen_ssa/src/back/lto.rs | 6 +++--- compiler/rustc_codegen_ssa/src/back/write.rs | 14 +++++++++----- compiler/rustc_codegen_ssa/src/base.rs | 6 +++++- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs index 404064fb7a06..08c36d2b7318 100644 --- a/compiler/rustc_codegen_gcc/src/back/lto.rs +++ b/compiler/rustc_codegen_gcc/src/back/lto.rs @@ -290,7 +290,7 @@ pub(crate) fn run_thin( let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); - if cgcx.opts.cg.linker_plugin_lto.enabled() { + if cgcx.use_linker_plugin_lto { unreachable!( "We should never reach this case if the LTO step \ is deferred to the linker" diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index b820b992105f..3137ed288fa8 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -179,7 +179,7 @@ pub(crate) fn run_thin( prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); - if cgcx.opts.cg.linker_plugin_lto.enabled() { + if cgcx.use_linker_plugin_lto { unreachable!( "We should never reach this case if the LTO step \ is deferred to the linker" diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 95539059653b..a49d4c010d67 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -398,7 +398,7 @@ impl<'a> DiagnosticHandlers<'a> { }) .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok())); - let pgo_available = cgcx.opts.cg.profile_use.is_some(); + let pgo_available = cgcx.module_config.pgo_use.is_some(); let data = Box::into_raw(Box::new((cgcx, dcx))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); @@ -738,7 +738,7 @@ pub(crate) unsafe fn llvm_optimize( &*module.module_llvm.tm.raw(), to_pass_builder_opt_level(opt_level), opt_stage, - cgcx.opts.cg.linker_plugin_lto.enabled(), + cgcx.use_linker_plugin_lto, config.no_prepopulate_passes, config.verify_llvm_ir, config.lint_llvm_ir, @@ -801,7 +801,7 @@ pub(crate) fn optimize( let opt_stage = match cgcx.lto { Lto::Fat => llvm::OptStage::PreLinkFatLTO, Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.use_linker_plugin_lto => llvm::OptStage::PreLinkThinLTO, _ => llvm::OptStage::PreLinkNoLTO, }; diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index e6df6a2469f3..e3dc985bf782 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -137,15 +137,15 @@ pub(super) fn check_lto_allowed(cgcx: &CodegenContext if !crate_type_allows_lto(*crate_type) { dcx.handle().emit_fatal(LtoDisallowed); } else if *crate_type == CrateType::Dylib { - if !cgcx.opts.unstable_opts.dylib_lto { + if !cgcx.dylib_lto { dcx.handle().emit_fatal(LtoDylib); } - } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto { + } else if *crate_type == CrateType::ProcMacro && !cgcx.dylib_lto { dcx.handle().emit_fatal(LtoProcMacro); } } - if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto { + if cgcx.prefer_dynamic && !cgcx.dylib_lto { dcx.handle().emit_fatal(DynamicLinkingWithLTO); } } diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 885af95b1f2c..bc7c32931613 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -326,10 +326,12 @@ pub struct CodegenContext { // Resources needed when running LTO pub prof: SelfProfilerRef, pub lto: Lto, + pub use_linker_plugin_lto: bool, + pub dylib_lto: bool, + pub prefer_dynamic: bool, pub save_temps: bool, pub fewer_names: bool, pub time_trace: bool, - pub opts: Arc, pub crate_types: Vec, pub output_filenames: Arc, pub invocation_temp: Option, @@ -796,13 +798,12 @@ pub(crate) enum ComputedLtoType { pub(crate) fn compute_per_cgu_lto_type( sess_lto: &Lto, - opts: &config::Options, + linker_does_lto: bool, sess_crate_types: &[CrateType], ) -> ComputedLtoType { // If the linker does LTO, we don't have to do it. Note that we // keep doing full LTO, if it is requested, as not to break the // assumption that the output will be a single module. - let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); // We ignore a request for full crate graph LTO if the crate type // is only an rlib, as there is no full crate graph to process, @@ -838,7 +839,8 @@ fn execute_optimize_work_item( // back to the coordinator thread for further LTO processing (which // has to wait for all the initial modules to be optimized). - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types); + let lto_type = + compute_per_cgu_lto_type(&cgcx.lto, cgcx.use_linker_plugin_lto, &cgcx.crate_types); // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. @@ -1279,10 +1281,12 @@ fn start_executing_work( let cgcx = CodegenContext:: { crate_types: tcx.crate_types().to_vec(), lto: sess.lto(), + use_linker_plugin_lto: sess.opts.cg.linker_plugin_lto.enabled(), + dylib_lto: sess.opts.unstable_opts.dylib_lto, + prefer_dynamic: sess.opts.cg.prefer_dynamic, fewer_names: sess.fewer_names(), save_temps: sess.opts.cg.save_temps, time_trace: sess.opts.unstable_opts.llvm_time_trace, - opts: Arc::new(sess.opts.clone()), prof: sess.prof.clone(), remark: sess.opts.cg.remark.clone(), remark_dir, diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 815a3ee706f8..2943df8f02aa 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -1130,7 +1130,11 @@ pub fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> // We can re-use either the pre- or the post-thinlto state. If no LTO is // being performed then we can use post-LTO artifacts, otherwise we must // reuse pre-LTO artifacts - match compute_per_cgu_lto_type(&tcx.sess.lto(), &tcx.sess.opts, tcx.crate_types()) { + match compute_per_cgu_lto_type( + &tcx.sess.lto(), + tcx.sess.opts.cg.linker_plugin_lto.enabled(), + tcx.crate_types(), + ) { ComputedLtoType::No => CguReuse::PostLto, _ => CguReuse::PreLto, } From 2d7c571391e558a55767dc9aaeca6989a63c77c4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 21 Nov 2025 15:08:09 +0000 Subject: [PATCH 0039/1843] Remove SharedEmitter from CodegenContext --- compiler/rustc_codegen_gcc/src/back/lto.rs | 12 +- compiler/rustc_codegen_gcc/src/back/write.rs | 8 +- compiler/rustc_codegen_gcc/src/lib.rs | 14 ++- compiler/rustc_codegen_llvm/src/back/lto.rs | 27 +++-- compiler/rustc_codegen_llvm/src/back/write.rs | 43 ++++--- compiler/rustc_codegen_llvm/src/lib.rs | 29 +++-- compiler/rustc_codegen_ssa/src/back/lto.rs | 8 +- compiler/rustc_codegen_ssa/src/back/write.rs | 112 +++++++++++------- .../rustc_codegen_ssa/src/traits/write.rs | 8 +- 9 files changed, 165 insertions(+), 96 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs index 08c36d2b7318..24be3ee4c34d 100644 --- a/compiler/rustc_codegen_gcc/src/back/lto.rs +++ b/compiler/rustc_codegen_gcc/src/back/lto.rs @@ -26,11 +26,11 @@ use std::sync::atomic::Ordering; use gccjit::{Context, OutputKind}; use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared}; -use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, SharedEmitter}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::memmap::Mmap; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_log::tracing::info; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; @@ -112,10 +112,11 @@ fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> { /// for further optimization. pub(crate) fn run_fat( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> ModuleCodegen { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); /*let symbols_below_threshold = @@ -283,12 +284,11 @@ impl ModuleBufferMethods for ModuleBuffer { /// can simply be copied over from the incr. comp. cache. pub(crate) fn run_thin( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { - let dcx = cgcx.create_dcx(); - let dcx = dcx.handle(); let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); if cgcx.use_linker_plugin_lto { unreachable!( @@ -522,8 +522,6 @@ pub fn optimize_thin_module( thin_module: ThinModule, _cgcx: &CodegenContext, ) -> ModuleCodegen { - //let dcx = cgcx.create_dcx(); - //let module_name = &thin_module.shared.module_names[thin_module.idx]; /*let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&dcx, e))?;*/ diff --git a/compiler/rustc_codegen_gcc/src/back/write.rs b/compiler/rustc_codegen_gcc/src/back/write.rs index eae0f2aa00f6..b6223c5be370 100644 --- a/compiler/rustc_codegen_gcc/src/back/write.rs +++ b/compiler/rustc_codegen_gcc/src/back/write.rs @@ -2,8 +2,11 @@ use std::{env, fs}; use gccjit::{Context, OutputKind}; use rustc_codegen_ssa::back::link::ensure_removed; -use rustc_codegen_ssa::back::write::{BitcodeSection, CodegenContext, EmitObj, ModuleConfig}; +use rustc_codegen_ssa::back::write::{ + BitcodeSection, CodegenContext, EmitObj, ModuleConfig, SharedEmitter, +}; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_errors::DiagCtxt; use rustc_fs_util::link_or_copy; use rustc_log::tracing::debug; use rustc_session::config::OutputType; @@ -15,10 +18,11 @@ use crate::{GccCodegenBackend, GccContext, LtoMode}; pub(crate) fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); let _timer = cgcx.prof.generic_activity_with_arg("GCC_module_codegen", &*module.name); diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index 409b7886740a..5a2ec0a2fc68 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -82,7 +82,7 @@ use gccjit::{TargetInfo, Version}; use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn, + CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter, TargetMachineFactoryFn, }; use rustc_codegen_ssa::base::codegen_crate; use rustc_codegen_ssa::target_features::cfg_target_feature; @@ -371,23 +371,25 @@ impl WriteBackendMethods for GccCodegenBackend { fn run_and_optimize_fat_lto( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, // FIXME(bjorn3): Limit LTO exports to these symbols _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> ModuleCodegen { - back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules) + back::lto::run_fat(cgcx, shared_emitter, each_linked_rlib_for_lto, modules) } fn run_thin_lto( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, // FIXME(bjorn3): Limit LTO exports to these symbols _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { - back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules) + back::lto::run_thin(cgcx, dcx, each_linked_rlib_for_lto, modules, cached_modules) } fn print_pass_timings(&self) { @@ -400,7 +402,7 @@ impl WriteBackendMethods for GccCodegenBackend { fn optimize( _cgcx: &CodegenContext, - _dcx: DiagCtxtHandle<'_>, + _shared_emitter: &SharedEmitter, module: &mut ModuleCodegen, config: &ModuleConfig, ) { @@ -409,6 +411,7 @@ impl WriteBackendMethods for GccCodegenBackend { fn optimize_thin( cgcx: &CodegenContext, + _shared_emitter: &SharedEmitter, thin: ThinModule, ) -> ModuleCodegen { back::lto::optimize_thin_module(thin, cgcx) @@ -416,10 +419,11 @@ impl WriteBackendMethods for GccCodegenBackend { fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { - back::write::codegen(cgcx, module, config) + back::write::codegen(cgcx, shared_emitter, module, config) } fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 3137ed288fa8..3210afc063a9 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -9,12 +9,12 @@ use std::{io, iter, slice}; use object::read::archive::ArchiveFile; use object::{Object, ObjectSection}; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared}; -use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, SharedEmitter}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_hir::attrs::SanitizerSet; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; @@ -150,17 +150,18 @@ fn get_bitcode_slice_from_object_data<'a>( /// for further optimization. pub(crate) fn run_fat( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> ModuleCodegen { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); - fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold) + fat_lto(cgcx, dcx, shared_emitter, modules, upstream_modules, &symbols_below_threshold) } /// Performs thin LTO by performing necessary global analysis and returning two @@ -168,13 +169,12 @@ pub(crate) fn run_fat( /// can simply be copied over from the incr. comp. cache. pub(crate) fn run_thin( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { - let dcx = cgcx.create_dcx(); - let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = @@ -197,6 +197,7 @@ pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBu fn fat_lto( cgcx: &CodegenContext, dcx: DiagCtxtHandle<'_>, + shared_emitter: &SharedEmitter, modules: Vec>, mut serialized_modules: Vec<(SerializedModule, CString)>, symbols_below_threshold: &[*const libc::c_char], @@ -265,8 +266,13 @@ fn fat_lto( // The linking steps below may produce errors and diagnostics within LLVM // which we'd like to handle and print, so set up our diagnostic handlers // (which get unregistered when they go out of scope below). - let _handler = - DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::LTO); + let _handler = DiagnosticHandlers::new( + cgcx, + shared_emitter, + llcx, + &module, + CodegenDiagnosticsStage::LTO, + ); // For all other modules we codegened we'll need to link them into our own // bitcode. All modules were codegened in their own LLVM context, however, @@ -730,10 +736,11 @@ impl Drop for ThinBuffer { } pub(crate) fn optimize_thin_module( - thin_module: ThinModule, cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, + thin_module: ThinModule, ) -> ModuleCodegen { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); let module_name = &thin_module.shared.module_names[thin_module.idx]; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index a49d4c010d67..a5b6ea08a66d 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -9,7 +9,7 @@ use libc::{c_char, c_int, c_void, size_t}; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::back::write::{ - BitcodeSection, CodegenContext, EmitObj, InlineAsmError, ModuleConfig, + BitcodeSection, CodegenContext, EmitObj, InlineAsmError, ModuleConfig, SharedEmitter, TargetMachineFactoryConfig, TargetMachineFactoryFn, }; use rustc_codegen_ssa::base::wants_wasm_eh; @@ -17,7 +17,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{DiagCtxtHandle, Level}; +use rustc_errors::{DiagCtxt, DiagCtxtHandle, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -356,7 +356,7 @@ pub(crate) enum CodegenDiagnosticsStage { } pub(crate) struct DiagnosticHandlers<'a> { - data: *mut (&'a CodegenContext, DiagCtxtHandle<'a>), + data: *mut (&'a CodegenContext, &'a SharedEmitter), llcx: &'a llvm::Context, old_handler: Option<&'a llvm::DiagnosticHandler>, } @@ -364,7 +364,7 @@ pub(crate) struct DiagnosticHandlers<'a> { impl<'a> DiagnosticHandlers<'a> { pub(crate) fn new( cgcx: &'a CodegenContext, - dcx: DiagCtxtHandle<'a>, + shared_emitter: &'a SharedEmitter, llcx: &'a llvm::Context, module: &ModuleCodegen, stage: CodegenDiagnosticsStage, @@ -399,7 +399,7 @@ impl<'a> DiagnosticHandlers<'a> { .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok())); let pgo_available = cgcx.module_config.pgo_use.is_some(); - let data = Box::into_raw(Box::new((cgcx, dcx))); + let data = Box::into_raw(Box::new((cgcx, shared_emitter))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); llvm::LLVMRustContextConfigureDiagnosticHandler( @@ -461,12 +461,16 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void if user.is_null() { return; } - let (cgcx, dcx) = - unsafe { *(user as *const (&CodegenContext, DiagCtxtHandle<'_>)) }; + let (cgcx, shared_emitter) = + unsafe { *(user as *const (&CodegenContext, &SharedEmitter)) }; + + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + let dcx = dcx.handle(); match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } { llvm::diagnostic::InlineAsm(inline) => { - cgcx.diag_emitter.inline_asm_error(report_inline_asm( + // FIXME use dcx + shared_emitter.inline_asm_error(report_inline_asm( cgcx, inline.message, inline.level, @@ -777,14 +781,18 @@ pub(crate) unsafe fn llvm_optimize( // Unsafe due to LLVM calls. pub(crate) fn optimize( cgcx: &CodegenContext, - dcx: DiagCtxtHandle<'_>, + shared_emitter: &SharedEmitter, module: &mut ModuleCodegen, config: &ModuleConfig, ) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + let dcx = dcx.handle(); + let llcx = &*module.module_llvm.llcx; - let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt); + let _handlers = + DiagnosticHandlers::new(cgcx, shared_emitter, llcx, module, CodegenDiagnosticsStage::Opt); if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext_for_cgu( @@ -865,19 +873,26 @@ pub(crate) fn optimize( pub(crate) fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { - let dcx = cgcx.create_dcx(); + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name); + + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); - let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name); { let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; let tm = &*module.module_llvm.tm; - let _handlers = - DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen); + let _handlers = DiagnosticHandlers::new( + cgcx, + shared_emitter, + llcx, + &module, + CodegenDiagnosticsStage::Codegen, + ); if cgcx.msvc_imps_needed { create_msvc_imps(cgcx, llcx, llmod); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 1b65a133d58c..3901662442e9 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -30,12 +30,13 @@ use llvm_util::target_config; use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, + CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter, TargetMachineFactoryConfig, + TargetMachineFactoryFn, }; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetConfig}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -166,14 +167,20 @@ impl WriteBackendMethods for LlvmCodegenBackend { } fn run_and_optimize_fat_lto( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> ModuleCodegen { - let mut module = - back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules); + let mut module = back::lto::run_fat( + cgcx, + shared_emitter, + exported_symbols_for_lto, + each_linked_rlib_for_lto, + modules, + ); - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); back::lto::run_pass_manager(cgcx, dcx, &mut module, false); @@ -181,6 +188,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { } fn run_thin_lto( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, @@ -188,6 +196,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { ) -> (Vec>, Vec) { back::lto::run_thin( cgcx, + dcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules, @@ -196,24 +205,26 @@ impl WriteBackendMethods for LlvmCodegenBackend { } fn optimize( cgcx: &CodegenContext, - dcx: DiagCtxtHandle<'_>, + shared_emitter: &SharedEmitter, module: &mut ModuleCodegen, config: &ModuleConfig, ) { - back::write::optimize(cgcx, dcx, module, config) + back::write::optimize(cgcx, shared_emitter, module, config) } fn optimize_thin( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, thin: ThinModule, ) -> ModuleCodegen { - back::lto::optimize_thin_module(thin, cgcx) + back::lto::optimize_thin_module(cgcx, shared_emitter, thin) } fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { - back::write::codegen(cgcx, module, config) + back::write::codegen(cgcx, shared_emitter, module, config) } fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { back::lto::prepare_thin(module) diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index e3dc985bf782..ef4c193c4c2a 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -2,6 +2,7 @@ use std::ffi::CString; use std::sync::Arc; use rustc_data_structures::memmap::Mmap; +use rustc_errors::DiagCtxtHandle; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportLevel}; use rustc_middle::ty::TyCtxt; @@ -124,14 +125,15 @@ pub(super) fn exported_symbols_for_lto( symbols_below_threshold } -pub(super) fn check_lto_allowed(cgcx: &CodegenContext) { +pub(super) fn check_lto_allowed( + cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, +) { if cgcx.lto == Lto::ThinLocal { // Crate local LTO is always allowed return; } - let dcx = cgcx.create_dcx(); - // Make sure we actually can run LTO for crate_type in cgcx.crate_types.iter() { if !crate_type_allows_lto(*crate_type) { diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index bc7c32931613..76f76c84dbdc 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -15,8 +15,8 @@ use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard}; use rustc_errors::emitter::Emitter; use rustc_errors::translation::Translator; use rustc_errors::{ - Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FatalErrorMarker, Level, - MultiSpan, Style, Suggestions, + Diag, DiagArgMap, DiagCtxt, DiagCtxtHandle, DiagMessage, ErrCode, FatalError, FatalErrorMarker, + Level, MultiSpan, Style, Suggestions, }; use rustc_fs_util::link_or_copy; use rustc_incremental::{ @@ -348,8 +348,6 @@ pub struct CodegenContext { pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, pub pointer_size: Size, - /// Emitter to use for diagnostics produced during codegen. - pub diag_emitter: SharedEmitter, /// LLVM optimizations for which we want to print remarks. pub remark: Passes, /// Directory into which should the LLVM optimization remarks be written. @@ -364,14 +362,9 @@ pub struct CodegenContext { pub parallel: bool, } -impl CodegenContext { - pub fn create_dcx(&self) -> DiagCtxt { - DiagCtxt::new(Box::new(self.diag_emitter.clone())) - } -} - fn generate_thin_lto_work( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], needs_thin_lto: Vec<(String, B::ThinBuffer)>, @@ -381,6 +374,7 @@ fn generate_thin_lto_work( let (lto_modules, copy_jobs) = B::run_thin_lto( cgcx, + dcx, exported_symbols_for_lto, each_linked_rlib_for_lto, needs_thin_lto, @@ -825,14 +819,12 @@ pub(crate) fn compute_per_cgu_lto_type( fn execute_optimize_work_item( cgcx: &CodegenContext, + shared_emitter: SharedEmitter, mut module: ModuleCodegen, ) -> WorkItemResult { let _timer = cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*module.name); - let dcx = cgcx.create_dcx(); - let dcx = dcx.handle(); - - B::optimize(cgcx, dcx, &mut module, &cgcx.module_config); + B::optimize(cgcx, &shared_emitter, &mut module, &cgcx.module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it @@ -853,7 +845,7 @@ fn execute_optimize_work_item( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, &cgcx.module_config); + let module = B::codegen(cgcx, &shared_emitter, module, &cgcx.module_config); WorkItemResult::Finished(module) } ComputedLtoType::Thin => { @@ -883,12 +875,16 @@ fn execute_optimize_work_item( fn execute_copy_from_cache_work_item( cgcx: &CodegenContext, + shared_emitter: SharedEmitter, module: CachedModuleCodegen, ) -> CompiledModule { let _timer = cgcx .prof .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*module.name); + let dcx = DiagCtxt::new(Box::new(shared_emitter)); + let dcx = dcx.handle(); + let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); let mut links_from_incr_cache = Vec::new(); @@ -907,11 +903,7 @@ fn execute_copy_from_cache_work_item( Some(output_path) } Err(error) => { - cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf { - source_file, - output_path, - error, - }); + dcx.emit_err(errors::CopyPathBuf { source_file, output_path, error }); None } } @@ -954,7 +946,7 @@ fn execute_copy_from_cache_work_item( let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode); let object = load_from_incr_cache(should_emit_obj, OutputType::Object); if should_emit_obj && object.is_none() { - cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name }) + dcx.emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name }) } CompiledModule { @@ -971,6 +963,7 @@ fn execute_copy_from_cache_work_item( fn do_fat_lto( cgcx: &CodegenContext, + shared_emitter: SharedEmitter, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], mut needs_fat_lto: Vec>, @@ -978,7 +971,10 @@ fn do_fat_lto( ) -> CompiledModule { let _timer = cgcx.prof.verbose_generic_activity("LLVM_fatlto"); - check_lto_allowed(&cgcx); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + let dcx = dcx.handle(); + + check_lto_allowed(&cgcx, dcx); for (module, wp) in import_only_modules { needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module }) @@ -986,15 +982,17 @@ fn do_fat_lto( let module = B::run_and_optimize_fat_lto( cgcx, + &shared_emitter, exported_symbols_for_lto, each_linked_rlib_for_lto, needs_fat_lto, ); - B::codegen(cgcx, module, &cgcx.module_config) + B::codegen(cgcx, &shared_emitter, module, &cgcx.module_config) } fn do_thin_lto<'a, B: ExtraBackendMethods>( cgcx: &'a CodegenContext, + shared_emitter: SharedEmitter, exported_symbols_for_lto: Arc>, each_linked_rlib_for_lto: Vec, needs_thin_lto: Vec<(String, ::ThinBuffer)>, @@ -1005,7 +1003,10 @@ fn do_thin_lto<'a, B: ExtraBackendMethods>( ) -> Vec { let _timer = cgcx.prof.verbose_generic_activity("LLVM_thinlto"); - check_lto_allowed(&cgcx); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + let dcx = dcx.handle(); + + check_lto_allowed(&cgcx, dcx); let (coordinator_send, coordinator_receive) = channel(); @@ -1030,6 +1031,7 @@ fn do_thin_lto<'a, B: ExtraBackendMethods>( // we don't worry about tokens. for (work, cost) in generate_thin_lto_work( cgcx, + dcx, &exported_symbols_for_lto, &each_linked_rlib_for_lto, needs_thin_lto, @@ -1071,7 +1073,7 @@ fn do_thin_lto<'a, B: ExtraBackendMethods>( while used_token_count < tokens.len() + 1 && let Some((item, _)) = work_items.pop() { - spawn_thin_lto_work(&cgcx, coordinator_send.clone(), item); + spawn_thin_lto_work(&cgcx, shared_emitter.clone(), coordinator_send.clone(), item); used_token_count += 1; } } else { @@ -1095,7 +1097,7 @@ fn do_thin_lto<'a, B: ExtraBackendMethods>( } Err(e) => { let msg = &format!("failed to acquire jobserver token: {e}"); - cgcx.diag_emitter.fatal(msg); + shared_emitter.fatal(msg); codegen_aborted = Some(FatalError); } }, @@ -1133,12 +1135,13 @@ fn do_thin_lto<'a, B: ExtraBackendMethods>( fn execute_thin_lto_work_item( cgcx: &CodegenContext, + shared_emitter: SharedEmitter, module: lto::ThinModule, ) -> CompiledModule { let _timer = cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", module.name()); - let module = B::optimize_thin(cgcx, module); - B::codegen(cgcx, module, &cgcx.module_config) + let module = B::optimize_thin(cgcx, &shared_emitter, module); + B::codegen(cgcx, &shared_emitter, module, &cgcx.module_config) } /// Messages sent to the coordinator. @@ -1291,7 +1294,6 @@ fn start_executing_work( remark: sess.opts.cg.remark.clone(), remark_dir, incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), - diag_emitter: shared_emitter.clone(), output_filenames: Arc::clone(tcx.output_filenames(())), module_config: regular_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), @@ -1488,10 +1490,7 @@ fn start_executing_work( let mut llvm_start_time: Option> = None; if let Some(allocator_module) = &mut allocator_module { - let dcx = cgcx.create_dcx(); - let dcx = dcx.handle(); - - B::optimize(&cgcx, dcx, allocator_module, &allocator_config); + B::optimize(&cgcx, &shared_emitter, allocator_module, &allocator_config); } // Run the message loop while there's still anything that needs message @@ -1529,7 +1528,13 @@ fn start_executing_work( let (item, _) = work_items.pop().expect("queue empty - queue_full_enough() broken?"); main_thread_state = MainThreadState::Lending; - spawn_work(&cgcx, coordinator_send.clone(), &mut llvm_start_time, item); + spawn_work( + &cgcx, + shared_emitter.clone(), + coordinator_send.clone(), + &mut llvm_start_time, + item, + ); } } } else if codegen_state == Completed { @@ -1547,7 +1552,13 @@ fn start_executing_work( MainThreadState::Idle => { if let Some((item, _)) = work_items.pop() { main_thread_state = MainThreadState::Lending; - spawn_work(&cgcx, coordinator_send.clone(), &mut llvm_start_time, item); + spawn_work( + &cgcx, + shared_emitter.clone(), + coordinator_send.clone(), + &mut llvm_start_time, + item, + ); } else { // There is no unstarted work, so let the main thread // take over for a running worker. Otherwise the @@ -1583,7 +1594,13 @@ fn start_executing_work( while running_with_own_token < tokens.len() && let Some((item, _)) = work_items.pop() { - spawn_work(&cgcx, coordinator_send.clone(), &mut llvm_start_time, item); + spawn_work( + &cgcx, + shared_emitter.clone(), + coordinator_send.clone(), + &mut llvm_start_time, + item, + ); running_with_own_token += 1; } } @@ -1726,6 +1743,7 @@ fn start_executing_work( // This uses the implicit token let module = do_fat_lto( &cgcx, + shared_emitter.clone(), &exported_symbols_for_lto, &each_linked_rlib_file_for_lto, needs_fat_lto, @@ -1745,6 +1763,7 @@ fn start_executing_work( compiled_modules.extend(do_thin_lto( &cgcx, + shared_emitter.clone(), exported_symbols_for_lto, each_linked_rlib_file_for_lto, needs_thin_lto, @@ -1759,8 +1778,9 @@ fn start_executing_work( Ok(CompiledModules { modules: compiled_modules, - allocator_module: allocator_module - .map(|allocator_module| B::codegen(&cgcx, allocator_module, &allocator_config)), + allocator_module: allocator_module.map(|allocator_module| { + B::codegen(&cgcx, &shared_emitter, allocator_module, &allocator_config) + }), }) }) .expect("failed to spawn coordinator thread"); @@ -1829,6 +1849,7 @@ pub(crate) struct WorkerFatalError; fn spawn_work<'a, B: ExtraBackendMethods>( cgcx: &'a CodegenContext, + shared_emitter: SharedEmitter, coordinator_send: Sender>, llvm_start_time: &mut Option>, work: WorkItem, @@ -1841,10 +1862,10 @@ fn spawn_work<'a, B: ExtraBackendMethods>( B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { - WorkItem::Optimize(m) => execute_optimize_work_item(&cgcx, m), - WorkItem::CopyPostLtoArtifacts(m) => { - WorkItemResult::Finished(execute_copy_from_cache_work_item(&cgcx, m)) - } + WorkItem::Optimize(m) => execute_optimize_work_item(&cgcx, shared_emitter, m), + WorkItem::CopyPostLtoArtifacts(m) => WorkItemResult::Finished( + execute_copy_from_cache_work_item(&cgcx, shared_emitter, m), + ), })); let msg = match result { @@ -1866,6 +1887,7 @@ fn spawn_work<'a, B: ExtraBackendMethods>( fn spawn_thin_lto_work<'a, B: ExtraBackendMethods>( cgcx: &'a CodegenContext, + shared_emitter: SharedEmitter, coordinator_send: Sender, work: ThinLtoWorkItem, ) { @@ -1873,8 +1895,10 @@ fn spawn_thin_lto_work<'a, B: ExtraBackendMethods>( B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { - ThinLtoWorkItem::CopyPostLtoArtifacts(m) => execute_copy_from_cache_work_item(&cgcx, m), - ThinLtoWorkItem::ThinLto(m) => execute_thin_lto_work_item(&cgcx, m), + ThinLtoWorkItem::CopyPostLtoArtifacts(m) => { + execute_copy_from_cache_work_item(&cgcx, shared_emitter, m) + } + ThinLtoWorkItem::ThinLto(m) => execute_thin_lto_work_item(&cgcx, shared_emitter, m), })); let msg = match result { diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index 1ac1d7ef2e2e..e1d23841118c 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -4,7 +4,7 @@ use rustc_errors::DiagCtxtHandle; use rustc_middle::dep_graph::WorkProduct; use crate::back::lto::{SerializedModule, ThinModule}; -use crate::back::write::{CodegenContext, FatLtoInput, ModuleConfig}; +use crate::back::write::{CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter}; use crate::{CompiledModule, ModuleCodegen}; pub trait WriteBackendMethods: Clone + 'static { @@ -19,6 +19,7 @@ pub trait WriteBackendMethods: Clone + 'static { /// if necessary and running any further optimizations fn run_and_optimize_fat_lto( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, @@ -28,6 +29,7 @@ pub trait WriteBackendMethods: Clone + 'static { /// can simply be copied over from the incr. comp. cache. fn run_thin_lto( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, @@ -37,16 +39,18 @@ pub trait WriteBackendMethods: Clone + 'static { fn print_statistics(&self); fn optimize( cgcx: &CodegenContext, - dcx: DiagCtxtHandle<'_>, + shared_emitter: &SharedEmitter, module: &mut ModuleCodegen, config: &ModuleConfig, ); fn optimize_thin( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, thin: ThinModule, ) -> ModuleCodegen; fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule; From 67c61cefaf87cb842b2248935dcc481faad21b14 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 23 Oct 2025 09:39:13 +0000 Subject: [PATCH 0040/1843] Remove opts field from CodegenContext --- src/back/lto.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/back/lto.rs b/src/back/lto.rs index 404064fb7a06..08c36d2b7318 100644 --- a/src/back/lto.rs +++ b/src/back/lto.rs @@ -290,7 +290,7 @@ pub(crate) fn run_thin( let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); - if cgcx.opts.cg.linker_plugin_lto.enabled() { + if cgcx.use_linker_plugin_lto { unreachable!( "We should never reach this case if the LTO step \ is deferred to the linker" From b13bb4b2da208c2c26d000354816400243df1b4a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 21 Nov 2025 15:26:47 +0000 Subject: [PATCH 0041/1843] Move LTO to OngoingCodegen::join --- compiler/rustc_codegen_ssa/src/back/write.rs | 140 +++++++++++++++---- compiler/rustc_driver_impl/src/lib.rs | 20 +-- compiler/rustc_errors/src/lib.rs | 2 +- compiler/rustc_span/src/fatal_error.rs | 17 +++ 4 files changed, 131 insertions(+), 48 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 76f76c84dbdc..ddcd8decf303 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -16,7 +16,7 @@ use rustc_errors::emitter::Emitter; use rustc_errors::translation::Translator; use rustc_errors::{ Diag, DiagArgMap, DiagCtxt, DiagCtxtHandle, DiagMessage, ErrCode, FatalError, FatalErrorMarker, - Level, MultiSpan, Style, Suggestions, + Level, MultiSpan, Style, Suggestions, catch_fatal_errors, }; use rustc_fs_util::link_or_copy; use rustc_incremental::{ @@ -403,6 +403,29 @@ struct CompiledModules { allocator_module: Option, } +enum MaybeLtoModules { + NoLto { + modules: Vec, + allocator_module: Option, + }, + FatLto { + cgcx: CodegenContext, + exported_symbols_for_lto: Arc>, + each_linked_rlib_file_for_lto: Vec, + needs_fat_lto: Vec>, + lto_import_only_modules: + Vec<(SerializedModule<::ModuleBuffer>, WorkProduct)>, + }, + ThinLto { + cgcx: CodegenContext, + exported_symbols_for_lto: Arc>, + each_linked_rlib_file_for_lto: Vec, + needs_thin_lto: Vec<(String, ::ThinBuffer)>, + lto_import_only_modules: + Vec<(SerializedModule<::ModuleBuffer>, WorkProduct)>, + }, +} + fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool { let sess = tcx.sess; sess.opts.cg.embed_bitcode @@ -1239,7 +1262,7 @@ fn start_executing_work( allocator_config: Arc, mut allocator_module: Option>, coordinator_send: Sender>, -) -> thread::JoinHandle> { +) -> thread::JoinHandle, ()>> { let sess = tcx.sess; let mut each_linked_rlib_for_lto = Vec::new(); @@ -1740,43 +1763,43 @@ fn start_executing_work( needs_fat_lto.push(FatLtoInput::InMemory(allocator_module)); } - // This uses the implicit token - let module = do_fat_lto( - &cgcx, - shared_emitter.clone(), - &exported_symbols_for_lto, - &each_linked_rlib_file_for_lto, + return Ok(MaybeLtoModules::FatLto { + cgcx, + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, needs_fat_lto, lto_import_only_modules, - ); - compiled_modules.push(module); + }); } else if !needs_thin_lto.is_empty() || !lto_import_only_modules.is_empty() { assert!(compiled_modules.is_empty()); assert!(needs_fat_lto.is_empty()); - if cgcx.lto != Lto::ThinLocal { + if cgcx.lto == Lto::ThinLocal { + compiled_modules.extend(do_thin_lto( + &cgcx, + shared_emitter.clone(), + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, + needs_thin_lto, + lto_import_only_modules, + )); + } else { if let Some(allocator_module) = allocator_module.take() { let (name, thin_buffer) = B::prepare_thin(allocator_module); needs_thin_lto.push((name, thin_buffer)); } - } - compiled_modules.extend(do_thin_lto( - &cgcx, - shared_emitter.clone(), - exported_symbols_for_lto, - each_linked_rlib_file_for_lto, - needs_thin_lto, - lto_import_only_modules, - )); + return Ok(MaybeLtoModules::ThinLto { + cgcx, + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, + needs_thin_lto, + lto_import_only_modules, + }); + } } - // Regardless of what order these modules completed in, report them to - // the backend in the same order every time to ensure that we're handing - // out deterministic results. - compiled_modules.sort_by(|a, b| a.name.cmp(&b.name)); - - Ok(CompiledModules { + Ok(MaybeLtoModules::NoLto { modules: compiled_modules, allocator_module: allocator_module.map(|allocator_module| { B::codegen(&cgcx, &shared_emitter, allocator_module, &allocator_config) @@ -2074,13 +2097,13 @@ impl SharedEmitterMain { pub struct Coordinator { sender: Sender>, - future: Option>>, + future: Option, ()>>>, // Only used for the Message type. phantom: PhantomData, } impl Coordinator { - fn join(mut self) -> std::thread::Result> { + fn join(mut self) -> std::thread::Result, ()>> { self.future.take().unwrap().join() } } @@ -2111,8 +2134,9 @@ pub struct OngoingCodegen { impl OngoingCodegen { pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap) { self.shared_emitter_main.check(sess, true); - let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() { - Ok(Ok(compiled_modules)) => compiled_modules, + + let maybe_lto_modules = sess.time("join_worker_thread", || match self.coordinator.join() { + Ok(Ok(maybe_lto_modules)) => maybe_lto_modules, Ok(Err(())) => { sess.dcx().abort_if_errors(); panic!("expected abort due to worker thread errors") @@ -2124,6 +2148,62 @@ impl OngoingCodegen { sess.dcx().abort_if_errors(); + let (shared_emitter, shared_emitter_main) = SharedEmitter::new(); + + // Catch fatal errors to ensure shared_emitter_main.check() can emit the actual diagnostics + let compiled_modules = catch_fatal_errors(|| match maybe_lto_modules { + MaybeLtoModules::NoLto { modules, allocator_module } => { + drop(shared_emitter); + CompiledModules { modules, allocator_module } + } + MaybeLtoModules::FatLto { + cgcx, + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, + needs_fat_lto, + lto_import_only_modules, + } => CompiledModules { + modules: vec![do_fat_lto( + &cgcx, + shared_emitter, + &exported_symbols_for_lto, + &each_linked_rlib_file_for_lto, + needs_fat_lto, + lto_import_only_modules, + )], + allocator_module: None, + }, + MaybeLtoModules::ThinLto { + cgcx, + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, + needs_thin_lto, + lto_import_only_modules, + } => CompiledModules { + modules: do_thin_lto( + &cgcx, + shared_emitter, + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, + needs_thin_lto, + lto_import_only_modules, + ), + allocator_module: None, + }, + }); + + shared_emitter_main.check(sess, true); + + sess.dcx().abort_if_errors(); + + let mut compiled_modules = + compiled_modules.expect("fatal error emitted but not sent to SharedEmitter"); + + // Regardless of what order these modules completed in, report them to + // the backend in the same order every time to ensure that we're handing + // out deterministic results. + compiled_modules.modules.sort_by(|a, b| a.name.cmp(&b.name)); + let work_products = copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules); produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames); diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 9a3d7cc506cf..cf5a5fdd234c 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -19,7 +19,7 @@ use std::ffi::OsString; use std::fmt::Write as _; use std::fs::{self, File}; use std::io::{self, IsTerminal, Read, Write}; -use std::panic::{self, PanicHookInfo, catch_unwind}; +use std::panic::{self, PanicHookInfo}; use std::path::{Path, PathBuf}; use std::process::{self, Command, Stdio}; use std::sync::OnceLock; @@ -33,10 +33,11 @@ use rustc_codegen_ssa::{CodegenErrors, CodegenResults}; use rustc_data_structures::profiling::{ TimePassesFormat, get_resident_set_size, print_time_passes_entry, }; +pub use rustc_errors::catch_fatal_errors; use rustc_errors::emitter::stderr_destination; use rustc_errors::registry::Registry; use rustc_errors::translation::Translator; -use rustc_errors::{ColorConfig, DiagCtxt, ErrCode, FatalError, PResult, markdown}; +use rustc_errors::{ColorConfig, DiagCtxt, ErrCode, PResult, markdown}; use rustc_feature::find_gated_cfg; // This avoids a false positive with `-Wunused_crate_dependencies`. // `rust_index` isn't used in this crate's code, but it must be named in the @@ -1306,21 +1307,6 @@ fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> { parser.parse_inner_attributes() } -/// Runs a closure and catches unwinds triggered by fatal errors. -/// -/// The compiler currently unwinds with a special sentinel value to abort -/// compilation on fatal errors. This function catches that sentinel and turns -/// the panic into a `Result` instead. -pub fn catch_fatal_errors R, R>(f: F) -> Result { - catch_unwind(panic::AssertUnwindSafe(f)).map_err(|value| { - if value.is::() { - FatalError - } else { - panic::resume_unwind(value); - } - }) -} - /// Variant of `catch_fatal_errors` for the `interface::Result` return type /// that also computes the exit code. pub fn catch_with_exit_code(f: impl FnOnce()) -> i32 { diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index bbdda155496f..ada32bf0c283 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -68,7 +68,7 @@ pub use rustc_lint_defs::{Applicability, listify, pluralize}; use rustc_lint_defs::{Lint, LintExpectationId}; use rustc_macros::{Decodable, Encodable}; pub use rustc_span::ErrorGuaranteed; -pub use rustc_span::fatal_error::{FatalError, FatalErrorMarker}; +pub use rustc_span::fatal_error::{FatalError, FatalErrorMarker, catch_fatal_errors}; use rustc_span::source_map::SourceMap; use rustc_span::{BytePos, DUMMY_SP, Loc, Span}; pub use snippet::Style; diff --git a/compiler/rustc_span/src/fatal_error.rs b/compiler/rustc_span/src/fatal_error.rs index 26c5711099c6..5e2d82681a11 100644 --- a/compiler/rustc_span/src/fatal_error.rs +++ b/compiler/rustc_span/src/fatal_error.rs @@ -3,6 +3,8 @@ #[must_use] pub struct FatalError; +use std::panic; + pub use rustc_data_structures::FatalErrorMarker; // Don't implement Send on FatalError. This makes it impossible to `panic_any!(FatalError)`. @@ -22,3 +24,18 @@ impl std::fmt::Display for FatalError { } impl std::error::Error for FatalError {} + +/// Runs a closure and catches unwinds triggered by fatal errors. +/// +/// The compiler currently unwinds with a special sentinel value to abort +/// compilation on fatal errors. This function catches that sentinel and turns +/// the panic into a `Result` instead. +pub fn catch_fatal_errors R, R>(f: F) -> Result { + panic::catch_unwind(panic::AssertUnwindSafe(f)).map_err(|value| { + if value.is::() { + FatalError + } else { + panic::resume_unwind(value); + } + }) +} From 385590e9dc85761402f278c61fd081c24c184cb6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 21 Nov 2025 15:08:09 +0000 Subject: [PATCH 0042/1843] Remove SharedEmitter from CodegenContext --- src/back/lto.rs | 12 +++++------- src/back/write.rs | 8 ++++++-- src/lib.rs | 14 +++++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/back/lto.rs b/src/back/lto.rs index 08c36d2b7318..24be3ee4c34d 100644 --- a/src/back/lto.rs +++ b/src/back/lto.rs @@ -26,11 +26,11 @@ use std::sync::atomic::Ordering; use gccjit::{Context, OutputKind}; use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared}; -use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, SharedEmitter}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::memmap::Mmap; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_log::tracing::info; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; @@ -112,10 +112,11 @@ fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> { /// for further optimization. pub(crate) fn run_fat( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> ModuleCodegen { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); /*let symbols_below_threshold = @@ -283,12 +284,11 @@ impl ModuleBufferMethods for ModuleBuffer { /// can simply be copied over from the incr. comp. cache. pub(crate) fn run_thin( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { - let dcx = cgcx.create_dcx(); - let dcx = dcx.handle(); let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); if cgcx.use_linker_plugin_lto { unreachable!( @@ -522,8 +522,6 @@ pub fn optimize_thin_module( thin_module: ThinModule, _cgcx: &CodegenContext, ) -> ModuleCodegen { - //let dcx = cgcx.create_dcx(); - //let module_name = &thin_module.shared.module_names[thin_module.idx]; /*let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&dcx, e))?;*/ diff --git a/src/back/write.rs b/src/back/write.rs index eae0f2aa00f6..b6223c5be370 100644 --- a/src/back/write.rs +++ b/src/back/write.rs @@ -2,8 +2,11 @@ use std::{env, fs}; use gccjit::{Context, OutputKind}; use rustc_codegen_ssa::back::link::ensure_removed; -use rustc_codegen_ssa::back::write::{BitcodeSection, CodegenContext, EmitObj, ModuleConfig}; +use rustc_codegen_ssa::back::write::{ + BitcodeSection, CodegenContext, EmitObj, ModuleConfig, SharedEmitter, +}; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_errors::DiagCtxt; use rustc_fs_util::link_or_copy; use rustc_log::tracing::debug; use rustc_session::config::OutputType; @@ -15,10 +18,11 @@ use crate::{GccCodegenBackend, GccContext, LtoMode}; pub(crate) fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { - let dcx = cgcx.create_dcx(); + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); let dcx = dcx.handle(); let _timer = cgcx.prof.generic_activity_with_arg("GCC_module_codegen", &*module.name); diff --git a/src/lib.rs b/src/lib.rs index 409b7886740a..5a2ec0a2fc68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,7 +82,7 @@ use gccjit::{TargetInfo, Version}; use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn, + CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter, TargetMachineFactoryFn, }; use rustc_codegen_ssa::base::codegen_crate; use rustc_codegen_ssa::target_features::cfg_target_feature; @@ -371,23 +371,25 @@ impl WriteBackendMethods for GccCodegenBackend { fn run_and_optimize_fat_lto( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, // FIXME(bjorn3): Limit LTO exports to these symbols _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> ModuleCodegen { - back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules) + back::lto::run_fat(cgcx, shared_emitter, each_linked_rlib_for_lto, modules) } fn run_thin_lto( cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, // FIXME(bjorn3): Limit LTO exports to these symbols _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { - back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules) + back::lto::run_thin(cgcx, dcx, each_linked_rlib_for_lto, modules, cached_modules) } fn print_pass_timings(&self) { @@ -400,7 +402,7 @@ impl WriteBackendMethods for GccCodegenBackend { fn optimize( _cgcx: &CodegenContext, - _dcx: DiagCtxtHandle<'_>, + _shared_emitter: &SharedEmitter, module: &mut ModuleCodegen, config: &ModuleConfig, ) { @@ -409,6 +411,7 @@ impl WriteBackendMethods for GccCodegenBackend { fn optimize_thin( cgcx: &CodegenContext, + _shared_emitter: &SharedEmitter, thin: ThinModule, ) -> ModuleCodegen { back::lto::optimize_thin_module(thin, cgcx) @@ -416,10 +419,11 @@ impl WriteBackendMethods for GccCodegenBackend { fn codegen( cgcx: &CodegenContext, + shared_emitter: &SharedEmitter, module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { - back::write::codegen(cgcx, module, config) + back::write::codegen(cgcx, shared_emitter, module, config) } fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { From 09f774da8eccc725f54b38715c88cf7d52315361 Mon Sep 17 00:00:00 2001 From: Jason Newcomb Date: Sun, 23 Nov 2025 01:56:15 -0500 Subject: [PATCH 0043/1843] Mark all arms as being from a desugaring when lowering `?` and for loops. --- compiler/rustc_ast_lowering/src/expr.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index 524f8b054cb4..0ab3191f23be 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -941,14 +941,14 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::ExprKind::Break(this.lower_loop_destination(None), Some(x_expr)); this.arena.alloc(this.expr(gen_future_span, expr_break)) }); - self.arm(ready_pat, break_x) + self.arm(ready_pat, break_x, span) }; // `::std::task::Poll::Pending => {}` let pending_arm = { let pending_pat = self.pat_lang_item_variant(span, hir::LangItem::PollPending, &[]); let empty_block = self.expr_block_empty(span); - self.arm(pending_pat, empty_block) + self.arm(pending_pat, empty_block, span) }; let inner_match_stmt = { @@ -1002,7 +1002,7 @@ impl<'hir> LoweringContext<'_, 'hir> { }); // mut __awaitee => loop { ... } - let awaitee_arm = self.arm(awaitee_pat, loop_expr); + let awaitee_arm = self.arm(awaitee_pat, loop_expr, span); // `match ::std::future::IntoFuture::into_future() { ... }` let into_future_expr = match await_kind { @@ -1788,7 +1788,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let break_expr = self.with_loop_scope(loop_hir_id, |this| this.expr_break_alloc(for_span)); let pat = self.pat_none(for_span); - self.arm(pat, break_expr) + self.arm(pat, break_expr, for_span) }; // Some() => , @@ -1797,7 +1797,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let body_block = self.with_loop_scope(loop_hir_id, |this| this.lower_block(body, false)); let body_expr = self.arena.alloc(self.expr_block(body_block)); - self.arm(some_pat, body_expr) + self.arm(some_pat, body_expr, for_span) }; // `mut iter` @@ -1856,7 +1856,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let loop_expr = self.arena.alloc(hir::Expr { hir_id: loop_hir_id, kind, span: for_span }); // `mut iter => { ... }` - let iter_arm = self.arm(iter_pat, loop_expr); + let iter_arm = self.arm(iter_pat, loop_expr, for_span); let match_expr = match loop_kind { ForLoopKind::For => { @@ -1901,7 +1901,7 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::LangItem::IntoAsyncIterIntoIter, arena_vec![self; head], ); - let iter_arm = self.arm(async_iter_pat, inner_match_expr); + let iter_arm = self.arm(async_iter_pat, inner_match_expr, for_span); self.arena.alloc(self.expr_match( for_span, iter, @@ -1968,7 +1968,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let val_expr = self.expr_ident(span, val_ident, val_pat_nid); self.lower_attrs(val_expr.hir_id, &attrs, span, Target::Expression); let continue_pat = self.pat_cf_continue(unstable_span, val_pat); - self.arm(continue_pat, val_expr) + self.arm(continue_pat, val_expr, try_span) }; // `ControlFlow::Break(residual) => @@ -2004,7 +2004,7 @@ impl<'hir> LoweringContext<'_, 'hir> { self.lower_attrs(ret_expr.hir_id, &attrs, span, Target::Expression); let break_pat = self.pat_cf_break(try_span, residual_local); - self.arm(break_pat, ret_expr) + self.arm(break_pat, ret_expr, try_span) }; hir::ExprKind::Match( @@ -2329,12 +2329,13 @@ impl<'hir> LoweringContext<'_, 'hir> { &mut self, pat: &'hir hir::Pat<'hir>, expr: &'hir hir::Expr<'hir>, + span: Span, ) -> hir::Arm<'hir> { hir::Arm { hir_id: self.next_id(), pat, guard: None, - span: self.lower_span(expr.span), + span: self.lower_span(span), body: expr, } } From 1deb487076519970fdba20742cc10c380962d2c3 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 30 Nov 2025 14:21:08 +0100 Subject: [PATCH 0044/1843] thread::scope: document how join interacts with TLS destructors --- library/std/src/thread/join_handle.rs | 2 ++ library/std/src/thread/scoped.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/library/std/src/thread/join_handle.rs b/library/std/src/thread/join_handle.rs index 8714ceeb4f46..bb3759cd3a07 100644 --- a/library/std/src/thread/join_handle.rs +++ b/library/std/src/thread/join_handle.rs @@ -102,6 +102,8 @@ impl JoinHandle { /// Waits for the associated thread to finish. /// /// This function will return immediately if the associated thread has already finished. + /// Otherwise, it fully waits for the thread to finish, including all destructors + /// for thread-local variables that might be running after the main function of the thread. /// /// In terms of [atomic memory orderings], the completion of the associated /// thread synchronizes with this function returning. In other words, all diff --git a/library/std/src/thread/scoped.rs b/library/std/src/thread/scoped.rs index 301f5e949cac..368fb819962b 100644 --- a/library/std/src/thread/scoped.rs +++ b/library/std/src/thread/scoped.rs @@ -80,6 +80,9 @@ impl ScopeData { /// /// All threads spawned within the scope that haven't been manually joined /// will be automatically joined before this function returns. +/// However, note that joining will only wait for the main function of these threads to finish; even +/// when this function returns, destructors of thread-local variables in these threads might still +/// be running. /// /// # Panics /// @@ -292,6 +295,8 @@ impl<'scope, T> ScopedJoinHandle<'scope, T> { /// Waits for the associated thread to finish. /// /// This function will return immediately if the associated thread has already finished. + /// Otherwise, it fully waits for the thread to finish, including all destructors + /// for thread-local variables that might be running after the main function of the thread. /// /// In terms of [atomic memory orderings], the completion of the associated /// thread synchronizes with this function returning. From a8ffa1cbb035ddeb6e5c6326ffca26f58e7bb2b6 Mon Sep 17 00:00:00 2001 From: Jieyou Xu Date: Sat, 6 Dec 2025 15:12:20 +0800 Subject: [PATCH 0045/1843] docs: update default branch name of `rust-lang/rust` --- book/src/development/infrastructure/sync.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/book/src/development/infrastructure/sync.md b/book/src/development/infrastructure/sync.md index 2bbdf47a8358..4506ff15d8ff 100644 --- a/book/src/development/infrastructure/sync.md +++ b/book/src/development/infrastructure/sync.md @@ -79,7 +79,7 @@ to be run inside the `rust` directory): ```bash git fetch upstream # assuming upstream is the rust-lang/rust remote git switch rustup - git merge upstream/master --no-ff + git merge upstream/main --no-ff ``` > Note: This is one of the few instances where a merge commit is allowed in > a PR. @@ -99,7 +99,7 @@ to be run inside the `rust` directory): All the following commands have to be run inside the `rust` directory. -1. Make sure you have checked out the latest `master` of `rust-lang/rust`. +1. Make sure you have checked out the latest `main` of `rust-lang/rust`. 2. Sync the `rust-lang/rust-clippy` master to the rust-copy of Clippy: ```bash git switch -c clippy-subtree-update From 7553f464308db4d5508e81dd4d4ad92252186a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Thu, 4 Dec 2025 10:41:13 +0000 Subject: [PATCH 0046/1843] inline constant typeck constraint --- .../src/polonius/typeck_constraints.rs | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs index e4e52962bf7f..7d406837f5ae 100644 --- a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs @@ -45,7 +45,6 @@ pub(super) fn convert_typeck_constraints<'tcx>( { localize_statement_constraint( tcx, - body, stmt, &outlives_constraint, point, @@ -74,7 +73,6 @@ pub(super) fn convert_typeck_constraints<'tcx>( /// needed CFG `from`-`to` intra-block nodes. fn localize_statement_constraint<'tcx>( tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, stmt: &Statement<'tcx>, outlives_constraint: &OutlivesConstraint<'tcx>, current_point: PointIndex, @@ -114,28 +112,22 @@ fn localize_statement_constraint<'tcx>( }, "there should be no common regions between the LHS and RHS of an assignment" ); - - let lhs_ty = body.local_decls[lhs.local].ty; - let successor_point = current_point; - compute_constraint_direction( - tcx, - outlives_constraint, - &lhs_ty, - current_point, - successor_point, - universal_regions, - ) } _ => { - // For the other cases, we localize an outlives constraint to where it arises. - LocalizedOutlivesConstraint { - source: outlives_constraint.sup, - from: current_point, - target: outlives_constraint.sub, - to: current_point, - } + // Assignments should be the only statement that can both generate constraints that + // apply on entry (specific to the RHS place) *and* others that only apply on exit (the + // subset of RHS regions that actually flow into the LHS): i.e., where midpoints would + // be used to ensure the former happen before the latter, within the same MIR Location. } } + + // We generally localize an outlives constraint to where it arises. + LocalizedOutlivesConstraint { + source: outlives_constraint.sup, + from: current_point, + target: outlives_constraint.sub, + to: current_point, + } } /// For a given outlives constraint arising from a MIR terminator, localize the constraint with the From 38f795dd028fd02b28278c43a828907e34f82cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Thu, 4 Dec 2025 11:18:06 +0000 Subject: [PATCH 0047/1843] update comments --- .../rustc_borrowck/src/polonius/typeck_constraints.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs index 7d406837f5ae..cfe9376fb502 100644 --- a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs @@ -142,14 +142,12 @@ fn localize_terminator_constraint<'tcx>( universal_regions: &UniversalRegions<'tcx>, ) -> LocalizedOutlivesConstraint { // FIXME: check if other terminators need the same handling as `Call`s, in particular - // Assert/Yield/Drop. A handful of tests are failing with Drop related issues, as well as some - // coroutine tests, and that may be why. + // Assert/Yield/Drop. match &terminator.kind { // FIXME: also handle diverging calls. TerminatorKind::Call { destination, target: Some(target), .. } => { - // Calls are similar to assignments, and thus follow the same pattern. If there is a - // target for the call we also relate what flows into the destination here to entry to - // that successor. + // If there is a target for the call we also relate what flows into the destination here + // to entry to that successor. let destination_ty = destination.ty(&body.local_decls, tcx); let successor_location = Location { block: *target, statement_index: 0 }; let successor_point = liveness.point_from_location(successor_location); From 79b30d64e76cdc44275822b481c0dd1832b81d0e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Tue, 9 Dec 2025 10:52:13 +0100 Subject: [PATCH 0048/1843] thread, grid, and block dim/idx can only return non-negative values --- .../stdarch/crates/core_arch/src/nvptx/mod.rs | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/nvptx/mod.rs b/library/stdarch/crates/core_arch/src/nvptx/mod.rs index 8d16dfb53d43..5471ef819801 100644 --- a/library/stdarch/crates/core_arch/src/nvptx/mod.rs +++ b/library/stdarch/crates/core_arch/src/nvptx/mod.rs @@ -23,29 +23,29 @@ unsafe extern "C" { #[link_name = "llvm.nvvm.barrier0"] fn syncthreads() -> (); #[link_name = "llvm.nvvm.read.ptx.sreg.ntid.x"] - fn block_dim_x() -> i32; + fn block_dim_x() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.ntid.y"] - fn block_dim_y() -> i32; + fn block_dim_y() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.ntid.z"] - fn block_dim_z() -> i32; + fn block_dim_z() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.ctaid.x"] - fn block_idx_x() -> i32; + fn block_idx_x() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.ctaid.y"] - fn block_idx_y() -> i32; + fn block_idx_y() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.ctaid.z"] - fn block_idx_z() -> i32; + fn block_idx_z() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.nctaid.x"] - fn grid_dim_x() -> i32; + fn grid_dim_x() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.nctaid.y"] - fn grid_dim_y() -> i32; + fn grid_dim_y() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.nctaid.z"] - fn grid_dim_z() -> i32; + fn grid_dim_z() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.tid.x"] - fn thread_idx_x() -> i32; + fn thread_idx_x() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.tid.y"] - fn thread_idx_y() -> i32; + fn thread_idx_y() -> u32; #[link_name = "llvm.nvvm.read.ptx.sreg.tid.z"] - fn thread_idx_z() -> i32; + fn thread_idx_z() -> u32; } /// Synchronizes all threads in the block. @@ -58,84 +58,84 @@ pub unsafe fn _syncthreads() -> () { /// x-th thread-block dimension. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _block_dim_x() -> i32 { +pub unsafe fn _block_dim_x() -> u32 { block_dim_x() } /// y-th thread-block dimension. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _block_dim_y() -> i32 { +pub unsafe fn _block_dim_y() -> u32 { block_dim_y() } /// z-th thread-block dimension. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _block_dim_z() -> i32 { +pub unsafe fn _block_dim_z() -> u32 { block_dim_z() } /// x-th thread-block index. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _block_idx_x() -> i32 { +pub unsafe fn _block_idx_x() -> u32 { block_idx_x() } /// y-th thread-block index. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _block_idx_y() -> i32 { +pub unsafe fn _block_idx_y() -> u32 { block_idx_y() } /// z-th thread-block index. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _block_idx_z() -> i32 { +pub unsafe fn _block_idx_z() -> u32 { block_idx_z() } /// x-th block-grid dimension. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _grid_dim_x() -> i32 { +pub unsafe fn _grid_dim_x() -> u32 { grid_dim_x() } /// y-th block-grid dimension. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _grid_dim_y() -> i32 { +pub unsafe fn _grid_dim_y() -> u32 { grid_dim_y() } /// z-th block-grid dimension. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _grid_dim_z() -> i32 { +pub unsafe fn _grid_dim_z() -> u32 { grid_dim_z() } /// x-th thread index. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _thread_idx_x() -> i32 { +pub unsafe fn _thread_idx_x() -> u32 { thread_idx_x() } /// y-th thread index. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _thread_idx_y() -> i32 { +pub unsafe fn _thread_idx_y() -> u32 { thread_idx_y() } /// z-th thread index. #[inline] #[unstable(feature = "stdarch_nvptx", issue = "111199")] -pub unsafe fn _thread_idx_z() -> i32 { +pub unsafe fn _thread_idx_z() -> u32 { thread_idx_z() } From 4215b7228d28cb84016e8f0e41c0a5e0935a222f Mon Sep 17 00:00:00 2001 From: Andrew Cherry Date: Thu, 11 Dec 2025 13:58:40 +0000 Subject: [PATCH 0049/1843] removal of aarch64 special-casing (#497) --- crates/std_float/src/lib.rs | 98 ++++++++++++++----------------------- 1 file changed, 37 insertions(+), 61 deletions(-) diff --git a/crates/std_float/src/lib.rs b/crates/std_float/src/lib.rs index c3c9b76e50b8..b269efc9b1d7 100644 --- a/crates/std_float/src/lib.rs +++ b/crates/std_float/src/lib.rs @@ -66,28 +66,43 @@ pub trait StdFloat: Sealed + Sized { /// Produces a vector where every element has the sine of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn sin(self) -> Self; + fn sin(self) -> Self { + unsafe { intrinsics::simd_fsin(self) } + } /// Produces a vector where every element has the cosine of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn cos(self) -> Self; + fn cos(self) -> Self { + unsafe { intrinsics::simd_fcos(self) } + } /// Produces a vector where every element has the exponential (base e) of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn exp(self) -> Self; + fn exp(self) -> Self { + unsafe { intrinsics::simd_fexp(self) } + } /// Produces a vector where every element has the exponential (base 2) of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn exp2(self) -> Self; + fn exp2(self) -> Self { + unsafe { intrinsics::simd_fexp2(self) } + } /// Produces a vector where every element has the natural logarithm of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn ln(self) -> Self; + fn ln(self) -> Self { + unsafe { intrinsics::simd_flog(self) } + } /// Produces a vector where every element has the logarithm with respect to an arbitrary /// in the equivalently-indexed elements in `self` and `base`. @@ -99,13 +114,19 @@ pub trait StdFloat: Sealed + Sized { /// Produces a vector where every element has the base-2 logarithm of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn log2(self) -> Self; + fn log2(self) -> Self { + unsafe { intrinsics::simd_flog2(self) } + } /// Produces a vector where every element has the base-10 logarithm of the value /// in the equivalently-indexed element in `self`. + #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - fn log10(self) -> Self; + fn log10(self) -> Self { + unsafe { intrinsics::simd_flog10(self) } + } /// Returns the smallest integer greater than or equal to each element. #[must_use = "method returns a new vector and does not mutate the original value"] @@ -143,61 +164,16 @@ pub trait StdFloat: Sealed + Sized { impl Sealed for Simd {} impl Sealed for Simd {} -macro_rules! impl_float { - { - $($fn:ident: $intrinsic:ident,)* - } => { - impl StdFloat for Simd - { - #[inline] - fn fract(self) -> Self { - self - self.trunc() - } - - $( - #[inline] - fn $fn(self) -> Self { - unsafe { intrinsics::$intrinsic(self) } - } - )* - } - - impl StdFloat for Simd - { - #[inline] - fn fract(self) -> Self { - self - self.trunc() - } - - $( - #[inline] - fn $fn(self) -> Self { - // https://github.com/llvm/llvm-project/issues/83729 - #[cfg(target_arch = "aarch64")] - { - let mut ln = Self::splat(0f64); - for i in 0..N { - ln[i] = self[i].$fn() - } - ln - } - - #[cfg(not(target_arch = "aarch64"))] - { - unsafe { intrinsics::$intrinsic(self) } - } - } - )* - } +impl StdFloat for Simd { + #[inline] + fn fract(self) -> Self { + self - self.trunc() } } -impl_float! { - sin: simd_fsin, - cos: simd_fcos, - exp: simd_fexp, - exp2: simd_fexp2, - ln: simd_flog, - log2: simd_flog2, - log10: simd_flog10, +impl StdFloat for Simd { + #[inline] + fn fract(self) -> Self { + self - self.trunc() + } } From 31ce232fda9790c5025d6f89a23b8fd67e8b9369 Mon Sep 17 00:00:00 2001 From: Paul Murphy Date: Tue, 21 Oct 2025 15:02:52 -0500 Subject: [PATCH 0050/1843] Stabilize ppc inline assembly --- compiler/rustc_ast_lowering/src/asm.rs | 2 + .../asm-experimental-arch.md | 41 +------------------ 2 files changed, 4 insertions(+), 39 deletions(-) diff --git a/compiler/rustc_ast_lowering/src/asm.rs b/compiler/rustc_ast_lowering/src/asm.rs index d44faad017ee..afcbae9c51b4 100644 --- a/compiler/rustc_ast_lowering/src/asm.rs +++ b/compiler/rustc_ast_lowering/src/asm.rs @@ -51,6 +51,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { | asm::InlineAsmArch::LoongArch32 | asm::InlineAsmArch::LoongArch64 | asm::InlineAsmArch::S390x + | asm::InlineAsmArch::PowerPC + | asm::InlineAsmArch::PowerPC64 ); if !is_stable && !self.tcx.features().asm_experimental_arch() { feature_err( diff --git a/src/doc/unstable-book/src/language-features/asm-experimental-arch.md b/src/doc/unstable-book/src/language-features/asm-experimental-arch.md index 77d43315a6d4..23ac46b72ea7 100644 --- a/src/doc/unstable-book/src/language-features/asm-experimental-arch.md +++ b/src/doc/unstable-book/src/language-features/asm-experimental-arch.md @@ -8,7 +8,6 @@ The tracking issue for this feature is: [#93335] This feature tracks `asm!` and `global_asm!` support for the following architectures: - NVPTX -- PowerPC - Hexagon - MIPS32r2 and MIPS64r2 - wasm32 @@ -31,16 +30,6 @@ This feature tracks `asm!` and `global_asm!` support for the following architect | NVPTX | `reg64` | None\* | `l` | | Hexagon | `reg` | `r[0-28]` | `r` | | Hexagon | `preg` | `p[0-3]` | Only clobbers | -| PowerPC | `reg` | `r0`, `r[3-12]`, `r[14-29]`\* | `r` | -| PowerPC | `reg_nonzero` | `r[3-12]`, `r[14-29]`\* | `b` | -| PowerPC | `freg` | `f[0-31]` | `f` | -| PowerPC | `vreg` | `v[0-31]` | `v` | -| PowerPC | `vsreg | `vs[0-63]` | `wa` | -| PowerPC | `cr` | `cr[0-7]`, `cr` | Only clobbers | -| PowerPC | `ctr` | `ctr` | Only clobbers | -| PowerPC | `lr` | `lr` | Only clobbers | -| PowerPC | `xer` | `xer` | Only clobbers | -| PowerPC | `spe_acc` | `spe_acc` | Only clobbers | | wasm32 | `local` | None\* | `r` | | BPF | `reg` | `r[0-10]` | `r` | | BPF | `wreg` | `w[0-10]` | `w` | @@ -62,10 +51,6 @@ This feature tracks `asm!` and `global_asm!` support for the following architect > - NVPTX doesn't have a fixed register set, so named registers are not supported. > > - WebAssembly doesn't have registers, so named registers are not supported. -> -> - r29 is reserved only on 32 bit PowerPC targets. -> -> - spe_acc is only available on PowerPC SPE targets. # Register class supported types @@ -80,17 +65,6 @@ This feature tracks `asm!` and `global_asm!` support for the following architect | NVPTX | `reg64` | None | `i8`, `i16`, `i32`, `f32`, `i64`, `f64` | | Hexagon | `reg` | None | `i8`, `i16`, `i32`, `f32` | | Hexagon | `preg` | N/A | Only clobbers | -| PowerPC | `reg` | None | `i8`, `i16`, `i32`, `i64` (powerpc64 only) | -| PowerPC | `reg_nonzero` | None | `i8`, `i16`, `i32`, `i64` (powerpc64 only) | -| PowerPC | `freg` | None | `f32`, `f64` | -| PowerPC | `vreg` | `altivec` | `i8x16`, `i16x8`, `i32x4`, `f32x4` | -| PowerPC | `vreg` | `vsx` | `f32`, `f64`, `i64x2`, `f64x2` | -| PowerPC | `vsreg` | `vsx` | The union of vsx and altivec vreg types | -| PowerPC | `cr` | N/A | Only clobbers | -| PowerPC | `ctr` | N/A | Only clobbers | -| PowerPC | `lr` | N/A | Only clobbers | -| PowerPC | `xer` | N/A | Only clobbers | -| PowerPC | `spe_acc` | N/A | Only clobbers | | wasm32 | `local` | None | `i8` `i16` `i32` `i64` `f32` `f64` | | BPF | `reg` | None | `i8` `i16` `i32` `i64` | | BPF | `wreg` | `alu32` | `i8` `i16` `i32` | @@ -111,10 +85,6 @@ This feature tracks `asm!` and `global_asm!` support for the following architect | Hexagon | `r29` | `sp` | | Hexagon | `r30` | `fr` | | Hexagon | `r31` | `lr` | -| PowerPC | `r1` | `sp` | -| PowerPC | `r31` | `fp` | -| PowerPC | `r[0-31]` | `[0-31]` | -| PowerPC | `f[0-31]` | `fr[0-31]`| | BPF | `r[0-10]` | `w[0-10]` | | AVR | `XH` | `r27` | | AVR | `XL` | `r26` | @@ -153,16 +123,14 @@ This feature tracks `asm!` and `global_asm!` support for the following architect | Architecture | Unsupported register | Reason | | ------------ | --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | All | `sp`, `r14`/`o6` (SPARC) | The stack pointer must be restored to its original value at the end of an asm code block. | -| All | `fr` (Hexagon), `fp` (PowerPC), `$fp` (MIPS), `Y` (AVR), `r4` (MSP430), `a6` (M68k), `r30`/`i6` (SPARC) | The frame pointer cannot be used as an input or output. | -| All | `r19` (Hexagon), `r29` (PowerPC 32 bit only), `r30` (PowerPC) | These are used internally by LLVM as "base pointer" for functions with complex stack frames. | +| All | `fr` (Hexagon) `$fp` (MIPS), `Y` (AVR), `r4` (MSP430), `a6` (M68k), `r30`/`i6` (SPARC) | The frame pointer cannot be used as an input or output. | +| All | `r19` (Hexagon) | These are used internally by LLVM as "base pointer" for functions with complex stack frames. | | MIPS | `$0` or `$zero` | This is a constant zero register which can't be modified. | | MIPS | `$1` or `$at` | Reserved for assembler. | | MIPS | `$26`/`$k0`, `$27`/`$k1` | OS-reserved registers. | | MIPS | `$28`/`$gp` | Global pointer cannot be used as inputs or outputs. | | MIPS | `$ra` | Return address cannot be used as inputs or outputs. | | Hexagon | `lr` | This is the link register which cannot be used as an input or output. | -| PowerPC | `r2`, `r13` | These are system reserved registers. | -| PowerPC | `vrsave` | The vrsave register cannot be used as an input or output. | | AVR | `r0`, `r1`, `r1r0` | Due to an issue in LLVM, the `r0` and `r1` registers cannot be used as inputs or outputs. If modified, they must be restored to their original values before the end of the block. | |MSP430 | `r0`, `r2`, `r3` | These are the program counter, status register, and constant generator respectively. Neither the status register nor constant generator can be written to. | | M68k | `a4`, `a5` | Used internally by LLVM for the base pointer and global base pointer. | @@ -189,11 +157,6 @@ This feature tracks `asm!` and `global_asm!` support for the following architect | NVPTX | `reg32` | None | `r0` | None | | NVPTX | `reg64` | None | `rd0` | None | | Hexagon | `reg` | None | `r0` | None | -| PowerPC | `reg` | None | `0` | None | -| PowerPC | `reg_nonzero` | None | `3` | None | -| PowerPC | `freg` | None | `0` | None | -| PowerPC | `vreg` | None | `0` | None | -| PowerPC | `vsreg` | None | `0` | None | | SPARC | `reg` | None | `%o0` | None | | CSKY | `reg` | None | `r0` | None | | CSKY | `freg` | None | `f0` | None | From 9e638455220c0c67393cfb1ed14cc3f0c23bf8f0 Mon Sep 17 00:00:00 2001 From: Shoyu Vanilla Date: Fri, 12 Dec 2025 18:26:16 +0900 Subject: [PATCH 0051/1843] `-Znext-solver` Remove the forced ambiguity hack from search graph --- .../rustc_type_ir/src/search_graph/mod.rs | 31 ++----------- .../global-where-bound-normalization.rs | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 tests/ui/traits/next-solver/global-where-bound-normalization.rs diff --git a/compiler/rustc_type_ir/src/search_graph/mod.rs b/compiler/rustc_type_ir/src/search_graph/mod.rs index 8e6376b22ce6..7c58cd7303eb 100644 --- a/compiler/rustc_type_ir/src/search_graph/mod.rs +++ b/compiler/rustc_type_ir/src/search_graph/mod.rs @@ -916,10 +916,9 @@ impl, X: Cx> SearchGraph { /// heads from the stack. This may not necessarily mean that we've actually /// reached a fixpoint for that cycle head, which impacts the way we rebase /// provisional cache entries. -#[derive_where(Debug; X: Cx)] -enum RebaseReason { +#[derive(Debug)] +enum RebaseReason { NoCycleUsages, - Ambiguity(X::AmbiguityInfo), Overflow, /// We've actually reached a fixpoint. /// @@ -956,7 +955,7 @@ impl, X: Cx> SearchGraph { &mut self, cx: X, stack_entry: &StackEntry, - rebase_reason: RebaseReason, + rebase_reason: RebaseReason, ) { let popped_head_index = self.stack.next_index(); #[allow(rustc::potential_query_instability)] @@ -1035,9 +1034,6 @@ impl, X: Cx> SearchGraph { // is not actually equal to the final provisional result. We // need to discard the provisional cache entry in this case. RebaseReason::NoCycleUsages => return false, - RebaseReason::Ambiguity(info) => { - *result = D::propagate_ambiguity(cx, input, info); - } RebaseReason::Overflow => *result = D::fixpoint_overflow_result(cx, input), RebaseReason::ReachedFixpoint(None) => {} RebaseReason::ReachedFixpoint(Some(path_kind)) => { @@ -1352,27 +1348,6 @@ impl, X: Cx> SearchGraph { return EvaluationResult::finalize(stack_entry, encountered_overflow, result); } - // If computing this goal results in ambiguity with no constraints, - // we do not rerun it. It's incredibly difficult to get a different - // response in the next iteration in this case. These changes would - // likely either be caused by incompleteness or can change the maybe - // cause from ambiguity to overflow. Returning ambiguity always - // preserves soundness and completeness even if the goal is be known - // to succeed or fail. - // - // This prevents exponential blowup affecting multiple major crates. - // As we only get to this branch if we haven't yet reached a fixpoint, - // we also taint all provisional cache entries which depend on the - // current goal. - if let Some(info) = D::is_ambiguous_result(result) { - self.rebase_provisional_cache_entries( - cx, - &stack_entry, - RebaseReason::Ambiguity(info), - ); - return EvaluationResult::finalize(stack_entry, encountered_overflow, result); - }; - // If we've reached the fixpoint step limit, we bail with overflow and taint all // provisional cache entries which depend on the current goal. i += 1; diff --git a/tests/ui/traits/next-solver/global-where-bound-normalization.rs b/tests/ui/traits/next-solver/global-where-bound-normalization.rs new file mode 100644 index 000000000000..e57fbf378a0d --- /dev/null +++ b/tests/ui/traits/next-solver/global-where-bound-normalization.rs @@ -0,0 +1,45 @@ +//@ check-pass +//@ compile-flags: -Znext-solver + +// Regression test for https://github.com/rust-lang/trait-system-refactor-initiative/issues/257. + +#![feature(rustc_attrs)] +#![expect(internal_features)] +#![rustc_no_implicit_bounds] + +pub trait Bound {} +impl Bound for u8 {} + +pub trait Proj { + type Assoc; +} +impl Proj for U { + type Assoc = U; +} +impl Proj for MyField { + type Assoc = u8; +} + +// While wf-checking the global bounds of `fn foo`, elaborating this outlives predicate triggered a +// cycle in the search graph along a particular probe path, which was not an actual solution. +// That cycle then resulted in a forced false-positive ambiguity due to a performance hack in the +// search graph and then ended up floundering the root goal evaluation. +pub trait Field: Proj {} + +struct MyField; +impl Field for MyField {} + +trait IdReqField { + type This; +} +impl IdReqField for F { + type This = F; +} + +fn foo() +where + ::This: Field, +{ +} + +fn main() {} From d0d8258886dfb8c09d337b094ca1e6f26f85cd8b Mon Sep 17 00:00:00 2001 From: Urgau Date: Sun, 12 Oct 2025 17:19:17 +0200 Subject: [PATCH 0052/1843] Stabilize `-Zremap-path-scope` as `--remap-path-scope` In the process also document that new `--remap-path-scope` scopes may be added in the future, and that the `all` scope always represent all the scopes. Co-authored-by: David Wood --- compiler/rustc_session/src/config.rs | 48 ++++++++++++++++--- compiler/rustc_session/src/options.rs | 28 +---------- src/doc/rustc/src/command-line-arguments.md | 8 ++++ src/doc/rustc/src/remap-source-paths.md | 27 ++++++++++- .../src/compiler-flags/remap-path-scope.md | 23 --------- tests/coverage/remap-path-prefix.rs | 6 +-- ...emap-path-prefix.with_macro_scope.coverage | 6 +-- .../remap-path-prefix-consts/rmake.rs | 4 +- .../run-make/remap-path-prefix-dwarf/rmake.rs | 8 ++-- tests/run-make/remap-path-prefix/rmake.rs | 6 +-- tests/run-make/rustc-help/help-v.diff | 5 +- tests/run-make/rustc-help/help-v.stdout | 3 ++ tests/run-make/split-debuginfo/rmake.rs | 13 +++-- tests/ui/errors/auxiliary/file-debuginfo.rs | 2 +- tests/ui/errors/auxiliary/file-diag.rs | 2 +- tests/ui/errors/auxiliary/file-macro.rs | 2 +- tests/ui/errors/auxiliary/trait-debuginfo.rs | 2 +- tests/ui/errors/auxiliary/trait-diag.rs | 2 +- tests/ui/errors/auxiliary/trait-macro.rs | 2 +- .../errors/remap-path-prefix-diagnostics.rs | 10 ++-- tests/ui/errors/remap-path-prefix-macro.rs | 10 ++-- tests/ui/errors/remap-path-prefix.rs | 4 +- 22 files changed, 124 insertions(+), 97 deletions(-) delete mode 100644 src/doc/unstable-book/src/compiler-flags/remap-path-scope.md diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index be4b36e3b926..4d2ca8987df4 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -24,7 +24,9 @@ use rustc_hashes::Hash64; use rustc_macros::{BlobDecodable, Decodable, Encodable, HashStable_Generic}; use rustc_span::edition::{DEFAULT_EDITION, EDITION_NAME_LIST, Edition, LATEST_STABLE_EDITION}; use rustc_span::source_map::FilePathMapping; -use rustc_span::{FileName, RealFileName, SourceFileHashAlgorithm, Symbol, sym}; +use rustc_span::{ + FileName, RealFileName, RemapPathScopeComponents, SourceFileHashAlgorithm, Symbol, sym, +}; use rustc_target::spec::{ FramePointer, LinkSelfContainedComponents, LinkerFeatures, PanicStrategy, SplitDebuginfo, Target, TargetTuple, @@ -1315,6 +1317,29 @@ impl OutputFilenames { } } +pub(crate) fn parse_remap_path_scope( + early_dcx: &EarlyDiagCtxt, + matches: &getopts::Matches, +) -> RemapPathScopeComponents { + if let Some(v) = matches.opt_str("remap-path-scope") { + let mut slot = RemapPathScopeComponents::empty(); + for s in v.split(',') { + slot |= match s { + "macro" => RemapPathScopeComponents::MACRO, + "diagnostics" => RemapPathScopeComponents::DIAGNOSTICS, + "debuginfo" => RemapPathScopeComponents::DEBUGINFO, + "coverage" => RemapPathScopeComponents::COVERAGE, + "object" => RemapPathScopeComponents::OBJECT, + "all" => RemapPathScopeComponents::all(), + _ => early_dcx.early_fatal("argument for `--remap-path-scope` must be a comma separated list of scopes: `macro`, `diagnostics`, `debuginfo`, `coverage`, `object`, `all`"), + } + } + slot + } else { + RemapPathScopeComponents::all() + } +} + #[derive(Clone, Debug)] pub struct Sysroot { pub explicit: Option, @@ -1351,9 +1376,9 @@ pub fn host_tuple() -> &'static str { fn file_path_mapping( remap_path_prefix: Vec<(PathBuf, PathBuf)>, - unstable_opts: &UnstableOptions, + remap_path_scope: RemapPathScopeComponents, ) -> FilePathMapping { - FilePathMapping::new(remap_path_prefix.clone(), unstable_opts.remap_path_scope) + FilePathMapping::new(remap_path_prefix.clone(), remap_path_scope) } impl Default for Options { @@ -1365,7 +1390,7 @@ impl Default for Options { // to create a default working directory. let working_dir = { let working_dir = std::env::current_dir().unwrap(); - let file_mapping = file_path_mapping(Vec::new(), &unstable_opts); + let file_mapping = file_path_mapping(Vec::new(), RemapPathScopeComponents::empty()); file_mapping.to_real_filename(&RealFileName::empty(), &working_dir) }; @@ -1401,6 +1426,7 @@ impl Default for Options { cli_forced_codegen_units: None, cli_forced_local_thinlto_off: false, remap_path_prefix: Vec::new(), + remap_path_scope: RemapPathScopeComponents::all(), real_rust_source_base_dir: None, real_rustc_dev_source_base_dir: None, edition: DEFAULT_EDITION, @@ -1427,7 +1453,7 @@ impl Options { } pub fn file_path_mapping(&self) -> FilePathMapping { - file_path_mapping(self.remap_path_prefix.clone(), &self.unstable_opts) + file_path_mapping(self.remap_path_prefix.clone(), self.remap_path_scope) } /// Returns `true` if there will be an output file generated. @@ -1864,6 +1890,14 @@ pub fn rustc_optgroups() -> Vec { "Remap source names in all output (compiler messages and output files)", "=", ), + opt( + Stable, + Opt, + "", + "remap-path-scope", + "Defines which scopes of paths should be remapped by `--remap-path-prefix`", + "", + ), opt(Unstable, Multi, "", "env-set", "Inject an environment variable", "="), ]; options.extend(verbose_only.into_iter().map(|mut opt| { @@ -2704,6 +2738,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M let externs = parse_externs(early_dcx, matches, &unstable_opts); let remap_path_prefix = parse_remap_path_prefix(early_dcx, matches, &unstable_opts); + let remap_path_scope = parse_remap_path_scope(early_dcx, matches); let pretty = parse_pretty(early_dcx, &unstable_opts); @@ -2770,7 +2805,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M early_dcx.early_fatal(format!("Current directory is invalid: {e}")); }); - let file_mapping = file_path_mapping(remap_path_prefix.clone(), &unstable_opts); + let file_mapping = file_path_mapping(remap_path_prefix.clone(), remap_path_scope); file_mapping.to_real_filename(&RealFileName::empty(), &working_dir) }; @@ -2808,6 +2843,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M cli_forced_codegen_units: codegen_units, cli_forced_local_thinlto_off: disable_local_thinlto, remap_path_prefix, + remap_path_scope, real_rust_source_base_dir, real_rustc_dev_source_base_dir, edition, diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index aea0b73ee927..42d62a7f8437 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -454,6 +454,8 @@ top_level_options!( /// Remap source path prefixes in all output (messages, object files, debug, etc.). remap_path_prefix: Vec<(PathBuf, PathBuf)> [TRACKED_NO_CRATE_HASH], + /// Defines which scopes of paths should be remapped by `--remap-path-prefix`. + remap_path_scope: RemapPathScopeComponents [TRACKED_NO_CRATE_HASH], /// Base directory containing the `library/` directory for the Rust standard library. /// Right now it's always `$sysroot/lib/rustlib/src/rust` @@ -872,7 +874,6 @@ mod desc { pub(crate) const parse_branch_protection: &str = "a `,` separated combination of `bti`, `gcs`, `pac-ret`, (optionally with `pc`, `b-key`, `leaf` if `pac-ret` is set)"; pub(crate) const parse_proc_macro_execution_strategy: &str = "one of supported execution strategies (`same-thread`, or `cross-thread`)"; - pub(crate) const parse_remap_path_scope: &str = "comma separated list of scopes: `macro`, `diagnostics`, `debuginfo`, `coverage`, `object`, `all`"; pub(crate) const parse_inlining_threshold: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a non-negative number"; pub(crate) const parse_llvm_module_flag: &str = ":::. Type must currently be `u32`. Behavior should be one of (`error`, `warning`, `require`, `override`, `append`, `appendunique`, `max`, `min`)"; @@ -1711,29 +1712,6 @@ pub mod parse { true } - pub(crate) fn parse_remap_path_scope( - slot: &mut RemapPathScopeComponents, - v: Option<&str>, - ) -> bool { - if let Some(v) = v { - *slot = RemapPathScopeComponents::empty(); - for s in v.split(',') { - *slot |= match s { - "macro" => RemapPathScopeComponents::MACRO, - "diagnostics" => RemapPathScopeComponents::DIAGNOSTICS, - "debuginfo" => RemapPathScopeComponents::DEBUGINFO, - "coverage" => RemapPathScopeComponents::COVERAGE, - "object" => RemapPathScopeComponents::OBJECT, - "all" => RemapPathScopeComponents::all(), - _ => return false, - } - } - true - } else { - false - } - } - pub(crate) fn parse_relocation_model(slot: &mut Option, v: Option<&str>) -> bool { match v.and_then(|s| RelocModel::from_str(s).ok()) { Some(relocation_model) => *slot = Some(relocation_model), @@ -2584,8 +2562,6 @@ options! { "whether ELF relocations can be relaxed"), remap_cwd_prefix: Option = (None, parse_opt_pathbuf, [TRACKED], "remap paths under the current working directory to this path prefix"), - remap_path_scope: RemapPathScopeComponents = (RemapPathScopeComponents::all(), parse_remap_path_scope, [TRACKED], - "remap path scope (default: all)"), remark_dir: Option = (None, parse_opt_pathbuf, [UNTRACKED], "directory into which to write optimization remarks (if not specified, they will be \ written to standard error output)"), diff --git a/src/doc/rustc/src/command-line-arguments.md b/src/doc/rustc/src/command-line-arguments.md index 0b15fbc24dfc..1309ecd73679 100644 --- a/src/doc/rustc/src/command-line-arguments.md +++ b/src/doc/rustc/src/command-line-arguments.md @@ -428,6 +428,14 @@ specified multiple times. Refer to the [Remap source paths](remap-source-paths.md) section of this book for further details and explanation. + +## `--remap-path-scope`: remap source paths in output + +Defines which scopes of paths should be remapped by `--remap-path-prefix`. + +Refer to the [Remap source paths](remap-source-paths.md) section of this book for +further details and explanation. + ## `--json`: configure json messages printed by the compiler diff --git a/src/doc/rustc/src/remap-source-paths.md b/src/doc/rustc/src/remap-source-paths.md index 03f5d98091cc..d27359fb550f 100644 --- a/src/doc/rustc/src/remap-source-paths.md +++ b/src/doc/rustc/src/remap-source-paths.md @@ -6,7 +6,7 @@ output, including compiler diagnostics, debugging information, macro expansions, This is useful for normalizing build products, for example by removing the current directory out of the paths emitted into object files. -The remapping is done via the `--remap-path-prefix` option. +The remapping is done via the `--remap-path-prefix` flag and can be customized via the `--remap-path-scope` flag. ## `--remap-path-prefix` @@ -25,6 +25,31 @@ rustc --remap-path-prefix "/home/user/project=/redacted" This example replaces all occurrences of `/home/user/project` in emitted paths with `/redacted`. +## `--remap-path-scope` + +Defines which scopes of paths should be remapped by `--remap-path-prefix`. + +This flag accepts a comma-separated list of values and may be specified multiple times, in which case the scopes are aggregated together. + +The valid scopes are: + +- `macro` - apply remappings to the expansion of `std::file!()` macro. This is where paths in embedded panic messages come from +- `diagnostics` - apply remappings to printed compiler diagnostics +- `debuginfo` - apply remappings to debug information +- `coverage` - apply remappings to coverage information +- `object` - apply remappings to all paths in compiled executables or libraries, but not elsewhere. Currently an alias for `macro,coverage,debuginfo`. +- `all` (default) - an alias for all of the above, also equivalent to supplying only `--remap-path-prefix` without `--remap-path-scope`. + +The scopes accepted by `--remap-path-scope` are not exhaustive - new scopes may be added in future releases for eventual stabilisation. +This implies that the `all` scope can correspond to different scopes between releases. + +### Example + +```sh +# With `object` scope only the build outputs will be remapped, the diagnostics won't be remapped. +rustc --remap-path-prefix=$(PWD)=/remapped --remap-path-scope=object main.rs +``` + ## Caveats and Limitations ### Linkers generated paths diff --git a/src/doc/unstable-book/src/compiler-flags/remap-path-scope.md b/src/doc/unstable-book/src/compiler-flags/remap-path-scope.md deleted file mode 100644 index fb1c7d7a6878..000000000000 --- a/src/doc/unstable-book/src/compiler-flags/remap-path-scope.md +++ /dev/null @@ -1,23 +0,0 @@ -# `remap-path-scope` - -The tracking issue for this feature is: [#111540](https://github.com/rust-lang/rust/issues/111540). - ------------------------- - -When the `--remap-path-prefix` option is passed to rustc, source path prefixes in all output will be affected by default. -The `--remap-path-scope` argument can be used in conjunction with `--remap-path-prefix` to determine paths in which output context should be affected. -This flag accepts a comma-separated list of values and may be specified multiple times, in which case the scopes are aggregated together. The valid scopes are: - -- `macro` - apply remappings to the expansion of `std::file!()` macro. This is where paths in embedded panic messages come from -- `diagnostics` - apply remappings to printed compiler diagnostics -- `debuginfo` - apply remappings to debug information -- `coverage` - apply remappings to coverage information -- `object` - apply remappings to all paths in compiled executables or libraries, but not elsewhere. Currently an alias for `macro,debuginfo`. -- `all` - an alias for all of the above, also equivalent to supplying only `--remap-path-prefix` without `--remap-path-scope`. - -## Example -```sh -# This would produce an absolute path to main.rs in build outputs of -# "./main.rs". -rustc --remap-path-prefix=$(PWD)=/remapped -Zremap-path-scope=object main.rs -``` diff --git a/tests/coverage/remap-path-prefix.rs b/tests/coverage/remap-path-prefix.rs index 29c5826989c4..031b87b8f771 100644 --- a/tests/coverage/remap-path-prefix.rs +++ b/tests/coverage/remap-path-prefix.rs @@ -10,8 +10,8 @@ //@ revisions: with_remap with_coverage_scope with_object_scope with_macro_scope //@ compile-flags: --remap-path-prefix={{src-base}}=remapped // -//@[with_coverage_scope] compile-flags: -Zremap-path-scope=coverage -//@[with_object_scope] compile-flags: -Zremap-path-scope=object -//@[with_macro_scope] compile-flags: -Zremap-path-scope=macro +//@[with_coverage_scope] compile-flags: --remap-path-scope=coverage +//@[with_object_scope] compile-flags: --remap-path-scope=object +//@[with_macro_scope] compile-flags: --remap-path-scope=macro fn main() {} diff --git a/tests/coverage/remap-path-prefix.with_macro_scope.coverage b/tests/coverage/remap-path-prefix.with_macro_scope.coverage index 63979d8fe15a..9e8317672f90 100644 --- a/tests/coverage/remap-path-prefix.with_macro_scope.coverage +++ b/tests/coverage/remap-path-prefix.with_macro_scope.coverage @@ -10,9 +10,9 @@ LL| |//@ revisions: with_remap with_coverage_scope with_object_scope with_macro_scope LL| |//@ compile-flags: --remap-path-prefix={{src-base}}=remapped LL| |// - LL| |//@[with_coverage_scope] compile-flags: -Zremap-path-scope=coverage - LL| |//@[with_object_scope] compile-flags: -Zremap-path-scope=object - LL| |//@[with_macro_scope] compile-flags: -Zremap-path-scope=macro + LL| |//@[with_coverage_scope] compile-flags: --remap-path-scope=coverage + LL| |//@[with_object_scope] compile-flags: --remap-path-scope=object + LL| |//@[with_macro_scope] compile-flags: --remap-path-scope=macro LL| | LL| 1|fn main() {} diff --git a/tests/run-make/remap-path-prefix-consts/rmake.rs b/tests/run-make/remap-path-prefix-consts/rmake.rs index d07a5e00768a..07b5e2f97414 100644 --- a/tests/run-make/remap-path-prefix-consts/rmake.rs +++ b/tests/run-make/remap-path-prefix-consts/rmake.rs @@ -97,7 +97,7 @@ fn main() { location_caller .crate_type("lib") .remap_path_prefix(cwd(), "/remapped") - .arg("-Zremap-path-scope=object") + .arg("--remap-path-scope=object") .input(cwd().join("location-caller.rs")); location_caller.run(); @@ -105,7 +105,7 @@ fn main() { runner .crate_type("bin") .remap_path_prefix(cwd(), "/remapped") - .arg("-Zremap-path-scope=diagnostics") + .arg("--remap-path-scope=diagnostics") .input(cwd().join("runner.rs")) .output(&runner_bin); runner.run(); diff --git a/tests/run-make/remap-path-prefix-dwarf/rmake.rs b/tests/run-make/remap-path-prefix-dwarf/rmake.rs index 3b88fca0bb7f..ab6c1fb70d68 100644 --- a/tests/run-make/remap-path-prefix-dwarf/rmake.rs +++ b/tests/run-make/remap-path-prefix-dwarf/rmake.rs @@ -105,7 +105,7 @@ fn check_dwarf_deps(scope: &str, dwarf_test: DwarfDump) { let mut rustc_sm = rustc(); rustc_sm.input(cwd().join("src/some_value.rs")); rustc_sm.arg("-Cdebuginfo=2"); - rustc_sm.arg(format!("-Zremap-path-scope={}", scope)); + rustc_sm.arg(format!("--remap-path-scope={}", scope)); rustc_sm.arg("--remap-path-prefix"); rustc_sm.arg(format!("{}=/REMAPPED", cwd().display())); rustc_sm.arg("-Csplit-debuginfo=off"); @@ -117,7 +117,7 @@ fn check_dwarf_deps(scope: &str, dwarf_test: DwarfDump) { rustc_pv.input(cwd().join("src/print_value.rs")); rustc_pv.output(&print_value_rlib); rustc_pv.arg("-Cdebuginfo=2"); - rustc_pv.arg(format!("-Zremap-path-scope={}", scope)); + rustc_pv.arg(format!("--remap-path-scope={}", scope)); rustc_pv.arg("--remap-path-prefix"); rustc_pv.arg(format!("{}=/REMAPPED", cwd().display())); rustc_pv.arg("-Csplit-debuginfo=off"); @@ -158,8 +158,8 @@ fn check_dwarf(test: DwarfTest) { rustc.arg("-Cdebuginfo=2"); if let Some(scope) = test.scope { match scope { - ScopeType::Object => rustc.arg("-Zremap-path-scope=object"), - ScopeType::Diagnostics => rustc.arg("-Zremap-path-scope=diagnostics"), + ScopeType::Object => rustc.arg("--remap-path-scope=object"), + ScopeType::Diagnostics => rustc.arg("--remap-path-scope=diagnostics"), }; if is_darwin() { rustc.arg("-Csplit-debuginfo=off"); diff --git a/tests/run-make/remap-path-prefix/rmake.rs b/tests/run-make/remap-path-prefix/rmake.rs index b75ca9e796ac..22ffd4f1f0d1 100644 --- a/tests/run-make/remap-path-prefix/rmake.rs +++ b/tests/run-make/remap-path-prefix/rmake.rs @@ -38,9 +38,9 @@ fn main() { rmeta_contains("/the/aux/lib.rs"); rmeta_not_contains("auxiliary"); - out_object.arg("-Zremap-path-scope=object"); - out_macro.arg("-Zremap-path-scope=macro"); - out_diagobj.arg("-Zremap-path-scope=diagnostics,object"); + out_object.arg("--remap-path-scope=object"); + out_macro.arg("--remap-path-scope=macro"); + out_diagobj.arg("--remap-path-scope=diagnostics,object"); if is_darwin() { out_object.arg("-Csplit-debuginfo=off"); out_macro.arg("-Csplit-debuginfo=off"); diff --git a/tests/run-make/rustc-help/help-v.diff b/tests/run-make/rustc-help/help-v.diff index 60a9dfbe201d..94ed6a0ed027 100644 --- a/tests/run-make/rustc-help/help-v.diff +++ b/tests/run-make/rustc-help/help-v.diff @@ -1,4 +1,4 @@ -@@ -65,10 +65,28 @@ +@@ -65,10 +65,31 @@ Set a codegen option -V, --version Print version info and exit -v, --verbose Use verbose output @@ -20,6 +20,9 @@ + --remap-path-prefix = + Remap source names in all output (compiler messages + and output files) ++ --remap-path-scope ++ Defines which scopes of paths should be remapped by ++ `--remap-path-prefix` + @path Read newline separated options from `path` Additional help: diff --git a/tests/run-make/rustc-help/help-v.stdout b/tests/run-make/rustc-help/help-v.stdout index c41cb5e3bde8..fb08d114ad1a 100644 --- a/tests/run-make/rustc-help/help-v.stdout +++ b/tests/run-make/rustc-help/help-v.stdout @@ -83,6 +83,9 @@ Options: --remap-path-prefix = Remap source names in all output (compiler messages and output files) + --remap-path-scope + Defines which scopes of paths should be remapped by + `--remap-path-prefix` @path Read newline separated options from `path` Additional help: diff --git a/tests/run-make/split-debuginfo/rmake.rs b/tests/run-make/split-debuginfo/rmake.rs index e53b71010781..0d311607a11a 100644 --- a/tests/run-make/split-debuginfo/rmake.rs +++ b/tests/run-make/split-debuginfo/rmake.rs @@ -171,8 +171,7 @@ enum RemapPathPrefix { Unspecified, } -/// `-Zremap-path-scope`. See -/// . +/// `--remap-path-scope` #[derive(Debug, Clone)] enum RemapPathScope { /// Comma-separated list of remap scopes: `macro`, `diagnostics`, `debuginfo`, `object`, `all`. @@ -921,7 +920,7 @@ mod shared_linux_other_tests { .debuginfo(level.cli_value()) .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value())) .remap_path_prefix(cwd(), remapped_prefix) - .arg(format!("-Zremap-path-scope={scope}")) + .arg(format!("--remap-path-scope={scope}")) .run(); let found_files = cwd_filenames(); FileAssertions { expected_files: BTreeSet::from(["foo", "foo.dwp"]) } @@ -950,7 +949,7 @@ mod shared_linux_other_tests { .debuginfo(level.cli_value()) .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value())) .remap_path_prefix(cwd(), remapped_prefix) - .arg(format!("-Zremap-path-scope={scope}")) + .arg(format!("--remap-path-scope={scope}")) .run(); let found_files = cwd_filenames(); FileAssertions { expected_files: BTreeSet::from(["foo", "foo.dwp"]) } @@ -1202,7 +1201,7 @@ mod shared_linux_other_tests { .debuginfo(level.cli_value()) .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value())) .remap_path_prefix(cwd(), remapped_prefix) - .arg(format!("-Zremap-path-scope={scope}")) + .arg(format!("--remap-path-scope={scope}")) .run(); let found_files = cwd_filenames(); @@ -1242,7 +1241,7 @@ mod shared_linux_other_tests { .debuginfo(level.cli_value()) .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value())) .remap_path_prefix(cwd(), remapped_prefix) - .arg(format!("-Zremap-path-scope={scope}")) + .arg(format!("--remap-path-scope={scope}")) .run(); let found_files = cwd_filenames(); @@ -1356,7 +1355,7 @@ fn main() { // NOTE: these combinations are not exhaustive, because while porting to rmake.rs initially I // tried to preserve the existing test behavior closely. Notably, no attempt was made to // exhaustively cover all cases in the 6-fold Cartesian product of `{,-Csplit=debuginfo=...}` x - // `{,-Cdebuginfo=...}` x `{,--remap-path-prefix}` x `{,-Zremap-path-scope=...}` x + // `{,-Cdebuginfo=...}` x `{,--remap-path-prefix}` x `{,--remap-path-scope=...}` x // `{,-Zsplit-dwarf-kind=...}` x `{,-Clinker-plugin-lto}`. If you really want to, you can // identify which combination isn't exercised with a 6-layers nested for loop iterating through // each of the cli flag enum variants. diff --git a/tests/ui/errors/auxiliary/file-debuginfo.rs b/tests/ui/errors/auxiliary/file-debuginfo.rs index 08113ec26bfd..3c95512d8cc2 100644 --- a/tests/ui/errors/auxiliary/file-debuginfo.rs +++ b/tests/ui/errors/auxiliary/file-debuginfo.rs @@ -1,5 +1,5 @@ //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ compile-flags: -Zremap-path-scope=debuginfo +//@ compile-flags: --remap-path-scope=debuginfo #[macro_export] macro_rules! my_file { diff --git a/tests/ui/errors/auxiliary/file-diag.rs b/tests/ui/errors/auxiliary/file-diag.rs index f29c349f703b..61fc9d2b4829 100644 --- a/tests/ui/errors/auxiliary/file-diag.rs +++ b/tests/ui/errors/auxiliary/file-diag.rs @@ -1,5 +1,5 @@ //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ compile-flags: -Zremap-path-scope=diagnostics +//@ compile-flags: --remap-path-scope=diagnostics #[macro_export] macro_rules! my_file { diff --git a/tests/ui/errors/auxiliary/file-macro.rs b/tests/ui/errors/auxiliary/file-macro.rs index 11abc0549a7b..11e5de1e3274 100644 --- a/tests/ui/errors/auxiliary/file-macro.rs +++ b/tests/ui/errors/auxiliary/file-macro.rs @@ -1,5 +1,5 @@ //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ compile-flags: -Zremap-path-scope=macro +//@ compile-flags: --remap-path-scope=macro #[macro_export] macro_rules! my_file { diff --git a/tests/ui/errors/auxiliary/trait-debuginfo.rs b/tests/ui/errors/auxiliary/trait-debuginfo.rs index d5a0825fe6d1..dbe3d09d317d 100644 --- a/tests/ui/errors/auxiliary/trait-debuginfo.rs +++ b/tests/ui/errors/auxiliary/trait-debuginfo.rs @@ -1,4 +1,4 @@ //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ compile-flags: -Zremap-path-scope=debuginfo +//@ compile-flags: --remap-path-scope=debuginfo pub trait Trait: std::fmt::Display {} diff --git a/tests/ui/errors/auxiliary/trait-diag.rs b/tests/ui/errors/auxiliary/trait-diag.rs index e07961a276a9..3a30366683c4 100644 --- a/tests/ui/errors/auxiliary/trait-diag.rs +++ b/tests/ui/errors/auxiliary/trait-diag.rs @@ -1,4 +1,4 @@ //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ compile-flags: -Zremap-path-scope=diagnostics +//@ compile-flags: --remap-path-scope=diagnostics pub trait Trait: std::fmt::Display {} diff --git a/tests/ui/errors/auxiliary/trait-macro.rs b/tests/ui/errors/auxiliary/trait-macro.rs index 48673d04ee16..334b1c9bba2f 100644 --- a/tests/ui/errors/auxiliary/trait-macro.rs +++ b/tests/ui/errors/auxiliary/trait-macro.rs @@ -1,4 +1,4 @@ //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ compile-flags: -Zremap-path-scope=macro +//@ compile-flags: --remap-path-scope=macro pub trait Trait: std::fmt::Display {} diff --git a/tests/ui/errors/remap-path-prefix-diagnostics.rs b/tests/ui/errors/remap-path-prefix-diagnostics.rs index fac7e937cb0b..54dbcfd64711 100644 --- a/tests/ui/errors/remap-path-prefix-diagnostics.rs +++ b/tests/ui/errors/remap-path-prefix-diagnostics.rs @@ -1,4 +1,4 @@ -// This test exercises `-Zremap-path-scope`, diagnostics printing paths and dependency. +// This test exercises `--remap-path-scope`, diagnostics printing paths and dependency. // // We test different combinations with/without remap in deps, with/without remap in this // crate but always in deps and always here but never in deps. @@ -12,10 +12,10 @@ //@[with-debuginfo-in-deps] compile-flags: --remap-path-prefix={{src-base}}=remapped //@[not-diag-in-deps] compile-flags: --remap-path-prefix={{src-base}}=remapped -//@[with-diag-in-deps] compile-flags: -Zremap-path-scope=diagnostics -//@[with-macro-in-deps] compile-flags: -Zremap-path-scope=macro -//@[with-debuginfo-in-deps] compile-flags: -Zremap-path-scope=debuginfo -//@[not-diag-in-deps] compile-flags: -Zremap-path-scope=diagnostics +//@[with-diag-in-deps] compile-flags: --remap-path-scope=diagnostics +//@[with-macro-in-deps] compile-flags: --remap-path-scope=macro +//@[with-debuginfo-in-deps] compile-flags: --remap-path-scope=debuginfo +//@[not-diag-in-deps] compile-flags: --remap-path-scope=diagnostics //@[with-diag-in-deps] aux-build:trait-diag.rs //@[with-macro-in-deps] aux-build:trait-macro.rs diff --git a/tests/ui/errors/remap-path-prefix-macro.rs b/tests/ui/errors/remap-path-prefix-macro.rs index 3e93843f9164..1f895aeeb6b4 100644 --- a/tests/ui/errors/remap-path-prefix-macro.rs +++ b/tests/ui/errors/remap-path-prefix-macro.rs @@ -1,4 +1,4 @@ -// This test exercises `-Zremap-path-scope`, macros (like file!()) and dependency. +// This test exercises `--remap-path-scope`, macros (like file!()) and dependency. // // We test different combinations with/without remap in deps, with/without remap in // this crate but always in deps and always here but never in deps. @@ -15,10 +15,10 @@ //@[with-debuginfo-in-deps] compile-flags: --remap-path-prefix={{src-base}}=remapped //@[not-macro-in-deps] compile-flags: --remap-path-prefix={{src-base}}=remapped -//@[with-diag-in-deps] compile-flags: -Zremap-path-scope=diagnostics -//@[with-macro-in-deps] compile-flags: -Zremap-path-scope=macro -//@[with-debuginfo-in-deps] compile-flags: -Zremap-path-scope=debuginfo -//@[not-macro-in-deps] compile-flags: -Zremap-path-scope=macro +//@[with-diag-in-deps] compile-flags: --remap-path-scope=diagnostics +//@[with-macro-in-deps] compile-flags: --remap-path-scope=macro +//@[with-debuginfo-in-deps] compile-flags: --remap-path-scope=debuginfo +//@[not-macro-in-deps] compile-flags: --remap-path-scope=macro //@[with-diag-in-deps] aux-build:file-diag.rs //@[with-macro-in-deps] aux-build:file-macro.rs diff --git a/tests/ui/errors/remap-path-prefix.rs b/tests/ui/errors/remap-path-prefix.rs index de18aa8cc204..b49514711355 100644 --- a/tests/ui/errors/remap-path-prefix.rs +++ b/tests/ui/errors/remap-path-prefix.rs @@ -1,7 +1,7 @@ //@ revisions: normal with-diagnostic-scope without-diagnostic-scope //@ compile-flags: --remap-path-prefix={{src-base}}=remapped -//@ [with-diagnostic-scope]compile-flags: -Zremap-path-scope=diagnostics -//@ [without-diagnostic-scope]compile-flags: -Zremap-path-scope=object +//@ [with-diagnostic-scope]compile-flags: --remap-path-scope=diagnostics +//@ [without-diagnostic-scope]compile-flags: --remap-path-scope=object // Manually remap, so the remapped path remains in .stderr file. // The remapped paths are not normalized by compiletest. From 624135bd79ccfc6602b013225de4a33d5701035f Mon Sep 17 00:00:00 2001 From: Jens Reidel Date: Sat, 13 Dec 2025 20:16:12 +0100 Subject: [PATCH 0053/1843] Promote powerpc64-unknown-linux-musl to tier 2 with host tools Signed-off-by: Jens Reidel --- .../targets/powerpc64_unknown_linux_musl.rs | 4 +- src/bootstrap/src/core/build_steps/llvm.rs | 1 + src/bootstrap/src/core/download.rs | 1 + .../Dockerfile | 2 +- .../powerpc64-linux-gnu.defconfig | 0 .../dist-powerpc64-linux-musl/Dockerfile | 39 +++++++++++++++++++ .../powerpc64-unknown-linux-musl.defconfig | 15 +++++++ src/ci/github-actions/jobs.yml | 5 ++- src/doc/rustc/src/platform-support.md | 2 +- .../powerpc64-unknown-linux-musl.md | 10 +++-- 10 files changed, 70 insertions(+), 9 deletions(-) rename src/ci/docker/host-x86_64/{dist-powerpc64-linux => dist-powerpc64-linux-gnu}/Dockerfile (89%) rename src/ci/docker/host-x86_64/{dist-powerpc64-linux => dist-powerpc64-linux-gnu}/powerpc64-linux-gnu.defconfig (100%) create mode 100644 src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/Dockerfile create mode 100644 src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/powerpc64-unknown-linux-musl.defconfig diff --git a/compiler/rustc_target/src/spec/targets/powerpc64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/powerpc64_unknown_linux_musl.rs index b663ddf962ea..130bcacfc8cc 100644 --- a/compiler/rustc_target/src/spec/targets/powerpc64_unknown_linux_musl.rs +++ b/compiler/rustc_target/src/spec/targets/powerpc64_unknown_linux_musl.rs @@ -17,8 +17,8 @@ pub(crate) fn target() -> Target { llvm_target: "powerpc64-unknown-linux-musl".into(), metadata: TargetMetadata { description: Some("64-bit PowerPC Linux with musl 1.2.5".into()), - tier: Some(3), - host_tools: Some(false), + tier: Some(2), + host_tools: Some(true), std: Some(true), }, pointer_width: 64, diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index db2a76c4a2df..8acc92451519 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -223,6 +223,7 @@ pub(crate) fn is_ci_llvm_available_for_target( ("loongarch64-unknown-linux-musl", false), ("powerpc-unknown-linux-gnu", false), ("powerpc64-unknown-linux-gnu", false), + ("powerpc64-unknown-linux-musl", false), ("powerpc64le-unknown-linux-gnu", false), ("powerpc64le-unknown-linux-musl", false), ("riscv64gc-unknown-linux-gnu", false), diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index d950dc1a1c58..274bd56aff7e 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -460,6 +460,7 @@ pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: boo "loongarch64-unknown-linux-gnu", "powerpc-unknown-linux-gnu", "powerpc64-unknown-linux-gnu", + "powerpc64-unknown-linux-musl", "powerpc64le-unknown-linux-gnu", "powerpc64le-unknown-linux-musl", "riscv64gc-unknown-linux-gnu", diff --git a/src/ci/docker/host-x86_64/dist-powerpc64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64-linux-gnu/Dockerfile similarity index 89% rename from src/ci/docker/host-x86_64/dist-powerpc64-linux/Dockerfile rename to src/ci/docker/host-x86_64/dist-powerpc64-linux-gnu/Dockerfile index 298282a76463..046406224c34 100644 --- a/src/ci/docker/host-x86_64/dist-powerpc64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-powerpc64-linux-gnu/Dockerfile @@ -11,7 +11,7 @@ RUN sh /scripts/rustbuild-setup.sh WORKDIR /tmp COPY scripts/crosstool-ng-build.sh /scripts/ -COPY host-x86_64/dist-powerpc64-linux/powerpc64-linux-gnu.defconfig /tmp/crosstool.defconfig +COPY host-x86_64/dist-powerpc64-linux-gnu/powerpc64-linux-gnu.defconfig /tmp/crosstool.defconfig RUN /scripts/crosstool-ng-build.sh COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-powerpc64-linux/powerpc64-linux-gnu.defconfig b/src/ci/docker/host-x86_64/dist-powerpc64-linux-gnu/powerpc64-linux-gnu.defconfig similarity index 100% rename from src/ci/docker/host-x86_64/dist-powerpc64-linux/powerpc64-linux-gnu.defconfig rename to src/ci/docker/host-x86_64/dist-powerpc64-linux-gnu/powerpc64-linux-gnu.defconfig diff --git a/src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/Dockerfile new file mode 100644 index 000000000000..7c8a1e657ac2 --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:22.04 + +COPY scripts/cross-apt-packages.sh /scripts/ +RUN sh /scripts/cross-apt-packages.sh + +COPY scripts/crosstool-ng.sh /scripts/ +RUN sh /scripts/crosstool-ng.sh + +COPY scripts/rustbuild-setup.sh /scripts/ +RUN sh /scripts/rustbuild-setup.sh + +WORKDIR /tmp + +COPY scripts/crosstool-ng-build.sh /scripts/ +COPY host-x86_64/dist-powerpc64-linux-musl/powerpc64-unknown-linux-musl.defconfig /tmp/crosstool.defconfig +RUN /scripts/crosstool-ng-build.sh + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV PATH=$PATH:/x-tools/powerpc64-unknown-linux-musl/bin + +ENV \ + AR_powerpc64_unknown_linux_musl=powerpc64-unknown-linux-musl-ar \ + CC_powerpc64_unknown_linux_musl=powerpc64-unknown-linux-musl-gcc \ + CXX_powerpc64_unknown_linux_musl=powerpc64-unknown-linux-musl-g++ + +ENV HOSTS=powerpc64-unknown-linux-musl + +ENV RUST_CONFIGURE_ARGS \ + --enable-extended \ + --enable-full-tools \ + --enable-profiler \ + --enable-sanitizers \ + --disable-docs \ + --set target.powerpc64-unknown-linux-musl.crt-static=false \ + --musl-root-powerpc64=/x-tools/powerpc64-unknown-linux-musl/powerpc64-unknown-linux-musl/sysroot/usr + +ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/powerpc64-unknown-linux-musl.defconfig b/src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/powerpc64-unknown-linux-musl.defconfig new file mode 100644 index 000000000000..08132d3ab8ba --- /dev/null +++ b/src/ci/docker/host-x86_64/dist-powerpc64-linux-musl/powerpc64-unknown-linux-musl.defconfig @@ -0,0 +1,15 @@ +CT_CONFIG_VERSION="4" +CT_EXPERIMENTAL=y +CT_PREFIX_DIR="/x-tools/${CT_TARGET}" +CT_USE_MIRROR=y +CT_MIRROR_BASE_URL="https://ci-mirrors.rust-lang.org/rustc" +CT_ARCH_POWERPC=y +CT_ARCH_64=y +CT_ARCH_ABI="elfv2" +# CT_DEMULTILIB is not set +CT_KERNEL_LINUX=y +CT_LINUX_V_4_19=y +CT_LIBC_MUSL=y +CT_MUSL_V_1_2_5=y +CT_CC_LANG_CXX=y +CT_GETTEXT_NEEDED=y diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 9acad5c06b21..38c30f8b8e29 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -232,7 +232,10 @@ auto: - name: dist-powerpc-linux <<: *job-linux-4c - - name: dist-powerpc64-linux + - name: dist-powerpc64-linux-gnu + <<: *job-linux-4c + + - name: dist-powerpc64-linux-musl <<: *job-linux-4c - name: dist-powerpc64le-linux-gnu diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index d772702df76e..c2b26e56ef49 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -100,6 +100,7 @@ target | notes [`i686-pc-windows-gnu`](platform-support/windows-gnu.md) | 32-bit MinGW (Windows 10+, Windows Server 2016+, Pentium 4) [^x86_32-floats-return-ABI] [^win32-msvc-alignment] `powerpc-unknown-linux-gnu` | PowerPC Linux (kernel 3.2+, glibc 2.17) `powerpc64-unknown-linux-gnu` | PPC64 Linux (kernel 3.2+, glibc 2.17) +[`powerpc64-unknown-linux-musl`](platform-support/powerpc64-unknown-linux-musl.md) | PPC64 Linux (kernel 4.19+, musl 1.2.5) [`powerpc64le-unknown-linux-gnu`](platform-support/powerpc64le-unknown-linux-gnu.md) | PPC64LE Linux (kernel 3.10+, glibc 2.17) [`powerpc64le-unknown-linux-musl`](platform-support/powerpc64le-unknown-linux-musl.md) | PPC64LE Linux (kernel 4.19+, musl 1.2.5) [`riscv64gc-unknown-linux-gnu`](platform-support/riscv64gc-unknown-linux-gnu.md) | RISC-V Linux (kernel 4.20+, glibc 2.29) @@ -369,7 +370,6 @@ target | std | host | notes [`powerpc-wrs-vxworks-spe`](platform-support/vxworks.md) | ✓ | | [`powerpc64-ibm-aix`](platform-support/aix.md) | ? | | 64-bit AIX (7.2 and newer) [`powerpc64-unknown-freebsd`](platform-support/freebsd.md) | ✓ | ✓ | PPC64 FreeBSD (ELFv2) -[`powerpc64-unknown-linux-musl`](platform-support/powerpc64-unknown-linux-musl.md) | ✓ | ✓ | PPC64 Linux (kernel 4.19, musl 1.2.5) [`powerpc64-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | OpenBSD/powerpc64 [`powerpc64-wrs-vxworks`](platform-support/vxworks.md) | ✓ | | [`powerpc64le-unknown-freebsd`](platform-support/freebsd.md) | ✓ | ✓ | PPC64LE FreeBSD diff --git a/src/doc/rustc/src/platform-support/powerpc64-unknown-linux-musl.md b/src/doc/rustc/src/platform-support/powerpc64-unknown-linux-musl.md index 7213e54d5a1a..23cb31af6a26 100644 --- a/src/doc/rustc/src/platform-support/powerpc64-unknown-linux-musl.md +++ b/src/doc/rustc/src/platform-support/powerpc64-unknown-linux-musl.md @@ -1,10 +1,13 @@ # powerpc64-unknown-linux-musl -**Tier: 3** +**Tier: 2** Target for 64-bit big endian PowerPC Linux programs using musl libc. This target uses the ELF v2 ABI. +The baseline CPU required is a PowerPC 970, which means AltiVec is required and +the oldest IBM server CPU supported is therefore POWER6. + ## Target maintainers [@Gelbpunkt](https://github.com/Gelbpunkt) @@ -38,9 +41,8 @@ linker = "powerpc64-linux-musl-gcc" ## Building Rust programs -Rust does not yet ship pre-compiled artifacts for this target. To compile for -this target, you will first need to build Rust with the target enabled (see -"Building the target" above). +This target is distributed through `rustup`, and otherwise requires no +special configuration. ## Cross-compilation From fd1a753d2b555ff91cbd7a0f552466a9137745b6 Mon Sep 17 00:00:00 2001 From: hulxv Date: Wed, 17 Dec 2025 22:54:51 +0200 Subject: [PATCH 0054/1843] Refactor CPU affinity tests to use errno_check for error handling --- .../miri/tests/pass-dep/libc/libc-affinity.rs | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/src/tools/miri/tests/pass-dep/libc/libc-affinity.rs b/src/tools/miri/tests/pass-dep/libc/libc-affinity.rs index 400e3ca3d7db..87ef0510af4f 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-affinity.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-affinity.rs @@ -7,6 +7,10 @@ use std::mem::{size_of, size_of_val}; use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity}; +#[path = "../../utils/libc.rs"] +mod libc_utils; +use libc_utils::errno_check; + // If pid is zero, then the calling thread is used. const PID: i32 = 0; @@ -41,8 +45,7 @@ fn configure_unavailable_cpu() { // Safety: valid value for this type let mut cpuset: cpu_set_t = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }); // by default, only available CPUs are configured for i in 0..cpu_count { @@ -53,11 +56,9 @@ fn configure_unavailable_cpu() { // configure CPU that we don't have unsafe { libc::CPU_SET(cpu_count, &mut cpuset) }; - let err = unsafe { sched_setaffinity(PID, size_of::(), &cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_setaffinity(PID, size_of::(), &cpuset) }); - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }); // the CPU is not set because it is not available assert!(!unsafe { libc::CPU_ISSET(cpu_count, &cpuset) }); @@ -70,11 +71,11 @@ fn large_set() { // i.e. this has 2048 bits, twice the standard number let mut cpuset = [u64::MAX; 32]; - let err = unsafe { sched_setaffinity(PID, size_of_val(&cpuset), cpuset.as_ptr().cast()) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_setaffinity(PID, size_of_val(&cpuset), cpuset.as_ptr().cast()) }); - let err = unsafe { sched_getaffinity(PID, size_of_val(&cpuset), cpuset.as_mut_ptr().cast()) }; - assert_eq!(err, 0); + errno_check(unsafe { + sched_getaffinity(PID, size_of_val(&cpuset), cpuset.as_mut_ptr().cast()) + }); } fn get_small_cpu_mask() { @@ -91,8 +92,7 @@ fn get_small_cpu_mask() { assert_eq!(std::io::Error::last_os_error().kind(), std::io::ErrorKind::InvalidInput); } else { // other whole multiples of the size of c_ulong works - let err = unsafe { sched_getaffinity(PID, i, &mut cpuset) }; - assert_eq!(err, 0, "fail for {i}"); + errno_check(unsafe { sched_getaffinity(PID, i, &mut cpuset) }); } // anything else returns an error @@ -107,8 +107,7 @@ fn get_small_cpu_mask() { fn set_small_cpu_mask() { let mut cpuset: cpu_set_t = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }); // setting a mask of size 0 is invalid let err = unsafe { sched_setaffinity(PID, 0, &cpuset) }; @@ -122,8 +121,7 @@ fn set_small_cpu_mask() { if cfg!(target_endian = "little") { 1 } else { core::mem::size_of::() }; for i in cpu_zero_included_length..24 { - let err = unsafe { sched_setaffinity(PID, i, &cpuset) }; - assert_eq!(err, 0, "fail for {i}"); + errno_check(unsafe { sched_setaffinity(PID, i, &cpuset) }); } } @@ -135,8 +133,7 @@ fn set_custom_cpu_mask() { let mut cpuset: cpu_set_t = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; // at the start, thread 1 should be set - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }); assert!(unsafe { libc::CPU_ISSET(1, &cpuset) }); // make a valid mask @@ -144,12 +141,10 @@ fn set_custom_cpu_mask() { unsafe { libc::CPU_SET(0, &mut cpuset) }; // giving a smaller mask is fine - let err = unsafe { sched_setaffinity(PID, 8, &cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_setaffinity(PID, 8, &cpuset) }); // and actually disables other threads - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }); assert!(unsafe { !libc::CPU_ISSET(1, &cpuset) }); // it is important that we reset the cpu mask now for future tests @@ -157,8 +152,7 @@ fn set_custom_cpu_mask() { unsafe { libc::CPU_SET(i, &mut cpuset) }; } - let err = unsafe { sched_setaffinity(PID, size_of::(), &cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_setaffinity(PID, size_of::(), &cpuset) }); } fn parent_child() { @@ -170,15 +164,13 @@ fn parent_child() { let mut parent_cpuset: cpu_set_t = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; unsafe { libc::CPU_SET(0, &mut parent_cpuset) }; - let err = unsafe { sched_setaffinity(PID, size_of::(), &parent_cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_setaffinity(PID, size_of::(), &parent_cpuset) }); std::thread::scope(|spawner| { spawner.spawn(|| { let mut cpuset: cpu_set_t = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut cpuset) }); // the child inherits its parent's set assert!(unsafe { libc::CPU_ISSET(0, &cpuset) }); @@ -189,8 +181,7 @@ fn parent_child() { }); }); - let err = unsafe { sched_getaffinity(PID, size_of::(), &mut parent_cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_getaffinity(PID, size_of::(), &mut parent_cpuset) }); // the parent's set should be unaffected assert!(unsafe { !libc::CPU_ISSET(1, &parent_cpuset) }); @@ -201,8 +192,7 @@ fn parent_child() { unsafe { libc::CPU_SET(i, &mut cpuset) }; } - let err = unsafe { sched_setaffinity(PID, size_of::(), &cpuset) }; - assert_eq!(err, 0); + errno_check(unsafe { sched_setaffinity(PID, size_of::(), &cpuset) }); } fn main() { From d5d605acd46dd767c2aef0ff80c00328daf35b9f Mon Sep 17 00:00:00 2001 From: hulxv Date: Wed, 17 Dec 2025 23:05:04 +0200 Subject: [PATCH 0055/1843] Refactor libc time tests to use errno_check for error handling --- .../miri/tests/pass-dep/libc/libc-time.rs | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/src/tools/miri/tests/pass-dep/libc/libc-time.rs b/src/tools/miri/tests/pass-dep/libc/libc-time.rs index 9e9fadfca9e7..b80fb0025530 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-time.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-time.rs @@ -1,8 +1,13 @@ //@ignore-target: windows # no libc time APIs on Windows //@compile-flags: -Zmiri-disable-isolation + +#[path = "../../utils/libc.rs"] +mod libc_utils; use std::time::{Duration, Instant}; use std::{env, mem, ptr}; +use libc_utils::errno_check; + fn main() { test_clocks(); test_posix_gettimeofday(); @@ -39,30 +44,23 @@ fn main() { /// Tests whether clock support exists at all fn test_clocks() { let mut tp = mem::MaybeUninit::::uninit(); - let is_error = unsafe { libc::clock_gettime(libc::CLOCK_REALTIME, tp.as_mut_ptr()) }; - assert_eq!(is_error, 0); - let is_error = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, tp.as_mut_ptr()) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::clock_gettime(libc::CLOCK_REALTIME, tp.as_mut_ptr()) }); + errno_check(unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, tp.as_mut_ptr()) }); #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "android"))] { - let is_error = unsafe { libc::clock_gettime(libc::CLOCK_REALTIME_COARSE, tp.as_mut_ptr()) }; - assert_eq!(is_error, 0); - let is_error = - unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC_COARSE, tp.as_mut_ptr()) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::clock_gettime(libc::CLOCK_REALTIME_COARSE, tp.as_mut_ptr()) }); + errno_check(unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC_COARSE, tp.as_mut_ptr()) }); } #[cfg(target_os = "macos")] { - let is_error = unsafe { libc::clock_gettime(libc::CLOCK_UPTIME_RAW, tp.as_mut_ptr()) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::clock_gettime(libc::CLOCK_UPTIME_RAW, tp.as_mut_ptr()) }); } } fn test_posix_gettimeofday() { let mut tp = mem::MaybeUninit::::uninit(); let tz = ptr::null_mut::(); - let is_error = unsafe { libc::gettimeofday(tp.as_mut_ptr(), tz.cast()) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::gettimeofday(tp.as_mut_ptr(), tz.cast()) }); let tv = unsafe { tp.assume_init() }; assert!(tv.tv_sec > 0); assert!(tv.tv_usec >= 0); // Theoretically this could be 0. @@ -334,15 +332,13 @@ fn test_nanosleep() { let start_test_sleep = Instant::now(); let duration_zero = libc::timespec { tv_sec: 0, tv_nsec: 0 }; let remainder = ptr::null_mut::(); - let is_error = unsafe { libc::nanosleep(&duration_zero, remainder) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::nanosleep(&duration_zero, remainder) }); assert!(start_test_sleep.elapsed() < Duration::from_millis(100)); let start_test_sleep = Instant::now(); let duration_100_millis = libc::timespec { tv_sec: 0, tv_nsec: 1_000_000_000 / 10 }; let remainder = ptr::null_mut::(); - let is_error = unsafe { libc::nanosleep(&duration_100_millis, remainder) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::nanosleep(&duration_100_millis, remainder) }); assert!(start_test_sleep.elapsed() > Duration::from_millis(100)); } @@ -371,8 +367,7 @@ mod test_clock_nanosleep { /// Helper function to get the current time for testing relative sleeps fn timespec_now(clock: libc::clockid_t) -> libc::timespec { let mut timespec = mem::MaybeUninit::::uninit(); - let is_error = unsafe { libc::clock_gettime(clock, timespec.as_mut_ptr()) }; - assert_eq!(is_error, 0); + errno_check(unsafe { libc::clock_gettime(clock, timespec.as_mut_ptr()) }); unsafe { timespec.assume_init() } } @@ -380,7 +375,7 @@ mod test_clock_nanosleep { let start_test_sleep = Instant::now(); let before_start = libc::timespec { tv_sec: 0, tv_nsec: 0 }; let remainder = ptr::null_mut::(); - let error = unsafe { + errno_check(unsafe { // this will not sleep since unix time zero is in the past libc::clock_nanosleep( libc::CLOCK_MONOTONIC, @@ -388,22 +383,20 @@ mod test_clock_nanosleep { &before_start, remainder, ) - }; - assert_eq!(error, 0); + }); assert!(start_test_sleep.elapsed() < Duration::from_millis(100)); let start_test_sleep = Instant::now(); let hunderd_millis_after_start = add_100_millis(timespec_now(libc::CLOCK_MONOTONIC)); let remainder = ptr::null_mut::(); - let error = unsafe { + errno_check(unsafe { libc::clock_nanosleep( libc::CLOCK_MONOTONIC, libc::TIMER_ABSTIME, &hunderd_millis_after_start, remainder, ) - }; - assert_eq!(error, 0); + }); assert!(start_test_sleep.elapsed() > Duration::from_millis(100)); } @@ -413,19 +406,17 @@ mod test_clock_nanosleep { let start_test_sleep = Instant::now(); let duration_zero = libc::timespec { tv_sec: 0, tv_nsec: 0 }; let remainder = ptr::null_mut::(); - let error = unsafe { + errno_check(unsafe { libc::clock_nanosleep(libc::CLOCK_MONOTONIC, NO_FLAGS, &duration_zero, remainder) - }; - assert_eq!(error, 0); + }); assert!(start_test_sleep.elapsed() < Duration::from_millis(100)); let start_test_sleep = Instant::now(); let duration_100_millis = libc::timespec { tv_sec: 0, tv_nsec: 1_000_000_000 / 10 }; let remainder = ptr::null_mut::(); - let error = unsafe { + errno_check(unsafe { libc::clock_nanosleep(libc::CLOCK_MONOTONIC, NO_FLAGS, &duration_100_millis, remainder) - }; - assert_eq!(error, 0); + }); assert!(start_test_sleep.elapsed() > Duration::from_millis(100)); } } From f2283e2ef54539e51d47335fc3ab0435f6efd495 Mon Sep 17 00:00:00 2001 From: yanglsh Date: Fri, 19 Sep 2025 23:51:46 +0800 Subject: [PATCH 0056/1843] fix: `map_unwrap_or` fail to cover `Result::unwrap_or` --- clippy_lints/src/methods/map_unwrap_or.rs | 208 ++++++++++++++---- .../src/methods/map_unwrap_or_else.rs | 71 ++++++ clippy_lints/src/methods/mod.rs | 6 +- .../src/methods/option_map_unwrap_or.rs | 180 --------------- clippy_utils/src/msrvs.rs | 2 +- lintcheck/src/input.rs | 3 +- tests/ui/map_unwrap_or_fixable.fixed | 16 ++ tests/ui/map_unwrap_or_fixable.rs | 16 ++ tests/ui/map_unwrap_or_fixable.stderr | 50 ++++- 9 files changed, 325 insertions(+), 227 deletions(-) create mode 100644 clippy_lints/src/methods/map_unwrap_or_else.rs delete mode 100644 clippy_lints/src/methods/option_map_unwrap_or.rs diff --git a/clippy_lints/src/methods/map_unwrap_or.rs b/clippy_lints/src/methods/map_unwrap_or.rs index 8eb26fb50747..b29be88fb520 100644 --- a/clippy_lints/src/methods/map_unwrap_or.rs +++ b/clippy_lints/src/methods/map_unwrap_or.rs @@ -1,71 +1,199 @@ -use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::msrvs::{self, Msrv}; use clippy_utils::res::MaybeDef; -use clippy_utils::source::snippet; -use clippy_utils::usage::mutated_variables; +use clippy_utils::source::snippet_with_applicability; +use clippy_utils::ty::is_copy; +use rustc_data_structures::fx::FxHashSet; use rustc_errors::Applicability; -use rustc_hir as hir; +use rustc_hir::def::Res; +use rustc_hir::intravisit::{Visitor, walk_path}; +use rustc_hir::{ExprKind, HirId, Node, PatKind, Path, QPath}; use rustc_lint::LateContext; -use rustc_span::symbol::sym; +use rustc_middle::hir::nested_filter; +use rustc_span::{Span, sym}; +use std::ops::ControlFlow; use super::MAP_UNWRAP_OR; -/// lint use of `map().unwrap_or_else()` for `Option`s and `Result`s -/// -/// Returns true if the lint was emitted +/// lint use of `map().unwrap_or()` for `Option`s and `Result`s +#[expect(clippy::too_many_arguments)] pub(super) fn check<'tcx>( cx: &LateContext<'tcx>, - expr: &'tcx hir::Expr<'_>, - recv: &'tcx hir::Expr<'_>, - map_arg: &'tcx hir::Expr<'_>, - unwrap_arg: &'tcx hir::Expr<'_>, + expr: &rustc_hir::Expr<'_>, + recv: &rustc_hir::Expr<'_>, + map_arg: &'tcx rustc_hir::Expr<'_>, + unwrap_recv: &rustc_hir::Expr<'_>, + unwrap_arg: &'tcx rustc_hir::Expr<'_>, + map_span: Span, msrv: Msrv, ) -> bool { // lint if the caller of `map()` is an `Option` or a `Result`. let is_option = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Option); let is_result = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Result); - if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR_ELSE) { + if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR) { return false; } + // lint if the caller of `map()` is an `Option` if is_option || is_result { - // Don't make a suggestion that may fail to compile due to mutably borrowing - // the same variable twice. - let map_mutated_vars = mutated_variables(recv, cx); - let unwrap_mutated_vars = mutated_variables(unwrap_arg, cx); - if let (Some(map_mutated_vars), Some(unwrap_mutated_vars)) = (map_mutated_vars, unwrap_mutated_vars) { - if map_mutated_vars.intersection(&unwrap_mutated_vars).next().is_some() { + if !is_copy(cx, cx.typeck_results().expr_ty(unwrap_arg)) { + // Replacing `.map().unwrap_or()` with `.map_or(, )` can sometimes lead to + // borrowck errors, see #10579 for one such instance. + // In particular, if `a` causes a move and `f` references that moved binding, then we cannot lint: + // ``` + // let x = vec![1, 2]; + // x.get(0..1).map(|s| s.to_vec()).unwrap_or(x); + // ``` + // This compiles, but changing it to `map_or` will produce a compile error: + // ``` + // let x = vec![1, 2]; + // x.get(0..1).map_or(x, |s| s.to_vec()) + // ^ moving `x` here + // ^^^^^^^^^^^ while it is borrowed here (and later used in the closure) + // ``` + // So, we have to check that `a` is not referenced anywhere (even outside of the `.map` closure!) + // before the call to `unwrap_or`. + + let mut unwrap_visitor = UnwrapVisitor { + cx, + identifiers: FxHashSet::default(), + }; + unwrap_visitor.visit_expr(unwrap_arg); + + let mut reference_visitor = ReferenceVisitor { + cx, + identifiers: unwrap_visitor.identifiers, + unwrap_or_span: unwrap_arg.span, + }; + + let body = cx.tcx.hir_body_owned_by(cx.tcx.hir_enclosing_body_owner(expr.hir_id)); + + // Visit the body, and return if we've found a reference + if reference_visitor.visit_body(body).is_break() { return false; } - } else { + } + + if !unwrap_arg.span.eq_ctxt(map_span) { return false; } + // is_some_and is stabilised && `unwrap_or` argument is false; suggest `is_some_and` instead + let suggest_is_some_and = matches!(&unwrap_arg.kind, ExprKind::Lit(lit) + if matches!(lit.node, rustc_ast::LitKind::Bool(false))) + && msrv.meets(cx, msrvs::OPTION_RESULT_IS_VARIANT_AND); + + let mut applicability = Applicability::MachineApplicable; + // get snippet for unwrap_or() + let unwrap_snippet = snippet_with_applicability(cx, unwrap_arg.span, "..", &mut applicability); // lint message - let msg = if is_option { - "called `map().unwrap_or_else()` on an `Option` value" + // comparing the snippet from source to raw text ("None") below is safe + // because we already have checked the type. + let unwrap_snippet_none = is_option && unwrap_snippet == "None"; + let arg = if unwrap_snippet_none { + "None" + } else if suggest_is_some_and { + "false" } else { - "called `map().unwrap_or_else()` on a `Result` value" + "" }; - // get snippets for args to map() and unwrap_or_else() - let map_snippet = snippet(cx, map_arg.span, ".."); - let unwrap_snippet = snippet(cx, unwrap_arg.span, ".."); - // lint, with note if both map() and unwrap_or_else() have the same span - if map_arg.span.eq_ctxt(unwrap_arg.span) { - let var_snippet = snippet(cx, recv.span, ".."); - span_lint_and_sugg( - cx, - MAP_UNWRAP_OR, - expr.span, - msg, - "try", - format!("{var_snippet}.map_or_else({unwrap_snippet}, {map_snippet})"), - Applicability::MachineApplicable, - ); - return true; - } + let suggest = if unwrap_snippet_none { + "and_then()" + } else if suggest_is_some_and { + if is_result { + "is_ok_and()" + } else { + "is_some_and()" + } + } else { + "map_or(, )" + }; + let msg = format!( + "called `map().unwrap_or({arg})` on an `{}` value", + if is_option { "Option" } else { "Result" } + ); + + span_lint_and_then(cx, MAP_UNWRAP_OR, expr.span, msg, |diag| { + let map_arg_span = map_arg.span; + + let mut suggestion = vec![ + ( + map_span, + String::from(if unwrap_snippet_none { + "and_then" + } else if suggest_is_some_and { + if is_result { "is_ok_and" } else { "is_some_and" } + } else { + "map_or" + }), + ), + (expr.span.with_lo(unwrap_recv.span.hi()), String::new()), + ]; + + if !unwrap_snippet_none && !suggest_is_some_and { + suggestion.push((map_arg_span.with_hi(map_arg_span.lo()), format!("{unwrap_snippet}, "))); + } + + diag.multipart_suggestion(format!("use `{suggest}` instead"), suggestion, applicability); + }); + + return true; } false } + +struct UnwrapVisitor<'a, 'tcx> { + cx: &'a LateContext<'tcx>, + identifiers: FxHashSet, +} + +impl<'tcx> Visitor<'tcx> for UnwrapVisitor<'_, 'tcx> { + type NestedFilter = nested_filter::All; + + fn visit_path(&mut self, path: &Path<'tcx>, _: HirId) { + if let Res::Local(local_id) = path.res + && let Node::Pat(pat) = self.cx.tcx.hir_node(local_id) + && let PatKind::Binding(_, local_id, ..) = pat.kind + { + self.identifiers.insert(local_id); + } + walk_path(self, path); + } + + fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt { + self.cx.tcx + } +} + +struct ReferenceVisitor<'a, 'tcx> { + cx: &'a LateContext<'tcx>, + identifiers: FxHashSet, + unwrap_or_span: Span, +} + +impl<'tcx> Visitor<'tcx> for ReferenceVisitor<'_, 'tcx> { + type NestedFilter = nested_filter::All; + type Result = ControlFlow<()>; + fn visit_expr(&mut self, expr: &'tcx rustc_hir::Expr<'_>) -> ControlFlow<()> { + // If we haven't found a reference yet, check if this references + // one of the locals that was moved in the `unwrap_or` argument. + // We are only interested in exprs that appear before the `unwrap_or` call. + if expr.span < self.unwrap_or_span + && let ExprKind::Path(ref path) = expr.kind + && let QPath::Resolved(_, path) = path + && let Res::Local(local_id) = path.res + && let Node::Pat(pat) = self.cx.tcx.hir_node(local_id) + && let PatKind::Binding(_, local_id, ..) = pat.kind + && self.identifiers.contains(&local_id) + { + return ControlFlow::Break(()); + } + rustc_hir::intravisit::walk_expr(self, expr) + } + + fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt { + self.cx.tcx + } +} diff --git a/clippy_lints/src/methods/map_unwrap_or_else.rs b/clippy_lints/src/methods/map_unwrap_or_else.rs new file mode 100644 index 000000000000..8eb26fb50747 --- /dev/null +++ b/clippy_lints/src/methods/map_unwrap_or_else.rs @@ -0,0 +1,71 @@ +use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::msrvs::{self, Msrv}; +use clippy_utils::res::MaybeDef; +use clippy_utils::source::snippet; +use clippy_utils::usage::mutated_variables; +use rustc_errors::Applicability; +use rustc_hir as hir; +use rustc_lint::LateContext; +use rustc_span::symbol::sym; + +use super::MAP_UNWRAP_OR; + +/// lint use of `map().unwrap_or_else()` for `Option`s and `Result`s +/// +/// Returns true if the lint was emitted +pub(super) fn check<'tcx>( + cx: &LateContext<'tcx>, + expr: &'tcx hir::Expr<'_>, + recv: &'tcx hir::Expr<'_>, + map_arg: &'tcx hir::Expr<'_>, + unwrap_arg: &'tcx hir::Expr<'_>, + msrv: Msrv, +) -> bool { + // lint if the caller of `map()` is an `Option` or a `Result`. + let is_option = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Option); + let is_result = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Result); + + if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR_ELSE) { + return false; + } + + if is_option || is_result { + // Don't make a suggestion that may fail to compile due to mutably borrowing + // the same variable twice. + let map_mutated_vars = mutated_variables(recv, cx); + let unwrap_mutated_vars = mutated_variables(unwrap_arg, cx); + if let (Some(map_mutated_vars), Some(unwrap_mutated_vars)) = (map_mutated_vars, unwrap_mutated_vars) { + if map_mutated_vars.intersection(&unwrap_mutated_vars).next().is_some() { + return false; + } + } else { + return false; + } + + // lint message + let msg = if is_option { + "called `map().unwrap_or_else()` on an `Option` value" + } else { + "called `map().unwrap_or_else()` on a `Result` value" + }; + // get snippets for args to map() and unwrap_or_else() + let map_snippet = snippet(cx, map_arg.span, ".."); + let unwrap_snippet = snippet(cx, unwrap_arg.span, ".."); + // lint, with note if both map() and unwrap_or_else() have the same span + if map_arg.span.eq_ctxt(unwrap_arg.span) { + let var_snippet = snippet(cx, recv.span, ".."); + span_lint_and_sugg( + cx, + MAP_UNWRAP_OR, + expr.span, + msg, + "try", + format!("{var_snippet}.map_or_else({unwrap_snippet}, {map_snippet})"), + Applicability::MachineApplicable, + ); + return true; + } + } + + false +} diff --git a/clippy_lints/src/methods/mod.rs b/clippy_lints/src/methods/mod.rs index 48842c8739c0..163205ea87be 100644 --- a/clippy_lints/src/methods/mod.rs +++ b/clippy_lints/src/methods/mod.rs @@ -74,6 +74,7 @@ mod map_err_ignore; mod map_flatten; mod map_identity; mod map_unwrap_or; +mod map_unwrap_or_else; mod map_with_unused_argument_over_ranges; mod mut_mutex_lock; mod needless_as_bytes; @@ -89,7 +90,6 @@ mod open_options; mod option_as_ref_cloned; mod option_as_ref_deref; mod option_map_or_none; -mod option_map_unwrap_or; mod or_fun_call; mod or_then_unwrap; mod path_buf_push_overwrite; @@ -5607,7 +5607,7 @@ impl Methods { manual_saturating_arithmetic::check_unwrap_or(cx, expr, lhs, rhs, u_arg, arith); }, Some((sym::map, m_recv, [m_arg], span, _)) => { - option_map_unwrap_or::check(cx, expr, m_recv, m_arg, recv, u_arg, span, self.msrv); + map_unwrap_or::check(cx, expr, m_recv, m_arg, recv, u_arg, span, self.msrv); }, Some((then_method @ (sym::then | sym::then_some), t_recv, [t_arg], _, _)) => { obfuscated_if_else::check( @@ -5648,7 +5648,7 @@ impl Methods { (sym::unwrap_or_else, [u_arg]) => { match method_call(recv) { Some((sym::map, recv, [map_arg], _, _)) - if map_unwrap_or::check(cx, expr, recv, map_arg, u_arg, self.msrv) => {}, + if map_unwrap_or_else::check(cx, expr, recv, map_arg, u_arg, self.msrv) => {}, Some((then_method @ (sym::then | sym::then_some), t_recv, [t_arg], _, _)) => { obfuscated_if_else::check( cx, diff --git a/clippy_lints/src/methods/option_map_unwrap_or.rs b/clippy_lints/src/methods/option_map_unwrap_or.rs deleted file mode 100644 index 32a9b4fe7c58..000000000000 --- a/clippy_lints/src/methods/option_map_unwrap_or.rs +++ /dev/null @@ -1,180 +0,0 @@ -use clippy_utils::diagnostics::span_lint_and_then; -use clippy_utils::msrvs::{self, Msrv}; -use clippy_utils::res::MaybeDef; -use clippy_utils::source::snippet_with_applicability; -use clippy_utils::ty::is_copy; -use rustc_data_structures::fx::FxHashSet; -use rustc_errors::Applicability; -use rustc_hir::def::Res; -use rustc_hir::intravisit::{Visitor, walk_path}; -use rustc_hir::{ExprKind, HirId, Node, PatKind, Path, QPath}; -use rustc_lint::LateContext; -use rustc_middle::hir::nested_filter; -use rustc_span::{Span, sym}; -use std::ops::ControlFlow; - -use super::MAP_UNWRAP_OR; - -/// lint use of `map().unwrap_or()` for `Option`s -#[expect(clippy::too_many_arguments)] -pub(super) fn check<'tcx>( - cx: &LateContext<'tcx>, - expr: &rustc_hir::Expr<'_>, - recv: &rustc_hir::Expr<'_>, - map_arg: &'tcx rustc_hir::Expr<'_>, - unwrap_recv: &rustc_hir::Expr<'_>, - unwrap_arg: &'tcx rustc_hir::Expr<'_>, - map_span: Span, - msrv: Msrv, -) { - // lint if the caller of `map()` is an `Option` - if cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Option) { - if !is_copy(cx, cx.typeck_results().expr_ty(unwrap_arg)) { - // Replacing `.map().unwrap_or()` with `.map_or(, )` can sometimes lead to - // borrowck errors, see #10579 for one such instance. - // In particular, if `a` causes a move and `f` references that moved binding, then we cannot lint: - // ``` - // let x = vec![1, 2]; - // x.get(0..1).map(|s| s.to_vec()).unwrap_or(x); - // ``` - // This compiles, but changing it to `map_or` will produce a compile error: - // ``` - // let x = vec![1, 2]; - // x.get(0..1).map_or(x, |s| s.to_vec()) - // ^ moving `x` here - // ^^^^^^^^^^^ while it is borrowed here (and later used in the closure) - // ``` - // So, we have to check that `a` is not referenced anywhere (even outside of the `.map` closure!) - // before the call to `unwrap_or`. - - let mut unwrap_visitor = UnwrapVisitor { - cx, - identifiers: FxHashSet::default(), - }; - unwrap_visitor.visit_expr(unwrap_arg); - - let mut reference_visitor = ReferenceVisitor { - cx, - identifiers: unwrap_visitor.identifiers, - unwrap_or_span: unwrap_arg.span, - }; - - let body = cx.tcx.hir_body_owned_by(cx.tcx.hir_enclosing_body_owner(expr.hir_id)); - - // Visit the body, and return if we've found a reference - if reference_visitor.visit_body(body).is_break() { - return; - } - } - - if !unwrap_arg.span.eq_ctxt(map_span) { - return; - } - - // is_some_and is stabilised && `unwrap_or` argument is false; suggest `is_some_and` instead - let suggest_is_some_and = matches!(&unwrap_arg.kind, ExprKind::Lit(lit) - if matches!(lit.node, rustc_ast::LitKind::Bool(false))) - && msrv.meets(cx, msrvs::OPTION_RESULT_IS_VARIANT_AND); - - let mut applicability = Applicability::MachineApplicable; - // get snippet for unwrap_or() - let unwrap_snippet = snippet_with_applicability(cx, unwrap_arg.span, "..", &mut applicability); - // lint message - // comparing the snippet from source to raw text ("None") below is safe - // because we already have checked the type. - let arg = if unwrap_snippet == "None" { - "None" - } else if suggest_is_some_and { - "false" - } else { - "" - }; - let unwrap_snippet_none = unwrap_snippet == "None"; - let suggest = if unwrap_snippet_none { - "and_then()" - } else if suggest_is_some_and { - "is_some_and()" - } else { - "map_or(, )" - }; - let msg = format!("called `map().unwrap_or({arg})` on an `Option` value"); - - span_lint_and_then(cx, MAP_UNWRAP_OR, expr.span, msg, |diag| { - let map_arg_span = map_arg.span; - - let mut suggestion = vec![ - ( - map_span, - String::from(if unwrap_snippet_none { - "and_then" - } else if suggest_is_some_and { - "is_some_and" - } else { - "map_or" - }), - ), - (expr.span.with_lo(unwrap_recv.span.hi()), String::new()), - ]; - - if !unwrap_snippet_none && !suggest_is_some_and { - suggestion.push((map_arg_span.with_hi(map_arg_span.lo()), format!("{unwrap_snippet}, "))); - } - - diag.multipart_suggestion(format!("use `{suggest}` instead"), suggestion, applicability); - }); - } -} - -struct UnwrapVisitor<'a, 'tcx> { - cx: &'a LateContext<'tcx>, - identifiers: FxHashSet, -} - -impl<'tcx> Visitor<'tcx> for UnwrapVisitor<'_, 'tcx> { - type NestedFilter = nested_filter::All; - - fn visit_path(&mut self, path: &Path<'tcx>, _: HirId) { - if let Res::Local(local_id) = path.res - && let Node::Pat(pat) = self.cx.tcx.hir_node(local_id) - && let PatKind::Binding(_, local_id, ..) = pat.kind - { - self.identifiers.insert(local_id); - } - walk_path(self, path); - } - - fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt { - self.cx.tcx - } -} - -struct ReferenceVisitor<'a, 'tcx> { - cx: &'a LateContext<'tcx>, - identifiers: FxHashSet, - unwrap_or_span: Span, -} - -impl<'tcx> Visitor<'tcx> for ReferenceVisitor<'_, 'tcx> { - type NestedFilter = nested_filter::All; - type Result = ControlFlow<()>; - fn visit_expr(&mut self, expr: &'tcx rustc_hir::Expr<'_>) -> ControlFlow<()> { - // If we haven't found a reference yet, check if this references - // one of the locals that was moved in the `unwrap_or` argument. - // We are only interested in exprs that appear before the `unwrap_or` call. - if expr.span < self.unwrap_or_span - && let ExprKind::Path(ref path) = expr.kind - && let QPath::Resolved(_, path) = path - && let Res::Local(local_id) = path.res - && let Node::Pat(pat) = self.cx.tcx.hir_node(local_id) - && let PatKind::Binding(_, local_id, ..) = pat.kind - && self.identifiers.contains(&local_id) - { - return ControlFlow::Break(()); - } - rustc_hir::intravisit::walk_expr(self, expr) - } - - fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt { - self.cx.tcx - } -} diff --git a/clippy_utils/src/msrvs.rs b/clippy_utils/src/msrvs.rs index 4a7fa3472cae..144ed3dec7d0 100644 --- a/clippy_utils/src/msrvs.rs +++ b/clippy_utils/src/msrvs.rs @@ -61,7 +61,7 @@ msrv_aliases! { 1,45,0 { STR_STRIP_PREFIX } 1,43,0 { LOG2_10, LOG10_2, NUMERIC_ASSOCIATED_CONSTANTS } 1,42,0 { MATCHES_MACRO, SLICE_PATTERNS, PTR_SLICE_RAW_PARTS } - 1,41,0 { RE_REBALANCING_COHERENCE, RESULT_MAP_OR_ELSE } + 1,41,0 { RE_REBALANCING_COHERENCE, RESULT_MAP_OR, RESULT_MAP_OR_ELSE } 1,40,0 { MEM_TAKE, NON_EXHAUSTIVE, OPTION_AS_DEREF } 1,38,0 { POINTER_CAST, REM_EUCLID } 1,37,0 { TYPE_ALIAS_ENUM_VARIANTS } diff --git a/lintcheck/src/input.rs b/lintcheck/src/input.rs index 7dda2b7b25f8..ee3fcaa0a84a 100644 --- a/lintcheck/src/input.rs +++ b/lintcheck/src/input.rs @@ -281,8 +281,7 @@ impl CrateWithSource { CrateSource::Path { path } => { fn is_cache_dir(entry: &DirEntry) -> bool { fs::read(entry.path().join("CACHEDIR.TAG")) - .map(|x| x.starts_with(b"Signature: 8a477f597d28d172789f06886806bc55")) - .unwrap_or(false) + .is_ok_and(|x| x.starts_with(b"Signature: 8a477f597d28d172789f06886806bc55")) } // copy path into the dest_crate_root but skip directories that contain a CACHEDIR.TAG file. diff --git a/tests/ui/map_unwrap_or_fixable.fixed b/tests/ui/map_unwrap_or_fixable.fixed index 90f3cf8bab04..a7a2f0693210 100644 --- a/tests/ui/map_unwrap_or_fixable.fixed +++ b/tests/ui/map_unwrap_or_fixable.fixed @@ -51,3 +51,19 @@ fn main() { option_methods(); result_methods(); } + +fn issue15714() { + let o: Option = Some(3); + let r: Result = Ok(3); + println!("{}", o.map_or(3, |y| y + 1)); + //~^ map_unwrap_or + println!("{}", o.map_or_else(|| 3, |y| y + 1)); + //~^ map_unwrap_or + println!("{}", r.map_or(3, |y| y + 1)); + //~^ map_unwrap_or + println!("{}", r.map_or_else(|()| 3, |y| y + 1)); + //~^ map_unwrap_or + + println!("{}", r.is_ok_and(|y| y == 1)); + //~^ map_unwrap_or +} diff --git a/tests/ui/map_unwrap_or_fixable.rs b/tests/ui/map_unwrap_or_fixable.rs index 1078c7a3cf34..f12f058c1c4b 100644 --- a/tests/ui/map_unwrap_or_fixable.rs +++ b/tests/ui/map_unwrap_or_fixable.rs @@ -57,3 +57,19 @@ fn main() { option_methods(); result_methods(); } + +fn issue15714() { + let o: Option = Some(3); + let r: Result = Ok(3); + println!("{}", o.map(|y| y + 1).unwrap_or(3)); + //~^ map_unwrap_or + println!("{}", o.map(|y| y + 1).unwrap_or_else(|| 3)); + //~^ map_unwrap_or + println!("{}", r.map(|y| y + 1).unwrap_or(3)); + //~^ map_unwrap_or + println!("{}", r.map(|y| y + 1).unwrap_or_else(|()| 3)); + //~^ map_unwrap_or + + println!("{}", r.map(|y| y == 1).unwrap_or(false)); + //~^ map_unwrap_or +} diff --git a/tests/ui/map_unwrap_or_fixable.stderr b/tests/ui/map_unwrap_or_fixable.stderr index 99e660f8dbd1..083a2510bdf2 100644 --- a/tests/ui/map_unwrap_or_fixable.stderr +++ b/tests/ui/map_unwrap_or_fixable.stderr @@ -19,5 +19,53 @@ LL | let _ = res.map(|x| x + 1) LL | | .unwrap_or_else(|_e| 0); | |_______________________________^ help: try: `res.map_or_else(|_e| 0, |x| x + 1)` -error: aborting due to 2 previous errors +error: called `map().unwrap_or()` on an `Option` value + --> tests/ui/map_unwrap_or_fixable.rs:64:20 + | +LL | println!("{}", o.map(|y| y + 1).unwrap_or(3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: use `map_or(, )` instead + | +LL - println!("{}", o.map(|y| y + 1).unwrap_or(3)); +LL + println!("{}", o.map_or(3, |y| y + 1)); + | + +error: called `map().unwrap_or_else()` on an `Option` value + --> tests/ui/map_unwrap_or_fixable.rs:66:20 + | +LL | println!("{}", o.map(|y| y + 1).unwrap_or_else(|| 3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `o.map_or_else(|| 3, |y| y + 1)` + +error: called `map().unwrap_or()` on an `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:68:20 + | +LL | println!("{}", r.map(|y| y + 1).unwrap_or(3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: use `map_or(, )` instead + | +LL - println!("{}", r.map(|y| y + 1).unwrap_or(3)); +LL + println!("{}", r.map_or(3, |y| y + 1)); + | + +error: called `map().unwrap_or_else()` on a `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:70:20 + | +LL | println!("{}", r.map(|y| y + 1).unwrap_or_else(|()| 3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `r.map_or_else(|()| 3, |y| y + 1)` + +error: called `map().unwrap_or(false)` on an `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:73:20 + | +LL | println!("{}", r.map(|y| y == 1).unwrap_or(false)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: use `is_ok_and()` instead + | +LL - println!("{}", r.map(|y| y == 1).unwrap_or(false)); +LL + println!("{}", r.is_ok_and(|y| y == 1)); + | + +error: aborting due to 7 previous errors From 9565b28598976f62305ea3fb9a877b2628bb1c14 Mon Sep 17 00:00:00 2001 From: yanglsh Date: Wed, 24 Sep 2025 00:04:49 +0800 Subject: [PATCH 0057/1843] fix: `map_unwrap_or` FN on references --- clippy_lints/src/methods/map_unwrap_or.rs | 18 +++---- .../src/methods/map_unwrap_or_else.rs | 6 +-- tests/ui/map_unwrap_or_fixable.fixed | 19 +++++++ tests/ui/map_unwrap_or_fixable.rs | 19 +++++++ tests/ui/map_unwrap_or_fixable.stderr | 52 ++++++++++++++++--- 5 files changed, 92 insertions(+), 22 deletions(-) diff --git a/clippy_lints/src/methods/map_unwrap_or.rs b/clippy_lints/src/methods/map_unwrap_or.rs index b29be88fb520..f2e3cc5c0091 100644 --- a/clippy_lints/src/methods/map_unwrap_or.rs +++ b/clippy_lints/src/methods/map_unwrap_or.rs @@ -26,13 +26,13 @@ pub(super) fn check<'tcx>( unwrap_arg: &'tcx rustc_hir::Expr<'_>, map_span: Span, msrv: Msrv, -) -> bool { - // lint if the caller of `map()` is an `Option` or a `Result`. - let is_option = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Option); - let is_result = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Result); +) { + let recv_ty = cx.typeck_results().expr_ty(recv).peel_refs(); + let is_option = recv_ty.is_diag_item(cx, sym::Option); + let is_result = recv_ty.is_diag_item(cx, sym::Result); if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR) { - return false; + return; } // lint if the caller of `map()` is an `Option` @@ -71,12 +71,12 @@ pub(super) fn check<'tcx>( // Visit the body, and return if we've found a reference if reference_visitor.visit_body(body).is_break() { - return false; + return; } } if !unwrap_arg.span.eq_ctxt(map_span) { - return false; + return; } // is_some_and is stabilised && `unwrap_or` argument is false; suggest `is_some_and` instead @@ -137,11 +137,7 @@ pub(super) fn check<'tcx>( diag.multipart_suggestion(format!("use `{suggest}` instead"), suggestion, applicability); }); - - return true; } - - false } struct UnwrapVisitor<'a, 'tcx> { diff --git a/clippy_lints/src/methods/map_unwrap_or_else.rs b/clippy_lints/src/methods/map_unwrap_or_else.rs index 8eb26fb50747..a2f157c0cb8a 100644 --- a/clippy_lints/src/methods/map_unwrap_or_else.rs +++ b/clippy_lints/src/methods/map_unwrap_or_else.rs @@ -21,9 +21,9 @@ pub(super) fn check<'tcx>( unwrap_arg: &'tcx hir::Expr<'_>, msrv: Msrv, ) -> bool { - // lint if the caller of `map()` is an `Option` or a `Result`. - let is_option = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Option); - let is_result = cx.typeck_results().expr_ty(recv).is_diag_item(cx, sym::Result); + let recv_ty = cx.typeck_results().expr_ty(recv).peel_refs(); + let is_option = recv_ty.is_diag_item(cx, sym::Option); + let is_result = recv_ty.is_diag_item(cx, sym::Result); if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR_ELSE) { return false; diff --git a/tests/ui/map_unwrap_or_fixable.fixed b/tests/ui/map_unwrap_or_fixable.fixed index a7a2f0693210..1789c7f4d487 100644 --- a/tests/ui/map_unwrap_or_fixable.fixed +++ b/tests/ui/map_unwrap_or_fixable.fixed @@ -1,6 +1,7 @@ //@aux-build:option_helpers.rs #![warn(clippy::map_unwrap_or)] +#![allow(clippy::unnecessary_lazy_evaluations)] #[macro_use] extern crate option_helpers; @@ -67,3 +68,21 @@ fn issue15714() { println!("{}", r.is_ok_and(|y| y == 1)); //~^ map_unwrap_or } + +fn issue15713() { + let x = &Some(3); + println!("{}", x.map_or(3, |y| y + 1)); + //~^ map_unwrap_or + + let x: &Result = &Ok(3); + println!("{}", x.map_or(3, |y| y + 1)); + //~^ map_unwrap_or + + let x = &Some(3); + println!("{}", x.map_or_else(|| 3, |y| y + 1)); + //~^ map_unwrap_or + + let x: &Result = &Ok(3); + println!("{}", x.map_or_else(|_| 3, |y| y + 1)); + //~^ map_unwrap_or +} diff --git a/tests/ui/map_unwrap_or_fixable.rs b/tests/ui/map_unwrap_or_fixable.rs index f12f058c1c4b..309067edce4d 100644 --- a/tests/ui/map_unwrap_or_fixable.rs +++ b/tests/ui/map_unwrap_or_fixable.rs @@ -1,6 +1,7 @@ //@aux-build:option_helpers.rs #![warn(clippy::map_unwrap_or)] +#![allow(clippy::unnecessary_lazy_evaluations)] #[macro_use] extern crate option_helpers; @@ -73,3 +74,21 @@ fn issue15714() { println!("{}", r.map(|y| y == 1).unwrap_or(false)); //~^ map_unwrap_or } + +fn issue15713() { + let x = &Some(3); + println!("{}", x.map(|y| y + 1).unwrap_or(3)); + //~^ map_unwrap_or + + let x: &Result = &Ok(3); + println!("{}", x.map(|y| y + 1).unwrap_or(3)); + //~^ map_unwrap_or + + let x = &Some(3); + println!("{}", x.map(|y| y + 1).unwrap_or_else(|| 3)); + //~^ map_unwrap_or + + let x: &Result = &Ok(3); + println!("{}", x.map(|y| y + 1).unwrap_or_else(|_| 3)); + //~^ map_unwrap_or +} diff --git a/tests/ui/map_unwrap_or_fixable.stderr b/tests/ui/map_unwrap_or_fixable.stderr index 083a2510bdf2..696f516ee055 100644 --- a/tests/ui/map_unwrap_or_fixable.stderr +++ b/tests/ui/map_unwrap_or_fixable.stderr @@ -1,5 +1,5 @@ error: called `map().unwrap_or_else()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:16:13 + --> tests/ui/map_unwrap_or_fixable.rs:17:13 | LL | let _ = opt.map(|x| x + 1) | _____________^ @@ -11,7 +11,7 @@ LL | | .unwrap_or_else(|| 0); = help: to override `-D warnings` add `#[allow(clippy::map_unwrap_or)]` error: called `map().unwrap_or_else()` on a `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:47:13 + --> tests/ui/map_unwrap_or_fixable.rs:48:13 | LL | let _ = res.map(|x| x + 1) | _____________^ @@ -20,7 +20,7 @@ LL | | .unwrap_or_else(|_e| 0); | |_______________________________^ help: try: `res.map_or_else(|_e| 0, |x| x + 1)` error: called `map().unwrap_or()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:64:20 + --> tests/ui/map_unwrap_or_fixable.rs:65:20 | LL | println!("{}", o.map(|y| y + 1).unwrap_or(3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -32,13 +32,13 @@ LL + println!("{}", o.map_or(3, |y| y + 1)); | error: called `map().unwrap_or_else()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:66:20 + --> tests/ui/map_unwrap_or_fixable.rs:67:20 | LL | println!("{}", o.map(|y| y + 1).unwrap_or_else(|| 3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `o.map_or_else(|| 3, |y| y + 1)` error: called `map().unwrap_or()` on an `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:68:20 + --> tests/ui/map_unwrap_or_fixable.rs:69:20 | LL | println!("{}", r.map(|y| y + 1).unwrap_or(3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -50,13 +50,13 @@ LL + println!("{}", r.map_or(3, |y| y + 1)); | error: called `map().unwrap_or_else()` on a `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:70:20 + --> tests/ui/map_unwrap_or_fixable.rs:71:20 | LL | println!("{}", r.map(|y| y + 1).unwrap_or_else(|()| 3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `r.map_or_else(|()| 3, |y| y + 1)` error: called `map().unwrap_or(false)` on an `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:73:20 + --> tests/ui/map_unwrap_or_fixable.rs:74:20 | LL | println!("{}", r.map(|y| y == 1).unwrap_or(false)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -67,5 +67,41 @@ LL - println!("{}", r.map(|y| y == 1).unwrap_or(false)); LL + println!("{}", r.is_ok_and(|y| y == 1)); | -error: aborting due to 7 previous errors +error: called `map().unwrap_or()` on an `Option` value + --> tests/ui/map_unwrap_or_fixable.rs:80:20 + | +LL | println!("{}", x.map(|y| y + 1).unwrap_or(3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: use `map_or(, )` instead + | +LL - println!("{}", x.map(|y| y + 1).unwrap_or(3)); +LL + println!("{}", x.map_or(3, |y| y + 1)); + | + +error: called `map().unwrap_or()` on an `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:84:20 + | +LL | println!("{}", x.map(|y| y + 1).unwrap_or(3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: use `map_or(, )` instead + | +LL - println!("{}", x.map(|y| y + 1).unwrap_or(3)); +LL + println!("{}", x.map_or(3, |y| y + 1)); + | + +error: called `map().unwrap_or_else()` on an `Option` value + --> tests/ui/map_unwrap_or_fixable.rs:88:20 + | +LL | println!("{}", x.map(|y| y + 1).unwrap_or_else(|| 3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `x.map_or_else(|| 3, |y| y + 1)` + +error: called `map().unwrap_or_else()` on a `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:92:20 + | +LL | println!("{}", x.map(|y| y + 1).unwrap_or_else(|_| 3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `x.map_or_else(|_| 3, |y| y + 1)` + +error: aborting due to 11 previous errors From bd0341471abbf02331061d76ca54fe801f95fb52 Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Fri, 19 Dec 2025 11:20:38 +0100 Subject: [PATCH 0058/1843] Post `needless_continue` diagnostic on the right node --- clippy_lints/src/needless_continue.rs | 31 +++++++++++------------- tests/ui/needless_continue.rs | 35 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/clippy_lints/src/needless_continue.rs b/clippy_lints/src/needless_continue.rs index 55208ae708b9..d1d0d31ed91a 100644 --- a/clippy_lints/src/needless_continue.rs +++ b/clippy_lints/src/needless_continue.rs @@ -1,9 +1,9 @@ -use clippy_utils::diagnostics::span_lint_and_help; +use clippy_utils::diagnostics::span_lint_hir_and_then; use clippy_utils::higher; use clippy_utils::source::{indent_of, snippet_block, snippet_with_context}; use rustc_ast::Label; use rustc_errors::Applicability; -use rustc_hir::{Block, Expr, ExprKind, LoopSource, StmtKind}; +use rustc_hir::{Block, Expr, ExprKind, HirId, LoopSource, StmtKind}; use rustc_lint::{LateContext, LateLintPass, LintContext}; use rustc_session::declare_lint_pass; use rustc_span::{ExpnKind, Span}; @@ -336,14 +336,9 @@ fn emit_warning(cx: &LateContext<'_>, data: &LintData<'_>, header: &str, typ: Li data.if_expr, ), }; - span_lint_and_help( - cx, - NEEDLESS_CONTINUE, - expr.span, - message, - None, - format!("{header}\n{snip}"), - ); + span_lint_hir_and_then(cx, NEEDLESS_CONTINUE, expr.hir_id, expr.span, message, |diag| { + diag.help(format!("{header}\n{snip}")); + }); } fn suggestion_snippet_for_continue_inside_if(cx: &LateContext<'_>, data: &LintData<'_>) -> String { @@ -424,11 +419,11 @@ fn suggestion_snippet_for_continue_inside_else(cx: &LateContext<'_>, data: &Lint fn check_last_stmt_in_expr(cx: &LateContext<'_>, inner_expr: &Expr<'_>, func: &F) where - F: Fn(Option<&Label>, Span), + F: Fn(HirId, Option<&Label>, Span), { match inner_expr.kind { ExprKind::Continue(continue_label) => { - func(continue_label.label.as_ref(), inner_expr.span); + func(inner_expr.hir_id, continue_label.label.as_ref(), inner_expr.span); }, ExprKind::If(_, then_block, else_block) if let ExprKind::Block(then_block, _) = then_block.kind => { check_last_stmt_in_block(cx, then_block, func); @@ -454,7 +449,7 @@ where fn check_last_stmt_in_block(cx: &LateContext<'_>, b: &Block<'_>, func: &F) where - F: Fn(Option<&Label>, Span), + F: Fn(HirId, Option<&Label>, Span), { if let Some(expr) = b.expr { check_last_stmt_in_expr(cx, expr, func); @@ -470,15 +465,17 @@ where fn check_and_warn(cx: &LateContext<'_>, expr: &Expr<'_>) { with_loop_block(expr, |loop_block, label| { - let p = |continue_label: Option<&Label>, span: Span| { + let p = |continue_hir_id, continue_label: Option<&Label>, span: Span| { if compare_labels(label, continue_label) { - span_lint_and_help( + span_lint_hir_and_then( cx, NEEDLESS_CONTINUE, + continue_hir_id, span, MSG_REDUNDANT_CONTINUE_EXPRESSION, - None, - DROP_CONTINUE_EXPRESSION_MSG, + |diag| { + diag.help(DROP_CONTINUE_EXPRESSION_MSG); + }, ); } }; diff --git a/tests/ui/needless_continue.rs b/tests/ui/needless_continue.rs index 88b7905e7fa8..3275dddf1a0f 100644 --- a/tests/ui/needless_continue.rs +++ b/tests/ui/needless_continue.rs @@ -342,3 +342,38 @@ fn issue15548() { } } } + +fn issue16256() { + fn some_condition() -> bool { + true + } + fn another_condition() -> bool { + true + } + + for _ in 0..5 { + if some_condition() { + // ... + continue; + } + + if another_condition() { + // ... + // "this `continue` expression is redundant" is posted on + // the `continue` node. + #[expect(clippy::needless_continue)] + continue; + } + } + + for _ in 0..5 { + // "This `else` block is redundant" is posted on the + // `else` node. + #[expect(clippy::needless_continue)] + if some_condition() { + // ... + } else { + continue; + } + } +} From 9bfc9a4778d41831157e2af5063e38542960ece0 Mon Sep 17 00:00:00 2001 From: WaterWhisperer Date: Thu, 18 Dec 2025 23:29:54 +0800 Subject: [PATCH 0059/1843] fix: `main_recursion` enable lint in no_std crates and fix broken tests --- tests/ui/crate_level_checks/entrypoint_recursion.rs | 7 +++---- .../crate_level_checks/entrypoint_recursion.stderr | 12 ++++++++++++ .../ui/crate_level_checks/no_std_main_recursion.rs | 13 +++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 tests/ui/crate_level_checks/entrypoint_recursion.stderr create mode 100644 tests/ui/crate_level_checks/no_std_main_recursion.rs diff --git a/tests/ui/crate_level_checks/entrypoint_recursion.rs b/tests/ui/crate_level_checks/entrypoint_recursion.rs index 3ded902e36b1..84147d8e9c16 100644 --- a/tests/ui/crate_level_checks/entrypoint_recursion.rs +++ b/tests/ui/crate_level_checks/entrypoint_recursion.rs @@ -1,12 +1,11 @@ -//@check-pass //@ignore-target: apple - #![feature(rustc_attrs)] - #[warn(clippy::main_recursion)] #[allow(unconditional_recursion)] #[rustc_main] fn a() { - println!("Hello, World!"); a(); + //~^ main_recursion } + +fn main() {} diff --git a/tests/ui/crate_level_checks/entrypoint_recursion.stderr b/tests/ui/crate_level_checks/entrypoint_recursion.stderr new file mode 100644 index 000000000000..d9f50d2dfc4b --- /dev/null +++ b/tests/ui/crate_level_checks/entrypoint_recursion.stderr @@ -0,0 +1,12 @@ +error: recursing into entrypoint `a` + --> tests/ui/crate_level_checks/entrypoint_recursion.rs:7:5 + | +LL | a(); + | ^ + | + = help: consider using another function for this recursion + = note: `-D clippy::main-recursion` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::main_recursion)]` + +error: aborting due to 1 previous error + diff --git a/tests/ui/crate_level_checks/no_std_main_recursion.rs b/tests/ui/crate_level_checks/no_std_main_recursion.rs new file mode 100644 index 000000000000..74763d67dd78 --- /dev/null +++ b/tests/ui/crate_level_checks/no_std_main_recursion.rs @@ -0,0 +1,13 @@ +//@check-pass +//@compile-flags: -Cpanic=abort +#![no_std] +#[warn(clippy::main_recursion)] +#[allow(unconditional_recursion)] +fn main() { + main(); +} + +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + loop {} +} From 7b7ee8463f0b21e7cf2c6dc57ea8c026bc2f5829 Mon Sep 17 00:00:00 2001 From: Redddy Date: Sun, 21 Dec 2025 00:26:51 +0900 Subject: [PATCH 0060/1843] Add title field to ICE issue template --- .github/ISSUE_TEMPLATE/ice.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/ice.md b/.github/ISSUE_TEMPLATE/ice.md index 2afcd210a6eb..1ab1ddf46016 100644 --- a/.github/ISSUE_TEMPLATE/ice.md +++ b/.github/ISSUE_TEMPLATE/ice.md @@ -2,6 +2,7 @@ name: Internal Compiler Error about: Create a report for an internal compiler error in rustc. labels: C-bug, I-ICE, T-compiler +title: "[ICE]: " --- tests/ui/needless_collect.rs:228:26 + | +LL | let mut v = iter.collect::>(); + | ^^^^^^^ +... +LL | v.into_iter().map(|x| x + 1).collect() + | ------------- the iterator could be used here instead + | +help: use the original Iterator instead of collecting it and then producing a new one + | +LL ~ +LL | +LL ~ +LL ~ iter.chain([1]).map(|x| x + 1).collect() + | + +error: avoid using `collect()` when not needed + --> tests/ui/needless_collect.rs:240:26 + | +LL | let mut v = iter.collect::>(); + | ^^^^^^^ +... +LL | v.into_iter().map(|x| x + 1).collect() + | ------------- the iterator could be used here instead + | +help: use the original Iterator instead of collecting it and then producing a new one + | +LL ~ +LL | +LL ~ +LL ~ +LL ~ iter.chain([1]).chain([2]).map(|x| x + 1).collect() + | + +error: avoid using `collect()` when not needed + --> tests/ui/needless_collect.rs:248:26 + | +LL | let mut v = iter.collect::>(); + | ^^^^^^^ +... +LL | v.into_iter().map(|x| x + 1).collect() + | ------------- the iterator could be used here instead + | +help: use the original Iterator instead of collecting it and then producing a new one + | +LL ~ +LL | +LL ~ +LL ~ iter.chain([1]).map(|x| x + 1).collect() + | + +error: aborting due to 23 previous errors From e79fef40866bfbf2a4ae33b0030205ce51a7adf9 Mon Sep 17 00:00:00 2001 From: A4-Tacks Date: Thu, 1 Jan 2026 22:21:57 +0800 Subject: [PATCH 0069/1843] Fix incorrect Self path expand for inline_call Example --- ```rust trait Trait { fn f() -> Self; } struct Foo<'a>(&'a ()); impl<'a> Trait for Foo<'a> { fn f() -> Self { Self(&()) } } impl Foo<'_> { fn new() -> Self { Self::$0f() } } ``` **Before this PR** ```rust trait Trait { fn f() -> Self; } struct Foo<'a>(&'a ()); impl<'a> Trait for Foo<'a> { fn f() -> Self { Self(&()) } } impl Foo<'_> { fn new() -> Self { Foo<'a>(&()) } } ``` **After this PR** ```rust trait Trait { fn f() -> Self; } struct Foo<'a>(&'a ()); impl<'a> Trait for Foo<'a> { fn f() -> Self { Self(&()) } } impl Foo<'_> { fn new() -> Self { Foo(&()) } } ``` --- .../ide-assists/src/handlers/inline_call.rs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_call.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_call.rs index fa4f2a78c8b7..21f2249a19c9 100644 --- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_call.rs +++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_call.rs @@ -403,6 +403,12 @@ fn inline( .find(|tok| tok.kind() == SyntaxKind::SELF_TYPE_KW) { let replace_with = t.clone_subtree().syntax().clone_for_update(); + if !is_in_type_path(&self_tok) + && let Some(ty) = ast::Type::cast(replace_with.clone()) + && let Some(generic_arg_list) = ty.generic_arg_list() + { + ted::remove(generic_arg_list.syntax()); + } ted::replace(self_tok, replace_with); } } @@ -588,6 +594,17 @@ fn inline( } } +fn is_in_type_path(self_tok: &syntax::SyntaxToken) -> bool { + self_tok + .parent_ancestors() + .skip_while(|it| !ast::Path::can_cast(it.kind())) + .map_while(ast::Path::cast) + .last() + .and_then(|it| it.syntax().parent()) + .and_then(ast::PathType::cast) + .is_some() +} + fn path_expr_as_record_field(usage: &PathExpr) -> Option { let path = usage.path()?; let name_ref = path.as_single_name_ref()?; @@ -1694,6 +1711,41 @@ fn main() { ) } + #[test] + fn inline_trait_method_call_with_lifetimes() { + check_assist( + inline_call, + r#" +trait Trait { + fn f() -> Self; +} +struct Foo<'a>(&'a ()); +impl<'a> Trait for Foo<'a> { + fn f() -> Self { Self(&()) } +} +impl Foo<'_> { + fn new() -> Self { + Self::$0f() + } +} +"#, + r#" +trait Trait { + fn f() -> Self; +} +struct Foo<'a>(&'a ()); +impl<'a> Trait for Foo<'a> { + fn f() -> Self { Self(&()) } +} +impl Foo<'_> { + fn new() -> Self { + Foo(&()) + } +} +"#, + ) + } + #[test] fn method_by_reborrow() { check_assist( From 6417c8faaedca073da26585187f911946d094d7c Mon Sep 17 00:00:00 2001 From: Linshu Yang Date: Sun, 28 Dec 2025 22:26:15 +0000 Subject: [PATCH 0070/1843] Extend `needless_collect` to also cover `extend` --- clippy_lints/src/methods/needless_collect.rs | 89 +++++++++++++++----- tests/ui/needless_collect.fixed | 9 +- tests/ui/needless_collect.rs | 7 ++ tests/ui/needless_collect.stderr | 21 ++++- 4 files changed, 104 insertions(+), 22 deletions(-) diff --git a/clippy_lints/src/methods/needless_collect.rs b/clippy_lints/src/methods/needless_collect.rs index 312133689900..4992cdf615fa 100644 --- a/clippy_lints/src/methods/needless_collect.rs +++ b/clippy_lints/src/methods/needless_collect.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::ops::ControlFlow; use super::NEEDLESS_COLLECT; @@ -144,7 +145,11 @@ pub(super) fn check<'tcx>( iter_replacement.push_str(&iter_call.get_iter_method(cx)); let mut remove_suggestions = vec![(l.span, String::new())]; - remove_suggestions.extend(extra_calls.iter().map(|extra| (extra.span, String::new()))); + remove_suggestions.extend( + extra_calls + .iter() + .flat_map(|extra| extra.span().map(|s| (s, String::new()))), + ); remove_suggestions.push((iter_call.span, iter_replacement)); diag.multipart_suggestion( @@ -325,29 +330,48 @@ enum IterFunctionKind { Contains(Span), } -struct ExtraFunction { - kind: ExtraFunctionKind, - span: Span, +struct ExtraFunctionSpan { + /// Span of the function call + func_span: Span, + /// Span of the argument + arg_span: Span, +} + +enum ExtraFunction { + Push(Vec), + Extend(ExtraFunctionSpan), } impl ExtraFunction { fn get_iter_method(&self, cx: &LateContext<'_>) -> String { - match &self.kind { - ExtraFunctionKind::Push(span) => { - let s = snippet(cx, *span, ".."); + match &self { + ExtraFunction::Push(spans) => { + let s = spans + .iter() + .map(|span| snippet(cx, span.arg_span, "..")) + .intersperse(Cow::Borrowed(", ")) + .collect::(); format!(".chain([{s}])") }, + ExtraFunction::Extend(span) => { + let s = snippet(cx, span.arg_span, ".."); + format!(".chain({s})") + }, } } -} -enum ExtraFunctionKind { - Push(Span), + fn span(&self) -> Box + '_> { + match &self { + ExtraFunction::Push(spans) => Box::new(spans.iter().map(|s| s.func_span)), + ExtraFunction::Extend(span) => Box::new(std::iter::once(span.func_span)), + } + } } #[derive(Clone, Copy)] struct ExtraFunctionSpec { push_symbol: Option, + extend_symbol: Option, } impl ExtraFunctionSpec { @@ -355,14 +379,23 @@ impl ExtraFunctionSpec { match target { sym::Vec => Some(ExtraFunctionSpec { push_symbol: Some(sym::push), + extend_symbol: Some(sym::extend), }), sym::VecDeque | sym::LinkedList => Some(ExtraFunctionSpec { push_symbol: Some(sym::push_back), + extend_symbol: Some(sym::extend), + }), + sym::BinaryHeap => Some(ExtraFunctionSpec { + push_symbol: None, + extend_symbol: None, }), - sym::BinaryHeap => Some(ExtraFunctionSpec { push_symbol: None }), _ => None, } } + + fn is_extra_function(self, name: Symbol) -> bool { + self.push_symbol == Some(name) || self.extend_symbol == Some(name) + } } struct IterFunctionVisitor<'a, 'tcx> { @@ -397,6 +430,7 @@ impl<'tcx> Visitor<'tcx> for IterFunctionVisitor<'_, 'tcx> { } } + #[expect(clippy::too_many_lines)] fn visit_expr(&mut self, expr: &'tcx Expr<'tcx>) { // Check function calls on our collection if let ExprKind::MethodCall(method_name, recv, args, _) = &expr.kind { @@ -446,10 +480,30 @@ impl<'tcx> Visitor<'tcx> for IterFunctionVisitor<'_, 'tcx> { if let Node::Stmt(stmt) = self.cx.tcx.parent_hir_node(expr.hir_id) { span = stmt.span; } - self.extras.push(ExtraFunction { - kind: ExtraFunctionKind::Push(args[0].span), - span, - }); + if let Some(ExtraFunction::Push(spans)) = self.extras.last_mut() { + spans.push(ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + }); + } else { + self.extras.push(ExtraFunction::Push(vec![ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + }])); + } + }, + name if self.extra_spec.extend_symbol.is_some_and(|sym| name == sym) + && self.uses.is_empty() => + { + let mut span = expr.span; + // Remove the statement span if possible + if let Node::Stmt(stmt) = self.cx.tcx.parent_hir_node(expr.hir_id) { + span = stmt.span; + } + self.extras.push(ExtraFunction::Extend(ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + })); }, _ => { self.seen_other = true; @@ -551,10 +605,7 @@ impl<'tcx> Visitor<'tcx> for UsedCountVisitor<'_, 'tcx> { let parent = self.cx.tcx.parent_hir_node(expr.hir_id); if let Node::Expr(expr) = parent && let ExprKind::MethodCall(method_name, _, _, _) = &expr.kind - && self - .extra_spec - .push_symbol - .is_some_and(|sym| method_name.ident.name == sym) + && self.extra_spec.is_extra_function(method_name.ident.name) { return; } diff --git a/tests/ui/needless_collect.fixed b/tests/ui/needless_collect.fixed index fff383fb1328..e24c752870cc 100644 --- a/tests/ui/needless_collect.fixed +++ b/tests/ui/needless_collect.fixed @@ -241,7 +241,7 @@ mod collect_push_then_iter { //~^ needless_collect - iter.chain([1]).chain([2]).map(|x| x + 1).collect() + iter.chain([1, 2]).map(|x| x + 1).collect() } fn linked_list_push(iter: impl Iterator) -> LinkedList { @@ -263,4 +263,11 @@ mod collect_push_then_iter { v.push(1); ok } + + fn linked_list_extend(iter: impl Iterator, s: Vec) -> LinkedList { + + //~^ needless_collect + + iter.chain(s).map(|x| x + 1).collect() + } } diff --git a/tests/ui/needless_collect.rs b/tests/ui/needless_collect.rs index f6e966b09ea5..7666d5e33fdb 100644 --- a/tests/ui/needless_collect.rs +++ b/tests/ui/needless_collect.rs @@ -263,4 +263,11 @@ mod collect_push_then_iter { v.push(1); ok } + + fn linked_list_extend(iter: impl Iterator, s: Vec) -> LinkedList { + let mut ll = iter.collect::>(); + //~^ needless_collect + ll.extend(s); + ll.into_iter().map(|x| x + 1).collect() + } } diff --git a/tests/ui/needless_collect.stderr b/tests/ui/needless_collect.stderr index 93ca32a68d6a..0decd8cd136a 100644 --- a/tests/ui/needless_collect.stderr +++ b/tests/ui/needless_collect.stderr @@ -153,7 +153,7 @@ LL ~ LL | LL ~ LL ~ -LL ~ iter.chain([1]).chain([2]).map(|x| x + 1).collect() +LL ~ iter.chain([1, 2]).map(|x| x + 1).collect() | error: avoid using `collect()` when not needed @@ -173,5 +173,22 @@ LL ~ LL ~ iter.chain([1]).map(|x| x + 1).collect() | -error: aborting due to 23 previous errors +error: avoid using `collect()` when not needed + --> tests/ui/needless_collect.rs:268:27 + | +LL | let mut ll = iter.collect::>(); + | ^^^^^^^ +... +LL | ll.into_iter().map(|x| x + 1).collect() + | -------------- the iterator could be used here instead + | +help: use the original Iterator instead of collecting it and then producing a new one + | +LL ~ +LL | +LL ~ +LL ~ iter.chain(s).map(|x| x + 1).collect() + | + +error: aborting due to 24 previous errors From 0127720eb8587a3ffecc293020b0faa79c9b8199 Mon Sep 17 00:00:00 2001 From: Linshu Yang Date: Fri, 2 Jan 2026 01:26:15 +0000 Subject: [PATCH 0071/1843] Extend `needless_collect` to cover `push_front` --- clippy_lints/src/methods/needless_collect.rs | 128 ++++++++++++++----- clippy_utils/src/sym.rs | 1 + tests/ui/needless_collect.fixed | 26 +++- tests/ui/needless_collect.rs | 26 +++- tests/ui/needless_collect.stderr | 43 ++++++- 5 files changed, 187 insertions(+), 37 deletions(-) diff --git a/clippy_lints/src/methods/needless_collect.rs b/clippy_lints/src/methods/needless_collect.rs index 4992cdf615fa..6bdb40e46b38 100644 --- a/clippy_lints/src/methods/needless_collect.rs +++ b/clippy_lints/src/methods/needless_collect.rs @@ -141,7 +141,9 @@ pub(super) fn check<'tcx>( |diag| { let iter_snippet = Sugg::hir(cx, iter_expr, ".."); let mut iter_replacement = iter_snippet.to_string(); - iter_replacement.extend(extra_calls.iter().map(|extra| extra.get_iter_method(cx))); + for extra in &extra_calls { + iter_replacement = extra.apply_iter_method(cx, &iter_replacement); + } iter_replacement.push_str(&iter_call.get_iter_method(cx)); let mut remove_suggestions = vec![(l.span, String::new())]; @@ -338,39 +340,62 @@ struct ExtraFunctionSpan { } enum ExtraFunction { - Push(Vec), + Push { + back: Vec, + front: Vec, + }, Extend(ExtraFunctionSpan), } impl ExtraFunction { - fn get_iter_method(&self, cx: &LateContext<'_>) -> String { + fn apply_iter_method(&self, cx: &LateContext<'_>, inner: &str) -> String { match &self { - ExtraFunction::Push(spans) => { - let s = spans + ExtraFunction::Push { back, front } => { + let back_sugg = back .iter() .map(|span| snippet(cx, span.arg_span, "..")) .intersperse(Cow::Borrowed(", ")) .collect::(); - format!(".chain([{s}])") + let front = front + .iter() + .map(|span| snippet(cx, span.arg_span, "..")) + .intersperse(Cow::Borrowed(", ")) + .collect::(); + match (front.is_empty(), back_sugg.is_empty()) { + (true, true) => inner.to_string(), + (true, false) => format!("{inner}.chain([{back_sugg}])"), + (false, true) => format!("[{front}].into_iter().chain({inner})"), + (false, false) => format!("[{front}].into_iter().chain({inner}).chain([{back_sugg}])"), + } }, ExtraFunction::Extend(span) => { let s = snippet(cx, span.arg_span, ".."); - format!(".chain({s})") + format!("{inner}.chain({s})") }, } } fn span(&self) -> Box + '_> { match &self { - ExtraFunction::Push(spans) => Box::new(spans.iter().map(|s| s.func_span)), + ExtraFunction::Push { back, front } => Box::new( + back.iter() + .map(|s| s.func_span) + .chain(front.iter().map(|s| s.func_span)), + ), ExtraFunction::Extend(span) => Box::new(std::iter::once(span.func_span)), } } } +#[derive(Clone, Copy)] +struct ExtraFunctionPushSpec { + back: Option, + front: Option, +} + #[derive(Clone, Copy)] struct ExtraFunctionSpec { - push_symbol: Option, + push_symbol: ExtraFunctionPushSpec, extend_symbol: Option, } @@ -378,15 +403,24 @@ impl ExtraFunctionSpec { fn new(target: Symbol) -> Option { match target { sym::Vec => Some(ExtraFunctionSpec { - push_symbol: Some(sym::push), + push_symbol: ExtraFunctionPushSpec { + back: Some(sym::push), + front: None, + }, extend_symbol: Some(sym::extend), }), sym::VecDeque | sym::LinkedList => Some(ExtraFunctionSpec { - push_symbol: Some(sym::push_back), + push_symbol: ExtraFunctionPushSpec { + back: Some(sym::push_back), + front: Some(sym::push_front), + }, extend_symbol: Some(sym::extend), }), sym::BinaryHeap => Some(ExtraFunctionSpec { - push_symbol: None, + push_symbol: ExtraFunctionPushSpec { + back: None, + front: None, + }, extend_symbol: None, }), _ => None, @@ -394,7 +428,7 @@ impl ExtraFunctionSpec { } fn is_extra_function(self, name: Symbol) -> bool { - self.push_symbol == Some(name) || self.extend_symbol == Some(name) + self.push_symbol.back == Some(name) || self.push_symbol.front == Some(name) || self.extend_symbol == Some(name) } } @@ -474,32 +508,48 @@ impl<'tcx> Visitor<'tcx> for IterFunctionVisitor<'_, 'tcx> { func: IterFunctionKind::Contains(args[0].span), span: expr.span, })), - name if Some(name) == self.extra_spec.push_symbol && self.uses.is_empty() => { - let mut span = expr.span; - // Remove the statement span if possible - if let Node::Stmt(stmt) = self.cx.tcx.parent_hir_node(expr.hir_id) { - span = stmt.span; - } - if let Some(ExtraFunction::Push(spans)) = self.extras.last_mut() { - spans.push(ExtraFunctionSpan { - func_span: span, - arg_span: args[0].span, - }); - } else { - self.extras.push(ExtraFunction::Push(vec![ExtraFunctionSpan { - func_span: span, - arg_span: args[0].span, - }])); + name if let is_push_back = self.extra_spec.push_symbol.back.is_some_and(|sym| name == sym) + && (is_push_back || self.extra_spec.push_symbol.front.is_some_and(|sym| name == sym)) + && self.uses.is_empty() => + { + let span = get_span_of_expr_or_parent_stmt(self.cx, expr); + match self.extras.last_mut() { + Some(ExtraFunction::Push { back, .. }) if is_push_back => { + back.push(ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + }); + }, + Some(ExtraFunction::Push { front, .. }) => { + front.push(ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + }); + }, + _ if is_push_back => { + self.extras.push(ExtraFunction::Push { + back: vec![ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + }], + front: Vec::new(), + }); + }, + _ => { + self.extras.push(ExtraFunction::Push { + back: Vec::new(), + front: vec![ExtraFunctionSpan { + func_span: span, + arg_span: args[0].span, + }], + }); + }, } }, name if self.extra_spec.extend_symbol.is_some_and(|sym| name == sym) && self.uses.is_empty() => { - let mut span = expr.span; - // Remove the statement span if possible - if let Node::Stmt(stmt) = self.cx.tcx.parent_hir_node(expr.hir_id) { - span = stmt.span; - } + let span = get_span_of_expr_or_parent_stmt(self.cx, expr); self.extras.push(ExtraFunction::Extend(ExtraFunctionSpan { func_span: span, arg_span: args[0].span, @@ -542,6 +592,16 @@ impl<'tcx> Visitor<'tcx> for IterFunctionVisitor<'_, 'tcx> { } } +/// If parent of the `expr` is a statement, return the span of the statement, otherwise return the +/// span of the expression. +fn get_span_of_expr_or_parent_stmt<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'tcx>) -> Span { + if let Node::Stmt(stmt) = cx.tcx.parent_hir_node(expr.hir_id) { + stmt.span + } else { + expr.span + } +} + enum LoopKind<'tcx> { Conditional(&'tcx Expr<'tcx>), Loop, diff --git a/clippy_utils/src/sym.rs b/clippy_utils/src/sym.rs index a36f4954a06a..fbd5ce2dc2d1 100644 --- a/clippy_utils/src/sym.rs +++ b/clippy_utils/src/sym.rs @@ -272,6 +272,7 @@ generate! { product, push, push_back, + push_front, push_str, read, read_exact, diff --git a/tests/ui/needless_collect.fixed b/tests/ui/needless_collect.fixed index e24c752870cc..2d59c1d42b05 100644 --- a/tests/ui/needless_collect.fixed +++ b/tests/ui/needless_collect.fixed @@ -222,7 +222,7 @@ fn issue16270() { #[warn(clippy::needless_collect)] mod collect_push_then_iter { - use std::collections::{BinaryHeap, LinkedList}; + use std::collections::{BinaryHeap, LinkedList, VecDeque}; fn vec_push(iter: impl Iterator) -> Vec { @@ -270,4 +270,28 @@ mod collect_push_then_iter { iter.chain(s).map(|x| x + 1).collect() } + + fn deque_push_front(iter: impl Iterator) -> VecDeque { + + //~^ needless_collect + + + [1, 2].into_iter().chain(iter).map(|x| x + 1).collect() + } + + fn linked_list_push_front_mixed( + iter: impl Iterator, + iter2: impl Iterator, + ) -> LinkedList { + + //~^ needless_collect + + + + + + + + [5].into_iter().chain([1, 3].into_iter().chain(iter).chain([2]).chain(iter2)).chain([4, 6]).map(|x| x + 1).collect() + } } diff --git a/tests/ui/needless_collect.rs b/tests/ui/needless_collect.rs index 7666d5e33fdb..346cddd88e70 100644 --- a/tests/ui/needless_collect.rs +++ b/tests/ui/needless_collect.rs @@ -222,7 +222,7 @@ fn issue16270() { #[warn(clippy::needless_collect)] mod collect_push_then_iter { - use std::collections::{BinaryHeap, LinkedList}; + use std::collections::{BinaryHeap, LinkedList, VecDeque}; fn vec_push(iter: impl Iterator) -> Vec { let mut v = iter.collect::>(); @@ -270,4 +270,28 @@ mod collect_push_then_iter { ll.extend(s); ll.into_iter().map(|x| x + 1).collect() } + + fn deque_push_front(iter: impl Iterator) -> VecDeque { + let mut v = iter.collect::>(); + //~^ needless_collect + v.push_front(1); + v.push_front(2); + v.into_iter().map(|x| x + 1).collect() + } + + fn linked_list_push_front_mixed( + iter: impl Iterator, + iter2: impl Iterator, + ) -> LinkedList { + let mut v = iter.collect::>(); + //~^ needless_collect + v.push_front(1); + v.push_back(2); + v.push_front(3); + v.extend(iter2); + v.push_back(4); + v.push_front(5); + v.push_back(6); + v.into_iter().map(|x| x + 1).collect() + } } diff --git a/tests/ui/needless_collect.stderr b/tests/ui/needless_collect.stderr index 0decd8cd136a..b10312224c8e 100644 --- a/tests/ui/needless_collect.stderr +++ b/tests/ui/needless_collect.stderr @@ -190,5 +190,46 @@ LL ~ LL ~ iter.chain(s).map(|x| x + 1).collect() | -error: aborting due to 24 previous errors +error: avoid using `collect()` when not needed + --> tests/ui/needless_collect.rs:275:26 + | +LL | let mut v = iter.collect::>(); + | ^^^^^^^ +... +LL | v.into_iter().map(|x| x + 1).collect() + | ------------- the iterator could be used here instead + | +help: use the original Iterator instead of collecting it and then producing a new one + | +LL ~ +LL | +LL ~ +LL ~ +LL ~ [1, 2].into_iter().chain(iter).map(|x| x + 1).collect() + | + +error: avoid using `collect()` when not needed + --> tests/ui/needless_collect.rs:286:26 + | +LL | let mut v = iter.collect::>(); + | ^^^^^^^ +... +LL | v.into_iter().map(|x| x + 1).collect() + | ------------- the iterator could be used here instead + | +help: use the original Iterator instead of collecting it and then producing a new one + | +LL ~ +LL | +LL ~ +LL ~ +LL ~ +LL ~ +LL ~ +LL ~ +LL ~ +LL ~ [5].into_iter().chain([1, 3].into_iter().chain(iter).chain([2]).chain(iter2)).chain([4, 6]).map(|x| x + 1).collect() + | + +error: aborting due to 26 previous errors From 979704dacdbb0ba839de2c3f73acee891be17a63 Mon Sep 17 00:00:00 2001 From: A4-Tacks Date: Sun, 26 Oct 2025 16:18:50 +0800 Subject: [PATCH 0072/1843] Fix not applicable on statement for convert_to_guarded_return Fix not applicable in statement when exist else block Example --- ```rust fn main() { some_statements(); if$0 let Ok(x) = Err(92) { foo(x); } else { return; } some_statements(); } ``` **Before this PR** Assist not applicable **After this PR** ```rust fn main() { some_statements(); let Ok(x) = Err(92) else { return; }; foo(x); some_statements(); } ``` --- .../src/handlers/convert_to_guarded_return.rs | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/convert_to_guarded_return.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/convert_to_guarded_return.rs index 08b114072fd9..ea5c1637b760 100644 --- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/convert_to_guarded_return.rs +++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/convert_to_guarded_return.rs @@ -95,7 +95,9 @@ fn if_expr_to_guarded_return( let parent_block = if_expr.syntax().parent()?.ancestors().find_map(ast::BlockExpr::cast)?; - if parent_block.tail_expr()? != if_expr.clone().into() { + if parent_block.tail_expr() != Some(if_expr.clone().into()) + && !(else_block.is_some() && ast::ExprStmt::can_cast(if_expr.syntax().parent()?.kind())) + { return None; } @@ -502,6 +504,36 @@ fn main() { ); } + #[test] + fn convert_if_let_has_else_block_in_statement() { + check_assist( + convert_to_guarded_return, + r#" +fn main() { + some_statements(); + if$0 let Ok(x) = Err(92) { + foo(x); + } else { + // needless comment + return; + } + some_statements(); +} +"#, + r#" +fn main() { + some_statements(); + let Ok(x) = Err(92) else { + // needless comment + return; + }; + foo(x); + some_statements(); +} +"#, + ); + } + #[test] fn convert_if_let_result_inside_let() { check_assist( @@ -1136,6 +1168,44 @@ fn main() { ); } + #[test] + fn ignore_else_if() { + check_assist_not_applicable( + convert_to_guarded_return, + r#" +fn main() { + some_statements(); + if cond { + () + } else if$0 let Ok(x) = Err(92) { + foo(x); + } else { + return; + } + some_statements(); +} +"#, + ); + } + + #[test] + fn ignore_if_inside_let() { + check_assist_not_applicable( + convert_to_guarded_return, + r#" +fn main() { + some_statements(); + let _ = if$0 let Ok(x) = Err(92) { + foo(x); + } else { + return; + } + some_statements(); +} +"#, + ); + } + #[test] fn ignore_let_else_branch() { check_assist_not_applicable( From cecc26bf3efb41f27140bdd74f58e71a87a24ec6 Mon Sep 17 00:00:00 2001 From: hulxv Date: Fri, 2 Jan 2026 22:55:05 +0200 Subject: [PATCH 0073/1843] Refactor libc-fs-symlink tests to use errno_result --- .../tests/pass-dep/libc/libc-fs-symlink.rs | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/tools/miri/tests/pass-dep/libc/libc-fs-symlink.rs b/src/tools/miri/tests/pass-dep/libc/libc-fs-symlink.rs index fd7fc801dc28..52a0d978963e 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-fs-symlink.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-fs-symlink.rs @@ -4,11 +4,14 @@ //@compile-flags: -Zmiri-disable-isolation use std::ffi::CString; -use std::io::{Error, ErrorKind}; +use std::io::ErrorKind; use std::os::unix::ffi::OsStrExt; +#[path = "../../utils/libc.rs"] +mod libc_utils; #[path = "../../utils/mod.rs"] mod utils; +use libc_utils::errno_result; fn main() { test_readlink(); @@ -31,44 +34,48 @@ fn test_readlink() { // Make the buf one byte larger than it needs to be, // and check that the last byte is not overwritten. let mut large_buf = vec![0xFF; expected_path.len() + 1]; - let res = - unsafe { libc::readlink(symlink_c_ptr, large_buf.as_mut_ptr().cast(), large_buf.len()) }; + let res = errno_result(unsafe { + libc::readlink(symlink_c_ptr, large_buf.as_mut_ptr().cast(), large_buf.len()) + }) + .unwrap(); // Check that the resolved path was properly written into the buf. assert_eq!(&large_buf[..(large_buf.len() - 1)], expected_path); assert_eq!(large_buf.last(), Some(&0xFF)); - assert_eq!(res, large_buf.len() as isize - 1); + assert_eq!(res, (large_buf.len() - 1) as isize); // Test that the resolved path is truncated if the provided buffer // is too small. let mut small_buf = [0u8; 2]; - let res = - unsafe { libc::readlink(symlink_c_ptr, small_buf.as_mut_ptr().cast(), small_buf.len()) }; + let res = errno_result(unsafe { + libc::readlink(symlink_c_ptr, small_buf.as_mut_ptr().cast(), small_buf.len()) + }) + .unwrap(); assert_eq!(small_buf, &expected_path[..small_buf.len()]); assert_eq!(res, small_buf.len() as isize); // Test that we report a proper error for a missing path. - let res = unsafe { + let err = errno_result(unsafe { libc::readlink( c"MIRI_MISSING_FILE_NAME".as_ptr(), small_buf.as_mut_ptr().cast(), small_buf.len(), ) - }; - assert_eq!(res, -1); - assert_eq!(Error::last_os_error().kind(), ErrorKind::NotFound); + }) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NotFound); } fn test_nofollow_symlink() { - let bytes = b"Hello, World!\n"; - let path = utils::prepare_with_content("test_nofollow_symlink_target.txt", bytes); + let path = utils::prepare_with_content("test_nofollow_symlink_target.txt", b"Hello, World!\n"); let symlink_path = utils::prepare("test_nofollow_symlink.txt"); std::os::unix::fs::symlink(&path, &symlink_path).unwrap(); let symlink_cpath = CString::new(symlink_path.as_os_str().as_bytes()).unwrap(); - let ret = unsafe { libc::open(symlink_cpath.as_ptr(), libc::O_NOFOLLOW | libc::O_CLOEXEC) }; - assert_eq!(ret, -1); - let err = Error::last_os_error().raw_os_error().unwrap(); - assert_eq!(err, libc::ELOOP); + let err = errno_result(unsafe { + libc::open(symlink_cpath.as_ptr(), libc::O_NOFOLLOW | libc::O_CLOEXEC) + }) + .unwrap_err(); + assert_eq!(err.raw_os_error(), Some(libc::ELOOP)); } From 69c7a7e40509575840cba379c32dd4d8b216519b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= Date: Thu, 1 Jan 2026 22:47:41 +0100 Subject: [PATCH 0074/1843] Use a generic type to implement SIMD types in `core_arch::simd` This allows the `types` macro to easily implement `From>` and `Into>` --- library/stdarch/crates/core_arch/rustfmt.toml | 3 - library/stdarch/crates/core_arch/src/lib.rs | 5 +- .../stdarch/crates/core_arch/src/macros.rs | 16 + .../crates/core_arch/src/powerpc/altivec.rs | 2 - .../crates/core_arch/src/powerpc/macros.rs | 20 -- .../crates/core_arch/src/s390x/macros.rs | 20 -- .../crates/core_arch/src/s390x/vector.rs | 2 - library/stdarch/crates/core_arch/src/simd.rs | 306 ++++++++++++------ 8 files changed, 219 insertions(+), 155 deletions(-) diff --git a/library/stdarch/crates/core_arch/rustfmt.toml b/library/stdarch/crates/core_arch/rustfmt.toml index 4ae742ba8d50..e69de29bb2d1 100644 --- a/library/stdarch/crates/core_arch/rustfmt.toml +++ b/library/stdarch/crates/core_arch/rustfmt.toml @@ -1,3 +0,0 @@ -ignore = [ - "src/simd.rs", -] diff --git a/library/stdarch/crates/core_arch/src/lib.rs b/library/stdarch/crates/core_arch/src/lib.rs index 5bedefe42d6e..d69089b95b0c 100644 --- a/library/stdarch/crates/core_arch/src/lib.rs +++ b/library/stdarch/crates/core_arch/src/lib.rs @@ -37,7 +37,8 @@ avx10_target_feature, const_trait_impl, const_cmp, - const_eval_select + const_eval_select, + maybe_uninit_as_bytes )] #![cfg_attr(test, feature(test, abi_vectorcall, stdarch_internal))] #![deny(clippy::missing_inline_in_public_items)] @@ -87,4 +88,4 @@ pub mod arch { } #[allow(unused_imports)] -use core::{array, convert, ffi, fmt, hint, intrinsics, marker, mem, ops, ptr, sync}; +use core::{array, cmp, convert, ffi, fmt, hint, intrinsics, marker, mem, ops, ptr, sync}; diff --git a/library/stdarch/crates/core_arch/src/macros.rs b/library/stdarch/crates/core_arch/src/macros.rs index 1030d7e9740a..849297b1fc36 100644 --- a/library/stdarch/crates/core_arch/src/macros.rs +++ b/library/stdarch/crates/core_arch/src/macros.rs @@ -135,6 +135,22 @@ macro_rules! types { crate::core_arch::simd::debug_simd_finish(f, stringify!($name), self.as_array()) } } + + $(#[$stability])+ + impl crate::convert::From> for $name { + #[inline(always)] + fn from(simd: crate::core_arch::simd::Simd<$elem_type, $len>) -> Self { + unsafe { crate::mem::transmute(simd) } + } + } + + $(#[$stability])+ + impl crate::convert::From<$name> for crate::core_arch::simd::Simd<$elem_type, $len> { + #[inline(always)] + fn from(simd: $name) -> Self { + unsafe { crate::mem::transmute(simd) } + } + } )*); } diff --git a/library/stdarch/crates/core_arch/src/powerpc/altivec.rs b/library/stdarch/crates/core_arch/src/powerpc/altivec.rs index a7bbf35ed8d0..37b557c3b2af 100644 --- a/library/stdarch/crates/core_arch/src/powerpc/altivec.rs +++ b/library/stdarch/crates/core_arch/src/powerpc/altivec.rs @@ -364,8 +364,6 @@ unsafe extern "C" { fn vrfin(a: vector_float) -> vector_float; } -impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, f32x4 } - impl_neg! { i8x16 : 0 } impl_neg! { i16x8 : 0 } impl_neg! { i32x4 : 0 } diff --git a/library/stdarch/crates/core_arch/src/powerpc/macros.rs b/library/stdarch/crates/core_arch/src/powerpc/macros.rs index 24d86f1018c8..6097ca31adc9 100644 --- a/library/stdarch/crates/core_arch/src/powerpc/macros.rs +++ b/library/stdarch/crates/core_arch/src/powerpc/macros.rs @@ -274,25 +274,6 @@ macro_rules! t_b { }; } -macro_rules! impl_from { - ($s: ident) => { - #[unstable(feature = "stdarch_powerpc", issue = "111145")] - impl From<$s> for s_t_l!($s) { - #[inline] - fn from (v: $s) -> Self { - unsafe { - transmute(v) - } - } - } - }; - ($($s: ident),*) => { - $( - impl_from! { $s } - )* - }; -} - macro_rules! impl_neg { ($s: ident : $zero: expr) => { #[unstable(feature = "stdarch_powerpc", issue = "111145")] @@ -306,7 +287,6 @@ macro_rules! impl_neg { }; } -pub(crate) use impl_from; pub(crate) use impl_neg; pub(crate) use impl_vec_trait; pub(crate) use s_t_l; diff --git a/library/stdarch/crates/core_arch/src/s390x/macros.rs b/library/stdarch/crates/core_arch/src/s390x/macros.rs index 26afbaa45a74..b72560396c43 100644 --- a/library/stdarch/crates/core_arch/src/s390x/macros.rs +++ b/library/stdarch/crates/core_arch/src/s390x/macros.rs @@ -431,25 +431,6 @@ macro_rules! t_b { }; } -macro_rules! impl_from { - ($s: ident) => { - #[unstable(feature = "stdarch_s390x", issue = "135681")] - impl From<$s> for s_t_l!($s) { - #[inline] - fn from (v: $s) -> Self { - unsafe { - transmute(v) - } - } - } - }; - ($($s: ident),*) => { - $( - impl_from! { $s } - )* - }; -} - macro_rules! impl_neg { ($s: ident : $zero: expr) => { #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -463,7 +444,6 @@ macro_rules! impl_neg { }; } -pub(crate) use impl_from; pub(crate) use impl_neg; pub(crate) use impl_vec_trait; pub(crate) use l_t_t; diff --git a/library/stdarch/crates/core_arch/src/s390x/vector.rs b/library/stdarch/crates/core_arch/src/s390x/vector.rs index 1366b5c78273..d63f37eeb9a8 100644 --- a/library/stdarch/crates/core_arch/src/s390x/vector.rs +++ b/library/stdarch/crates/core_arch/src/s390x/vector.rs @@ -283,8 +283,6 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple; } -impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } - impl_neg! { i8x16 : 0 } impl_neg! { i16x8 : 0 } impl_neg! { i32x4 : 0 } diff --git a/library/stdarch/crates/core_arch/src/simd.rs b/library/stdarch/crates/core_arch/src/simd.rs index b9dbde75971d..926163ac4cad 100644 --- a/library/stdarch/crates/core_arch/src/simd.rs +++ b/library/stdarch/crates/core_arch/src/simd.rs @@ -16,129 +16,223 @@ pub(crate) const unsafe fn simd_imin(a: T, b: T) -> T { crate::intrinsics::simd::simd_select(mask, a, b) } +/// SAFETY: All bits patterns must be valid +pub(crate) unsafe trait SimdElement: + Copy + const PartialEq + crate::fmt::Debug +{ +} + +unsafe impl SimdElement for u8 {} +unsafe impl SimdElement for u16 {} +unsafe impl SimdElement for u32 {} +unsafe impl SimdElement for u64 {} + +unsafe impl SimdElement for i8 {} +unsafe impl SimdElement for i16 {} +unsafe impl SimdElement for i32 {} +unsafe impl SimdElement for i64 {} + +unsafe impl SimdElement for f16 {} +unsafe impl SimdElement for f32 {} +unsafe impl SimdElement for f64 {} + +#[repr(simd)] +#[derive(Copy)] +pub(crate) struct Simd([T; N]); + +impl Simd { + /// A value of this type where all elements are zeroed out. + // SAFETY: `T` implements `SimdElement`, so it is zeroable. + pub(crate) const ZERO: Self = unsafe { crate::mem::zeroed() }; + + #[inline(always)] + pub(crate) const fn from_array(elements: [T; N]) -> Self { + Self(elements) + } + + // FIXME: Workaround rust@60637 + #[inline(always)] + #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] + pub(crate) const fn splat(value: T) -> Self { + let one = Simd([value]); + // SAFETY: 0 is always in-bounds because we're shuffling + // a simd type with exactly one element. + unsafe { simd_shuffle!(one, one, [0; N]) } + } + + /// Extract the element at position `index`. + /// `index` is not a constant so this is not efficient! + /// Use for testing only. + // FIXME: Workaround rust@60637 + #[inline(always)] + pub(crate) const fn extract(&self, index: usize) -> T { + self.as_array()[index] + } + + #[inline] + pub(crate) const fn as_array(&self) -> &[T; N] { + let simd_ptr: *const Self = self; + let array_ptr: *const [T; N] = simd_ptr.cast(); + // SAFETY: We can always read the prefix of a simd type as an array. + // There might be more padding afterwards for some widths, but + // that's not a problem for reading less than that. + unsafe { &*array_ptr } + } +} + +// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::Simd is banned by MCP#838" +impl Clone for Simd { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] +impl const crate::cmp::PartialEq for Simd { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.as_array() == other.as_array() + } +} + +impl crate::fmt::Debug for Simd { + #[inline] + fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result { + debug_simd_finish(f, "Simd", self.as_array()) + } +} + +impl Simd { + #[inline] + pub(crate) const fn to_bits(self) -> Simd { + assert!(size_of::() == size_of::>()); + unsafe { crate::mem::transmute_copy(&self) } + } + + #[inline] + pub(crate) const fn from_bits(bits: Simd) -> Self { + assert!(size_of::() == size_of::>()); + unsafe { crate::mem::transmute_copy(&bits) } + } +} + +impl Simd { + #[inline] + pub(crate) const fn to_bits(self) -> Simd { + assert!(size_of::() == size_of::>()); + unsafe { crate::mem::transmute_copy(&self) } + } + + #[inline] + pub(crate) const fn from_bits(bits: Simd) -> Self { + assert!(size_of::() == size_of::>()); + unsafe { crate::mem::transmute_copy(&bits) } + } +} + +impl Simd { + #[inline] + pub(crate) const fn to_bits(self) -> Simd { + assert!(size_of::() == size_of::>()); + unsafe { crate::mem::transmute_copy(&self) } + } + + #[inline] + pub(crate) const fn from_bits(bits: Simd) -> Self { + assert!(size_of::() == size_of::>()); + unsafe { crate::mem::transmute_copy(&bits) } + } +} + macro_rules! simd_ty { ($id:ident [$elem_type:ty ; $len:literal]: $($param_name:ident),*) => { - #[repr(simd)] - #[derive(Copy, Clone)] - pub(crate) struct $id([$elem_type; $len]); + pub(crate) type $id = Simd<$elem_type, $len>; - #[allow(clippy::use_self)] impl $id { - /// A value of this type where all elements are zeroed out. - pub(crate) const ZERO: Self = unsafe { crate::mem::zeroed() }; - #[inline(always)] pub(crate) const fn new($($param_name: $elem_type),*) -> Self { - $id([$($param_name),*]) - } - #[inline(always)] - pub(crate) const fn from_array(elements: [$elem_type; $len]) -> Self { - $id(elements) - } - // FIXME: Workaround rust@60637 - #[inline(always)] - #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] - pub(crate) const fn splat(value: $elem_type) -> Self { - #[derive(Copy, Clone)] - #[repr(simd)] - struct JustOne([$elem_type; 1]); - let one = JustOne([value]); - // SAFETY: 0 is always in-bounds because we're shuffling - // a simd type with exactly one element. - unsafe { simd_shuffle!(one, one, [0; $len]) } - } - - /// Extract the element at position `index`. - /// `index` is not a constant so this is not efficient! - /// Use for testing only. - // FIXME: Workaround rust@60637 - #[inline(always)] - pub(crate) const fn extract(&self, index: usize) -> $elem_type { - self.as_array()[index] - } - - #[inline] - pub(crate) const fn as_array(&self) -> &[$elem_type; $len] { - let simd_ptr: *const Self = self; - let array_ptr: *const [$elem_type; $len] = simd_ptr.cast(); - // SAFETY: We can always read the prefix of a simd type as an array. - // There might be more padding afterwards for some widths, but - // that's not a problem for reading less than that. - unsafe { &*array_ptr } - } - } - - #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] - const impl core::cmp::PartialEq for $id { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.as_array() == other.as_array() - } - } - - impl core::fmt::Debug for $id { - #[inline] - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - debug_simd_finish(f, stringify!($id), self.as_array()) + Self([$($param_name),*]) } } } } +#[repr(simd)] +#[derive(Copy)] +pub(crate) struct SimdM([T; N]); + +impl SimdM { + #[inline(always)] + const fn bool_to_internal(x: bool) -> T { + // SAFETY: `T` implements `SimdElement`, so all bit patterns are valid. + let zeros = const { unsafe { crate::mem::zeroed::() } }; + let ones = const { + // Ideally, this would be `transmute([0xFFu8; size_of::()])`, but + // `size_of::()` is not allowed to use a generic parameter there. + let mut r = crate::mem::MaybeUninit::::uninit(); + let mut i = 0; + while i < crate::mem::size_of::() { + r.as_bytes_mut()[i] = crate::mem::MaybeUninit::new(0xFF); + i += 1; + } + unsafe { r.assume_init() } + }; + [zeros, ones][x as usize] + } + + // FIXME: Workaround rust@60637 + #[inline(always)] + #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] + pub(crate) const fn splat(value: bool) -> Self { + let one = SimdM([Self::bool_to_internal(value)]); + // SAFETY: 0 is always in-bounds because we're shuffling + // a simd type with exactly one element. + unsafe { simd_shuffle!(one, one, [0; N]) } + } + + #[inline] + pub(crate) const fn as_array(&self) -> &[T; N] { + let simd_ptr: *const Self = self; + let array_ptr: *const [T; N] = simd_ptr.cast(); + // SAFETY: We can always read the prefix of a simd type as an array. + // There might be more padding afterwards for some widths, but + // that's not a problem for reading less than that. + unsafe { &*array_ptr } + } +} + +// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::SimdM is banned by MCP#838" +impl Clone for SimdM { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] +impl const crate::cmp::PartialEq for SimdM { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.as_array() == other.as_array() + } +} + +impl crate::fmt::Debug for SimdM { + #[inline] + fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result { + debug_simd_finish(f, "SimdM", self.as_array()) + } +} + macro_rules! simd_m_ty { ($id:ident [$elem_type:ident ; $len:literal]: $($param_name:ident),*) => { - #[repr(simd)] - #[derive(Copy, Clone)] - pub(crate) struct $id([$elem_type; $len]); + pub(crate) type $id = SimdM<$elem_type, $len>; - #[allow(clippy::use_self)] impl $id { - #[inline(always)] - const fn bool_to_internal(x: bool) -> $elem_type { - [0 as $elem_type, !(0 as $elem_type)][x as usize] - } - #[inline(always)] pub(crate) const fn new($($param_name: bool),*) -> Self { - $id([$(Self::bool_to_internal($param_name)),*]) - } - - // FIXME: Workaround rust@60637 - #[inline(always)] - #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] - pub(crate) const fn splat(value: bool) -> Self { - #[derive(Copy, Clone)] - #[repr(simd)] - struct JustOne([$elem_type; 1]); - let one = JustOne([Self::bool_to_internal(value)]); - // SAFETY: 0 is always in-bounds because we're shuffling - // a simd type with exactly one element. - unsafe { simd_shuffle!(one, one, [0; $len]) } - } - - #[inline] - pub(crate) const fn as_array(&self) -> &[$elem_type; $len] { - let simd_ptr: *const Self = self; - let array_ptr: *const [$elem_type; $len] = simd_ptr.cast(); - // SAFETY: We can always read the prefix of a simd type as an array. - // There might be more padding afterwards for some widths, but - // that's not a problem for reading less than that. - unsafe { &*array_ptr } - } - } - - #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] - const impl core::cmp::PartialEq for $id { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.as_array() == other.as_array() - } - } - - impl core::fmt::Debug for $id { - #[inline] - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - debug_simd_finish(f, stringify!($id), self.as_array()) + Self([$(Self::bool_to_internal($param_name)),*]) } } } From dab553268a52a4f494f57e6db4feccae9e26a491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= Date: Thu, 1 Jan 2026 22:48:01 +0100 Subject: [PATCH 0075/1843] x86: avoid using `transmute` in some tests --- .../stdarch/crates/core_arch/src/x86/sse.rs | 266 +++++++++--------- .../stdarch/crates/core_arch/src/x86/sse2.rs | 38 ++- 2 files changed, 150 insertions(+), 154 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/sse.rs b/library/stdarch/crates/core_arch/src/x86/sse.rs index 751f969e50c3..f167e8381d27 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse.rs @@ -2079,7 +2079,7 @@ pub unsafe fn _mm_stream_ps(mem_addr: *mut f32, a: __m128) { #[cfg(test)] mod tests { use crate::core_arch::assert_eq_const as assert_eq; - use crate::{hint::black_box, mem::transmute, ptr}; + use crate::{hint::black_box, ptr}; use std::boxed; use stdarch_test::simd_test; @@ -2221,7 +2221,7 @@ mod tests { } #[simd_test(enable = "sse")] - unsafe fn test_mm_min_ps() { + fn test_mm_min_ps() { let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); let r = _mm_min_ps(a, b); @@ -2234,10 +2234,10 @@ mod tests { // `r1` to `a` and `r2` to `b`. let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); - let r1: [u8; 16] = transmute(_mm_min_ps(a, b)); - let r2: [u8; 16] = transmute(_mm_min_ps(b, a)); - let a: [u8; 16] = transmute(a); - let b: [u8; 16] = transmute(b); + let r1 = _mm_min_ps(a, b).as_f32x4().to_bits(); + let r2 = _mm_min_ps(b, a).as_f32x4().to_bits(); + let a = a.as_f32x4().to_bits(); + let b = b.as_f32x4().to_bits(); assert_eq!(r1, b); assert_eq!(r2, a); assert_ne!(a, b); // sanity check that -0.0 is actually present @@ -2252,7 +2252,7 @@ mod tests { } #[simd_test(enable = "sse")] - unsafe fn test_mm_max_ps() { + fn test_mm_max_ps() { let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); let r = _mm_max_ps(a, b); @@ -2261,67 +2261,67 @@ mod tests { // Check SSE-specific semantics for -0.0 handling. let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); - let r1: [u8; 16] = transmute(_mm_max_ps(a, b)); - let r2: [u8; 16] = transmute(_mm_max_ps(b, a)); - let a: [u8; 16] = transmute(a); - let b: [u8; 16] = transmute(b); + let r1 = _mm_max_ps(a, b).as_f32x4().to_bits(); + let r2 = _mm_max_ps(b, a).as_f32x4().to_bits(); + let a = a.as_f32x4().to_bits(); + let b = b.as_f32x4().to_bits(); assert_eq!(r1, b); assert_eq!(r2, a); assert_ne!(a, b); // sanity check that -0.0 is actually present } #[simd_test(enable = "sse")] - const unsafe fn test_mm_and_ps() { - let a = transmute(u32x4::splat(0b0011)); - let b = transmute(u32x4::splat(0b0101)); + const fn test_mm_and_ps() { + let a = f32x4::from_bits(u32x4::splat(0b0011)).as_m128(); + let b = f32x4::from_bits(u32x4::splat(0b0101)).as_m128(); let r = _mm_and_ps(*black_box(&a), *black_box(&b)); - let e = transmute(u32x4::splat(0b0001)); + let e = f32x4::from_bits(u32x4::splat(0b0001)).as_m128(); assert_eq_m128(r, e); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_andnot_ps() { - let a = transmute(u32x4::splat(0b0011)); - let b = transmute(u32x4::splat(0b0101)); + const fn test_mm_andnot_ps() { + let a = f32x4::from_bits(u32x4::splat(0b0011)).as_m128(); + let b = f32x4::from_bits(u32x4::splat(0b0101)).as_m128(); let r = _mm_andnot_ps(*black_box(&a), *black_box(&b)); - let e = transmute(u32x4::splat(0b0100)); + let e = f32x4::from_bits(u32x4::splat(0b0100)).as_m128(); assert_eq_m128(r, e); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_or_ps() { - let a = transmute(u32x4::splat(0b0011)); - let b = transmute(u32x4::splat(0b0101)); + const fn test_mm_or_ps() { + let a = f32x4::from_bits(u32x4::splat(0b0011)).as_m128(); + let b = f32x4::from_bits(u32x4::splat(0b0101)).as_m128(); let r = _mm_or_ps(*black_box(&a), *black_box(&b)); - let e = transmute(u32x4::splat(0b0111)); + let e = f32x4::from_bits(u32x4::splat(0b0111)).as_m128(); assert_eq_m128(r, e); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_xor_ps() { - let a = transmute(u32x4::splat(0b0011)); - let b = transmute(u32x4::splat(0b0101)); + const fn test_mm_xor_ps() { + let a = f32x4::from_bits(u32x4::splat(0b0011)).as_m128(); + let b = f32x4::from_bits(u32x4::splat(0b0101)).as_m128(); let r = _mm_xor_ps(*black_box(&a), *black_box(&b)); - let e = transmute(u32x4::splat(0b0110)); + let e = f32x4::from_bits(u32x4::splat(0b0110)).as_m128(); assert_eq_m128(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpeq_ss() { + fn test_mm_cmpeq_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(-1.0, 5.0, 6.0, 7.0); - let r: u32x4 = transmute(_mm_cmpeq_ss(a, b)); - let e: u32x4 = transmute(_mm_setr_ps(f32::from_bits(0), 2.0, 3.0, 4.0)); + let r = _mm_cmpeq_ss(a, b).as_f32x4().to_bits(); + let e = f32x4::new(f32::from_bits(0), 2.0, 3.0, 4.0).to_bits(); assert_eq!(r, e); let b2 = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let r2: u32x4 = transmute(_mm_cmpeq_ss(a, b2)); - let e2: u32x4 = transmute(_mm_setr_ps(f32::from_bits(0xffffffff), 2.0, 3.0, 4.0)); + let r2 = _mm_cmpeq_ss(a, b2).as_f32x4().to_bits(); + let e2 = f32x4::new(f32::from_bits(0xffffffff), 2.0, 3.0, 4.0).to_bits(); assert_eq!(r2, e2); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmplt_ss() { + fn test_mm_cmplt_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); @@ -2331,21 +2331,21 @@ mod tests { let c1 = 0u32; // a.extract(0) < c.extract(0) let d1 = !0u32; // a.extract(0) < d.extract(0) - let rb: u32x4 = transmute(_mm_cmplt_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmplt_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmplt_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmplt_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmplt_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmplt_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmple_ss() { + fn test_mm_cmple_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); @@ -2355,21 +2355,21 @@ mod tests { let c1 = !0u32; // a.extract(0) <= c.extract(0) let d1 = !0u32; // a.extract(0) <= d.extract(0) - let rb: u32x4 = transmute(_mm_cmple_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmple_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmple_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmple_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmple_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmple_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpgt_ss() { + fn test_mm_cmpgt_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); @@ -2379,21 +2379,21 @@ mod tests { let c1 = 0u32; // a.extract(0) > c.extract(0) let d1 = 0u32; // a.extract(0) > d.extract(0) - let rb: u32x4 = transmute(_mm_cmpgt_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpgt_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpgt_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpgt_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpgt_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpgt_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpge_ss() { + fn test_mm_cmpge_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); @@ -2403,21 +2403,21 @@ mod tests { let c1 = !0u32; // a.extract(0) >= c.extract(0) let d1 = 0u32; // a.extract(0) >= d.extract(0) - let rb: u32x4 = transmute(_mm_cmpge_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpge_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpge_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpge_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpge_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpge_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpneq_ss() { + fn test_mm_cmpneq_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); @@ -2427,21 +2427,21 @@ mod tests { let c1 = 0u32; // a.extract(0) != c.extract(0) let d1 = !0u32; // a.extract(0) != d.extract(0) - let rb: u32x4 = transmute(_mm_cmpneq_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpneq_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpneq_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpneq_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpneq_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpneq_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpnlt_ss() { + fn test_mm_cmpnlt_ss() { // TODO: this test is exactly the same as for `_mm_cmpge_ss`, but there // must be a difference. It may have to do with behavior in the // presence of NaNs (signaling or quiet). If so, we should add tests @@ -2456,21 +2456,21 @@ mod tests { let c1 = !0u32; // a.extract(0) >= c.extract(0) let d1 = 0u32; // a.extract(0) >= d.extract(0) - let rb: u32x4 = transmute(_mm_cmpnlt_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpnlt_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpnlt_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpnlt_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpnlt_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpnlt_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpnle_ss() { + fn test_mm_cmpnle_ss() { // TODO: this test is exactly the same as for `_mm_cmpgt_ss`, but there // must be a difference. It may have to do with behavior in the // presence @@ -2485,21 +2485,21 @@ mod tests { let c1 = 0u32; // a.extract(0) > c.extract(0) let d1 = 0u32; // a.extract(0) > d.extract(0) - let rb: u32x4 = transmute(_mm_cmpnle_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpnle_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpnle_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpnle_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpnle_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpnle_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpngt_ss() { + fn test_mm_cmpngt_ss() { // TODO: this test is exactly the same as for `_mm_cmple_ss`, but there // must be a difference. It may have to do with behavior in the // presence of NaNs (signaling or quiet). If so, we should add tests @@ -2514,21 +2514,21 @@ mod tests { let c1 = !0u32; // a.extract(0) <= c.extract(0) let d1 = !0u32; // a.extract(0) <= d.extract(0) - let rb: u32x4 = transmute(_mm_cmpngt_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpngt_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpngt_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpngt_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpngt_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpngt_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpnge_ss() { + fn test_mm_cmpnge_ss() { // TODO: this test is exactly the same as for `_mm_cmplt_ss`, but there // must be a difference. It may have to do with behavior in the // presence of NaNs (signaling or quiet). If so, we should add tests @@ -2543,21 +2543,21 @@ mod tests { let c1 = 0u32; // a.extract(0) < c.extract(0) let d1 = !0u32; // a.extract(0) < d.extract(0) - let rb: u32x4 = transmute(_mm_cmpnge_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpnge_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpnge_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpnge_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpnge_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpnge_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpord_ss() { + fn test_mm_cmpord_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); @@ -2567,21 +2567,21 @@ mod tests { let c1 = 0u32; // a.extract(0) ord c.extract(0) let d1 = !0u32; // a.extract(0) ord d.extract(0) - let rb: u32x4 = transmute(_mm_cmpord_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpord_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpord_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpord_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpord_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpord_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpunord_ss() { + fn test_mm_cmpunord_ss() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); @@ -2591,160 +2591,160 @@ mod tests { let c1 = !0u32; // a.extract(0) unord c.extract(0) let d1 = 0u32; // a.extract(0) unord d.extract(0) - let rb: u32x4 = transmute(_mm_cmpunord_ss(a, b)); - let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0)); + let rb = _mm_cmpunord_ss(a, b).as_f32x4().to_bits(); + let eb = f32x4::new(f32::from_bits(b1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rb, eb); - let rc: u32x4 = transmute(_mm_cmpunord_ss(a, c)); - let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0)); + let rc = _mm_cmpunord_ss(a, c).as_f32x4().to_bits(); + let ec = f32x4::new(f32::from_bits(c1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rc, ec); - let rd: u32x4 = transmute(_mm_cmpunord_ss(a, d)); - let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0)); + let rd = _mm_cmpunord_ss(a, d).as_f32x4().to_bits(); + let ed = f32x4::new(f32::from_bits(d1), 2.0, 3.0, 4.0).to_bits(); assert_eq!(rd, ed); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpeq_ps() { + fn test_mm_cmpeq_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); let tru = !0u32; let fls = 0u32; let e = u32x4::new(fls, fls, tru, fls); - let r: u32x4 = transmute(_mm_cmpeq_ps(a, b)); + let r = _mm_cmpeq_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmplt_ps() { + fn test_mm_cmplt_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); let tru = !0u32; let fls = 0u32; let e = u32x4::new(tru, fls, fls, fls); - let r: u32x4 = transmute(_mm_cmplt_ps(a, b)); + let r = _mm_cmplt_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmple_ps() { + fn test_mm_cmple_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, 4.0); let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); let tru = !0u32; let fls = 0u32; let e = u32x4::new(tru, fls, tru, fls); - let r: u32x4 = transmute(_mm_cmple_ps(a, b)); + let r = _mm_cmple_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpgt_ps() { + fn test_mm_cmpgt_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); let tru = !0u32; let fls = 0u32; let e = u32x4::new(fls, tru, fls, fls); - let r: u32x4 = transmute(_mm_cmpgt_ps(a, b)); + let r = _mm_cmpgt_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpge_ps() { + fn test_mm_cmpge_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); let tru = !0u32; let fls = 0u32; let e = u32x4::new(fls, tru, tru, fls); - let r: u32x4 = transmute(_mm_cmpge_ps(a, b)); + let r = _mm_cmpge_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpneq_ps() { + fn test_mm_cmpneq_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); let tru = !0u32; let fls = 0u32; let e = u32x4::new(tru, tru, fls, tru); - let r: u32x4 = transmute(_mm_cmpneq_ps(a, b)); + let r = _mm_cmpneq_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpnlt_ps() { + fn test_mm_cmpnlt_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); let tru = !0u32; let fls = 0u32; let e = u32x4::new(fls, tru, tru, tru); - let r: u32x4 = transmute(_mm_cmpnlt_ps(a, b)); + let r = _mm_cmpnlt_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpnle_ps() { + fn test_mm_cmpnle_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); let tru = !0u32; let fls = 0u32; let e = u32x4::new(fls, tru, fls, tru); - let r: u32x4 = transmute(_mm_cmpnle_ps(a, b)); + let r = _mm_cmpnle_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpngt_ps() { + fn test_mm_cmpngt_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); let tru = !0u32; let fls = 0u32; let e = u32x4::new(tru, fls, tru, tru); - let r: u32x4 = transmute(_mm_cmpngt_ps(a, b)); + let r = _mm_cmpngt_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpnge_ps() { + fn test_mm_cmpnge_ps() { let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); let tru = !0u32; let fls = 0u32; let e = u32x4::new(tru, fls, fls, tru); - let r: u32x4 = transmute(_mm_cmpnge_ps(a, b)); + let r = _mm_cmpnge_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpord_ps() { + fn test_mm_cmpord_ps() { let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); let tru = !0u32; let fls = 0u32; let e = u32x4::new(tru, fls, fls, fls); - let r: u32x4 = transmute(_mm_cmpord_ps(a, b)); + let r = _mm_cmpord_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } #[simd_test(enable = "sse")] - unsafe fn test_mm_cmpunord_ps() { + fn test_mm_cmpunord_ps() { let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); let tru = !0u32; let fls = 0u32; let e = u32x4::new(fls, tru, tru, tru); - let r: u32x4 = transmute(_mm_cmpunord_ps(a, b)); + let r = _mm_cmpunord_ps(a, b).as_f32x4().to_bits(); assert_eq!(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/sse2.rs b/library/stdarch/crates/core_arch/src/x86/sse2.rs index d7ac00287f5e..0850d05445c6 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse2.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse2.rs @@ -3272,11 +3272,7 @@ mod tests { core_arch::{simd::*, x86::*}, hint::black_box, }; - use std::{ - boxed, f32, f64, - mem::{self, transmute}, - ptr, - }; + use std::{boxed, f32, f64, mem, ptr}; use stdarch_test::simd_test; const NAN: f64 = f64::NAN; @@ -4585,38 +4581,38 @@ mod tests { } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_and_pd() { - let a = transmute(u64x2::splat(5)); - let b = transmute(u64x2::splat(3)); + const fn test_mm_and_pd() { + let a = f64x2::from_bits(u64x2::splat(5)).as_m128d(); + let b = f64x2::from_bits(u64x2::splat(3)).as_m128d(); let r = _mm_and_pd(a, b); - let e = transmute(u64x2::splat(1)); + let e = f64x2::from_bits(u64x2::splat(1)).as_m128d(); assert_eq_m128d(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_andnot_pd() { - let a = transmute(u64x2::splat(5)); - let b = transmute(u64x2::splat(3)); + const fn test_mm_andnot_pd() { + let a = f64x2::from_bits(u64x2::splat(5)).as_m128d(); + let b = f64x2::from_bits(u64x2::splat(3)).as_m128d(); let r = _mm_andnot_pd(a, b); - let e = transmute(u64x2::splat(2)); + let e = f64x2::from_bits(u64x2::splat(2)).as_m128d(); assert_eq_m128d(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_or_pd() { - let a = transmute(u64x2::splat(5)); - let b = transmute(u64x2::splat(3)); + const fn test_mm_or_pd() { + let a = f64x2::from_bits(u64x2::splat(5)).as_m128d(); + let b = f64x2::from_bits(u64x2::splat(3)).as_m128d(); let r = _mm_or_pd(a, b); - let e = transmute(u64x2::splat(7)); + let e = f64x2::from_bits(u64x2::splat(7)).as_m128d(); assert_eq_m128d(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_xor_pd() { - let a = transmute(u64x2::splat(5)); - let b = transmute(u64x2::splat(3)); + const fn test_mm_xor_pd() { + let a = f64x2::from_bits(u64x2::splat(5)).as_m128d(); + let b = f64x2::from_bits(u64x2::splat(3)).as_m128d(); let r = _mm_xor_pd(a, b); - let e = transmute(u64x2::splat(6)); + let e = f64x2::from_bits(u64x2::splat(6)).as_m128d(); assert_eq_m128d(r, e); } From 68701ff93ad6e3168f7aa8a947b895746bfcb9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= Date: Thu, 1 Jan 2026 22:48:34 +0100 Subject: [PATCH 0076/1843] arm_shared: avoid using `transmute` and `unsafe fn` in tests --- library/stdarch/ci/run.sh | 2 +- .../src/arm_shared/neon/load_tests.rs | 97 +- .../core_arch/src/arm_shared/neon/mod.rs | 2177 +++++++++-------- .../src/arm_shared/neon/store_tests.rs | 144 +- .../src/arm_shared/neon/table_lookup_tests.rs | 27 +- .../core_arch/src/arm_shared/test_support.rs | 8 +- .../stdarch/crates/simd-test-macro/src/lib.rs | 47 +- 7 files changed, 1409 insertions(+), 1093 deletions(-) diff --git a/library/stdarch/ci/run.sh b/library/stdarch/ci/run.sh index bb3dcf93e75d..8a0b5fa26f66 100755 --- a/library/stdarch/ci/run.sh +++ b/library/stdarch/ci/run.sh @@ -40,7 +40,7 @@ case ${TARGET} in export RUSTFLAGS="${RUSTFLAGS} -C llvm-args=-fast-isel=false" ;; armv7-*eabihf | thumbv7-*eabihf) - export RUSTFLAGS="${RUSTFLAGS} -Ctarget-feature=+neon" + export RUSTFLAGS="${RUSTFLAGS} -Ctarget-feature=+neon,+fp16" ;; amdgcn-*) export RUSTFLAGS="${RUSTFLAGS} -Ctarget-cpu=gfx1200" diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs index bdf511ecf881..cc821b4af202 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs @@ -13,194 +13,195 @@ use crate::core_arch::aarch64::*; use crate::core_arch::simd::*; use std::mem; use stdarch_test::simd_test; + #[simd_test(enable = "neon")] -unsafe fn test_vld1_s8() { +fn test_vld1_s8() { let a: [i8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i8x8 = transmute(vld1_s8(a[1..].as_ptr())); + let r = unsafe { i8x8::from(vld1_s8(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_s8() { +fn test_vld1q_s8() { let a: [i8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let e = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r: i8x16 = transmute(vld1q_s8(a[1..].as_ptr())); + let r = unsafe { i8x16::from(vld1q_s8(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_s16() { +fn test_vld1_s16() { let a: [i16; 5] = [0, 1, 2, 3, 4]; let e = i16x4::new(1, 2, 3, 4); - let r: i16x4 = transmute(vld1_s16(a[1..].as_ptr())); + let r = unsafe { i16x4::from(vld1_s16(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_s16() { +fn test_vld1q_s16() { let a: [i16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; let e = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i16x8 = transmute(vld1q_s16(a[1..].as_ptr())); + let r = unsafe { i16x8::from(vld1q_s16(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_s32() { +fn test_vld1_s32() { let a: [i32; 3] = [0, 1, 2]; let e = i32x2::new(1, 2); - let r: i32x2 = transmute(vld1_s32(a[1..].as_ptr())); + let r = unsafe { i32x2::from(vld1_s32(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_s32() { +fn test_vld1q_s32() { let a: [i32; 5] = [0, 1, 2, 3, 4]; let e = i32x4::new(1, 2, 3, 4); - let r: i32x4 = transmute(vld1q_s32(a[1..].as_ptr())); + let r = unsafe { i32x4::from(vld1q_s32(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_s64() { +fn test_vld1_s64() { let a: [i64; 2] = [0, 1]; let e = i64x1::new(1); - let r: i64x1 = transmute(vld1_s64(a[1..].as_ptr())); + let r = unsafe { i64x1::from(vld1_s64(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_s64() { +fn test_vld1q_s64() { let a: [i64; 3] = [0, 1, 2]; let e = i64x2::new(1, 2); - let r: i64x2 = transmute(vld1q_s64(a[1..].as_ptr())); + let r = unsafe { i64x2::from(vld1q_s64(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_u8() { +fn test_vld1_u8() { let a: [u8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vld1_u8(a[1..].as_ptr())); + let r = unsafe { u8x8::from(vld1_u8(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_u8() { +fn test_vld1q_u8() { let a: [u8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let e = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r: u8x16 = transmute(vld1q_u8(a[1..].as_ptr())); + let r = unsafe { u8x16::from(vld1q_u8(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_u16() { +fn test_vld1_u16() { let a: [u16; 5] = [0, 1, 2, 3, 4]; let e = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vld1_u16(a[1..].as_ptr())); + let r = unsafe { u16x4::from(vld1_u16(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_u16() { +fn test_vld1q_u16() { let a: [u16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u16x8 = transmute(vld1q_u16(a[1..].as_ptr())); + let r = unsafe { u16x8::from(vld1q_u16(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_u32() { +fn test_vld1_u32() { let a: [u32; 3] = [0, 1, 2]; let e = u32x2::new(1, 2); - let r: u32x2 = transmute(vld1_u32(a[1..].as_ptr())); + let r = unsafe { u32x2::from(vld1_u32(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_u32() { +fn test_vld1q_u32() { let a: [u32; 5] = [0, 1, 2, 3, 4]; let e = u32x4::new(1, 2, 3, 4); - let r: u32x4 = transmute(vld1q_u32(a[1..].as_ptr())); + let r = unsafe { u32x4::from(vld1q_u32(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_u64() { +fn test_vld1_u64() { let a: [u64; 2] = [0, 1]; let e = u64x1::new(1); - let r: u64x1 = transmute(vld1_u64(a[1..].as_ptr())); + let r = unsafe { u64x1::from(vld1_u64(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_u64() { +fn test_vld1q_u64() { let a: [u64; 3] = [0, 1, 2]; let e = u64x2::new(1, 2); - let r: u64x2 = transmute(vld1q_u64(a[1..].as_ptr())); + let r = unsafe { u64x2::from(vld1q_u64(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_p8() { +fn test_vld1_p8() { let a: [p8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vld1_p8(a[1..].as_ptr())); + let r = unsafe { u8x8::from(vld1_p8(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_p8() { +fn test_vld1q_p8() { let a: [p8; 17] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let e = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r: u8x16 = transmute(vld1q_p8(a[1..].as_ptr())); + let r = unsafe { u8x16::from(vld1q_p8(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_p16() { +fn test_vld1_p16() { let a: [p16; 5] = [0, 1, 2, 3, 4]; let e = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vld1_p16(a[1..].as_ptr())); + let r = unsafe { u16x4::from(vld1_p16(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_p16() { +fn test_vld1q_p16() { let a: [p16; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u16x8 = transmute(vld1q_p16(a[1..].as_ptr())); + let r = unsafe { u16x8::from(vld1q_p16(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon,aes")] -unsafe fn test_vld1_p64() { +fn test_vld1_p64() { let a: [p64; 2] = [0, 1]; let e = u64x1::new(1); - let r: u64x1 = transmute(vld1_p64(a[1..].as_ptr())); + let r = unsafe { u64x1::from(vld1_p64(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon,aes")] -unsafe fn test_vld1q_p64() { +fn test_vld1q_p64() { let a: [p64; 3] = [0, 1, 2]; let e = u64x2::new(1, 2); - let r: u64x2 = transmute(vld1q_p64(a[1..].as_ptr())); + let r = unsafe { u64x2::from(vld1q_p64(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1_f32() { +fn test_vld1_f32() { let a: [f32; 3] = [0., 1., 2.]; let e = f32x2::new(1., 2.); - let r: f32x2 = transmute(vld1_f32(a[1..].as_ptr())); + let r = unsafe { f32x2::from(vld1_f32(a[1..].as_ptr())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] -unsafe fn test_vld1q_f32() { +fn test_vld1q_f32() { let a: [f32; 5] = [0., 1., 2., 3., 4.]; let e = f32x4::new(1., 2., 3., 4.); - let r: f32x4 = transmute(vld1q_f32(a[1..].as_ptr())); + let r = unsafe { f32x4::from(vld1q_f32(a[1..].as_ptr())) }; assert_eq!(r, e) } diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs index 809892e0bfd4..1ca8ce2b1395 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs @@ -1247,1268 +1247,1273 @@ mod tests { use crate::core_arch::arm::*; use crate::core_arch::arm_shared::test_support::*; use crate::core_arch::simd::*; - use std::{mem::transmute, vec::Vec}; use stdarch_test::simd_test; #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s8() { + fn test_vld1_lane_s8() { let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let elem: i8 = 42; let e = i8x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: i8x8 = transmute(vld1_lane_s8::<7>(&elem, transmute(a))); + let r = unsafe { i8x8::from(vld1_lane_s8::<7>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s8() { + fn test_vld1q_lane_s8() { let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let elem: i8 = 42; let e = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); - let r: i8x16 = transmute(vld1q_lane_s8::<15>(&elem, transmute(a))); + let r = unsafe { i8x16::from(vld1q_lane_s8::<15>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s16() { + fn test_vld1_lane_s16() { let a = i16x4::new(0, 1, 2, 3); let elem: i16 = 42; let e = i16x4::new(0, 1, 2, 42); - let r: i16x4 = transmute(vld1_lane_s16::<3>(&elem, transmute(a))); + let r = unsafe { i16x4::from(vld1_lane_s16::<3>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s16() { + fn test_vld1q_lane_s16() { let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let elem: i16 = 42; let e = i16x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: i16x8 = transmute(vld1q_lane_s16::<7>(&elem, transmute(a))); + let r = unsafe { i16x8::from(vld1q_lane_s16::<7>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s32() { + fn test_vld1_lane_s32() { let a = i32x2::new(0, 1); let elem: i32 = 42; let e = i32x2::new(0, 42); - let r: i32x2 = transmute(vld1_lane_s32::<1>(&elem, transmute(a))); + let r = unsafe { i32x2::from(vld1_lane_s32::<1>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s32() { + fn test_vld1q_lane_s32() { let a = i32x4::new(0, 1, 2, 3); let elem: i32 = 42; let e = i32x4::new(0, 1, 2, 42); - let r: i32x4 = transmute(vld1q_lane_s32::<3>(&elem, transmute(a))); + let r = unsafe { i32x4::from(vld1q_lane_s32::<3>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s64() { + fn test_vld1_lane_s64() { let a = i64x1::new(0); let elem: i64 = 42; let e = i64x1::new(42); - let r: i64x1 = transmute(vld1_lane_s64::<0>(&elem, transmute(a))); + let r = unsafe { i64x1::from(vld1_lane_s64::<0>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s64() { + fn test_vld1q_lane_s64() { let a = i64x2::new(0, 1); let elem: i64 = 42; let e = i64x2::new(0, 42); - let r: i64x2 = transmute(vld1q_lane_s64::<1>(&elem, transmute(a))); + let r = unsafe { i64x2::from(vld1q_lane_s64::<1>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u8() { + fn test_vld1_lane_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let elem: u8 = 42; let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u8x8 = transmute(vld1_lane_u8::<7>(&elem, transmute(a))); + let r = unsafe { u8x8::from(vld1_lane_u8::<7>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u8() { + fn test_vld1q_lane_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let elem: u8 = 42; let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); - let r: u8x16 = transmute(vld1q_lane_u8::<15>(&elem, transmute(a))); + let r = unsafe { u8x16::from(vld1q_lane_u8::<15>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u16() { + fn test_vld1_lane_u16() { let a = u16x4::new(0, 1, 2, 3); let elem: u16 = 42; let e = u16x4::new(0, 1, 2, 42); - let r: u16x4 = transmute(vld1_lane_u16::<3>(&elem, transmute(a))); + let r = unsafe { u16x4::from(vld1_lane_u16::<3>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u16() { + fn test_vld1q_lane_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let elem: u16 = 42; let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u16x8 = transmute(vld1q_lane_u16::<7>(&elem, transmute(a))); + let r = unsafe { u16x8::from(vld1q_lane_u16::<7>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u32() { + fn test_vld1_lane_u32() { let a = u32x2::new(0, 1); let elem: u32 = 42; let e = u32x2::new(0, 42); - let r: u32x2 = transmute(vld1_lane_u32::<1>(&elem, transmute(a))); + let r = unsafe { u32x2::from(vld1_lane_u32::<1>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u32() { + fn test_vld1q_lane_u32() { let a = u32x4::new(0, 1, 2, 3); let elem: u32 = 42; let e = u32x4::new(0, 1, 2, 42); - let r: u32x4 = transmute(vld1q_lane_u32::<3>(&elem, transmute(a))); + let r = unsafe { u32x4::from(vld1q_lane_u32::<3>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u64() { + fn test_vld1_lane_u64() { let a = u64x1::new(0); let elem: u64 = 42; let e = u64x1::new(42); - let r: u64x1 = transmute(vld1_lane_u64::<0>(&elem, transmute(a))); + let r = unsafe { u64x1::from(vld1_lane_u64::<0>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u64() { + fn test_vld1q_lane_u64() { let a = u64x2::new(0, 1); let elem: u64 = 42; let e = u64x2::new(0, 42); - let r: u64x2 = transmute(vld1q_lane_u64::<1>(&elem, transmute(a))); + let r = unsafe { u64x2::from(vld1q_lane_u64::<1>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_p8() { + fn test_vld1_lane_p8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let elem: p8 = 42; let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u8x8 = transmute(vld1_lane_p8::<7>(&elem, transmute(a))); + let r = unsafe { u8x8::from(vld1_lane_p8::<7>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_p8() { + fn test_vld1q_lane_p8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let elem: p8 = 42; let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); - let r: u8x16 = transmute(vld1q_lane_p8::<15>(&elem, transmute(a))); + let r = unsafe { u8x16::from(vld1q_lane_p8::<15>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_p16() { + fn test_vld1_lane_p16() { let a = u16x4::new(0, 1, 2, 3); let elem: p16 = 42; let e = u16x4::new(0, 1, 2, 42); - let r: u16x4 = transmute(vld1_lane_p16::<3>(&elem, transmute(a))); + let r = unsafe { u16x4::from(vld1_lane_p16::<3>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_p16() { + fn test_vld1q_lane_p16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let elem: p16 = 42; let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u16x8 = transmute(vld1q_lane_p16::<7>(&elem, transmute(a))); + let r = unsafe { u16x8::from(vld1q_lane_p16::<7>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon,aes")] - unsafe fn test_vld1_lane_p64() { + fn test_vld1_lane_p64() { let a = u64x1::new(0); let elem: u64 = 42; let e = u64x1::new(42); - let r: u64x1 = transmute(vld1_lane_p64::<0>(&elem, transmute(a))); + let r = unsafe { u64x1::from(vld1_lane_p64::<0>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon,aes")] - unsafe fn test_vld1q_lane_p64() { + fn test_vld1q_lane_p64() { let a = u64x2::new(0, 1); let elem: u64 = 42; let e = u64x2::new(0, 42); - let r: u64x2 = transmute(vld1q_lane_p64::<1>(&elem, transmute(a))); + let r = unsafe { u64x2::from(vld1q_lane_p64::<1>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_f32() { + fn test_vld1_lane_f32() { let a = f32x2::new(0., 1.); let elem: f32 = 42.; let e = f32x2::new(0., 42.); - let r: f32x2 = transmute(vld1_lane_f32::<1>(&elem, transmute(a))); + let r = unsafe { f32x2::from(vld1_lane_f32::<1>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_f32() { + fn test_vld1q_lane_f32() { let a = f32x4::new(0., 1., 2., 3.); let elem: f32 = 42.; let e = f32x4::new(0., 1., 2., 42.); - let r: f32x4 = transmute(vld1q_lane_f32::<3>(&elem, transmute(a))); + let r = unsafe { f32x4::from(vld1q_lane_f32::<3>(&elem, a.into())) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s8() { + fn test_vld1_dup_s8() { let elem: i8 = 42; let e = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: i8x8 = transmute(vld1_dup_s8(&elem)); + let r = unsafe { i8x8::from(vld1_dup_s8(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s8() { + fn test_vld1q_dup_s8() { let elem: i8 = 42; let e = i8x16::new( 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, ); - let r: i8x16 = transmute(vld1q_dup_s8(&elem)); + let r = unsafe { i8x16::from(vld1q_dup_s8(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s16() { + fn test_vld1_dup_s16() { let elem: i16 = 42; let e = i16x4::new(42, 42, 42, 42); - let r: i16x4 = transmute(vld1_dup_s16(&elem)); + let r = unsafe { i16x4::from(vld1_dup_s16(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s16() { + fn test_vld1q_dup_s16() { let elem: i16 = 42; let e = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: i16x8 = transmute(vld1q_dup_s16(&elem)); + let r = unsafe { i16x8::from(vld1q_dup_s16(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s32() { + fn test_vld1_dup_s32() { let elem: i32 = 42; let e = i32x2::new(42, 42); - let r: i32x2 = transmute(vld1_dup_s32(&elem)); + let r = unsafe { i32x2::from(vld1_dup_s32(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s32() { + fn test_vld1q_dup_s32() { let elem: i32 = 42; let e = i32x4::new(42, 42, 42, 42); - let r: i32x4 = transmute(vld1q_dup_s32(&elem)); + let r = unsafe { i32x4::from(vld1q_dup_s32(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s64() { + fn test_vld1_dup_s64() { let elem: i64 = 42; let e = i64x1::new(42); - let r: i64x1 = transmute(vld1_dup_s64(&elem)); + let r = unsafe { i64x1::from(vld1_dup_s64(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s64() { + fn test_vld1q_dup_s64() { let elem: i64 = 42; let e = i64x2::new(42, 42); - let r: i64x2 = transmute(vld1q_dup_s64(&elem)); + let r = unsafe { i64x2::from(vld1q_dup_s64(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u8() { + fn test_vld1_dup_u8() { let elem: u8 = 42; let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u8x8 = transmute(vld1_dup_u8(&elem)); + let r = unsafe { u8x8::from(vld1_dup_u8(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u8() { + fn test_vld1q_dup_u8() { let elem: u8 = 42; let e = u8x16::new( 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, ); - let r: u8x16 = transmute(vld1q_dup_u8(&elem)); + let r = unsafe { u8x16::from(vld1q_dup_u8(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u16() { + fn test_vld1_dup_u16() { let elem: u16 = 42; let e = u16x4::new(42, 42, 42, 42); - let r: u16x4 = transmute(vld1_dup_u16(&elem)); + let r = unsafe { u16x4::from(vld1_dup_u16(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u16() { + fn test_vld1q_dup_u16() { let elem: u16 = 42; let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u16x8 = transmute(vld1q_dup_u16(&elem)); + let r = unsafe { u16x8::from(vld1q_dup_u16(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u32() { + fn test_vld1_dup_u32() { let elem: u32 = 42; let e = u32x2::new(42, 42); - let r: u32x2 = transmute(vld1_dup_u32(&elem)); + let r = unsafe { u32x2::from(vld1_dup_u32(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u32() { + fn test_vld1q_dup_u32() { let elem: u32 = 42; let e = u32x4::new(42, 42, 42, 42); - let r: u32x4 = transmute(vld1q_dup_u32(&elem)); + let r = unsafe { u32x4::from(vld1q_dup_u32(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u64() { + fn test_vld1_dup_u64() { let elem: u64 = 42; let e = u64x1::new(42); - let r: u64x1 = transmute(vld1_dup_u64(&elem)); + let r = unsafe { u64x1::from(vld1_dup_u64(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u64() { + fn test_vld1q_dup_u64() { let elem: u64 = 42; let e = u64x2::new(42, 42); - let r: u64x2 = transmute(vld1q_dup_u64(&elem)); + let r = unsafe { u64x2::from(vld1q_dup_u64(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_p8() { + fn test_vld1_dup_p8() { let elem: p8 = 42; let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u8x8 = transmute(vld1_dup_p8(&elem)); + let r = unsafe { u8x8::from(vld1_dup_p8(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_p8() { + fn test_vld1q_dup_p8() { let elem: p8 = 42; let e = u8x16::new( 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, ); - let r: u8x16 = transmute(vld1q_dup_p8(&elem)); + let r = unsafe { u8x16::from(vld1q_dup_p8(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_p16() { + fn test_vld1_dup_p16() { let elem: p16 = 42; let e = u16x4::new(42, 42, 42, 42); - let r: u16x4 = transmute(vld1_dup_p16(&elem)); + let r = unsafe { u16x4::from(vld1_dup_p16(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_p16() { + fn test_vld1q_dup_p16() { let elem: p16 = 42; let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u16x8 = transmute(vld1q_dup_p16(&elem)); + let r = unsafe { u16x8::from(vld1q_dup_p16(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon,aes")] - unsafe fn test_vld1_dup_p64() { + fn test_vld1_dup_p64() { let elem: u64 = 42; let e = u64x1::new(42); - let r: u64x1 = transmute(vld1_dup_p64(&elem)); + let r = unsafe { u64x1::from(vld1_dup_p64(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon,aes")] - unsafe fn test_vld1q_dup_p64() { + fn test_vld1q_dup_p64() { let elem: u64 = 42; let e = u64x2::new(42, 42); - let r: u64x2 = transmute(vld1q_dup_p64(&elem)); + let r = unsafe { u64x2::from(vld1q_dup_p64(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_f32() { + fn test_vld1_dup_f32() { let elem: f32 = 42.; let e = f32x2::new(42., 42.); - let r: f32x2 = transmute(vld1_dup_f32(&elem)); + let r = unsafe { f32x2::from(vld1_dup_f32(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_f32() { + fn test_vld1q_dup_f32() { let elem: f32 = 42.; let e = f32x4::new(42., 42., 42., 42.); - let r: f32x4 = transmute(vld1q_dup_f32(&elem)); + let r = unsafe { f32x4::from(vld1q_dup_f32(&elem)) }; assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_u8() { - let v = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r = vget_lane_u8::<1>(transmute(v)); + fn test_vget_lane_u8() { + let v = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r = vget_lane_u8::<1>(v.into()); assert_eq!(r, 2); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u32() { + fn test_vgetq_lane_u32() { + let v = u32x4::new(1, 2, 3, 4); + let r = vgetq_lane_u32::<1>(v.into()); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_s32() { let v = i32x4::new(1, 2, 3, 4); - let r = vgetq_lane_u32::<1>(transmute(v)); + let r = vgetq_lane_s32::<1>(v.into()); assert_eq!(r, 2); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_s32() { - let v = i32x4::new(1, 2, 3, 4); - let r = vgetq_lane_s32::<1>(transmute(v)); - assert_eq!(r, 2); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_u64() { - let v: u64 = 1; - let r = vget_lane_u64::<0>(transmute(v)); - assert_eq!(r, 1); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u16() { - let v = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r = vgetq_lane_u16::<1>(transmute(v)); - assert_eq!(r, 2); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_s8() { - let v = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let r = vget_lane_s8::<2>(transmute(v)); - assert_eq!(r, 2); - let r = vget_lane_s8::<4>(transmute(v)); - assert_eq!(r, 4); - let r = vget_lane_s8::<5>(transmute(v)); - assert_eq!(r, 5); - } - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_p8() { - let v = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let r = vget_lane_p8::<2>(transmute(v)); - assert_eq!(r, 2); - let r = vget_lane_p8::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vget_lane_p8::<5>(transmute(v)); - assert_eq!(r, 5); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_p16() { - let v = u16x4::new(0, 1, 2, 3); - let r = vget_lane_p16::<2>(transmute(v)); - assert_eq!(r, 2); - let r = vget_lane_p16::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vget_lane_p16::<0>(transmute(v)); - assert_eq!(r, 0); - let r = vget_lane_p16::<1>(transmute(v)); - assert_eq!(r, 1); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_s16() { - let v = i16x4::new(0, 1, 2, 3); - let r = vget_lane_s16::<2>(transmute(v)); - assert_eq!(r, 2); - let r = vget_lane_s16::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vget_lane_s16::<0>(transmute(v)); - assert_eq!(r, 0); - let r = vget_lane_s16::<1>(transmute(v)); - assert_eq!(r, 1); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_u16() { - let v = u16x4::new(0, 1, 2, 3); - let r = vget_lane_u16::<2>(transmute(v)); - assert_eq!(r, 2); - let r = vget_lane_u16::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vget_lane_u16::<0>(transmute(v)); - assert_eq!(r, 0); - let r = vget_lane_u16::<1>(transmute(v)); - assert_eq!(r, 1); - } - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_f32() { - let v = f32x2::new(0.0, 1.0); - let r = vget_lane_f32::<1>(transmute(v)); - assert_eq!(r, 1.0); - let r = vget_lane_f32::<0>(transmute(v)); - assert_eq!(r, 0.0); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_s32() { - let v = i32x2::new(0, 1); - let r = vget_lane_s32::<1>(transmute(v)); - assert_eq!(r, 1); - let r = vget_lane_s32::<0>(transmute(v)); - assert_eq!(r, 0); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_u32() { - let v = u32x2::new(0, 1); - let r = vget_lane_u32::<1>(transmute(v)); - assert_eq!(r, 1); - let r = vget_lane_u32::<0>(transmute(v)); - assert_eq!(r, 0); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_s64() { - let v = i64x1::new(1); - let r = vget_lane_s64::<0>(transmute(v)); - assert_eq!(r, 1); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_p64() { + fn test_vget_lane_u64() { let v = u64x1::new(1); - let r = vget_lane_p64::<0>(transmute(v)); + let r = vget_lane_u64::<0>(v.into()); assert_eq!(r, 1); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_s8() { - let v = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = vgetq_lane_s8::<7>(transmute(v)); - assert_eq!(r, 7); - let r = vgetq_lane_s8::<13>(transmute(v)); - assert_eq!(r, 13); - let r = vgetq_lane_s8::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vgetq_lane_s8::<0>(transmute(v)); - assert_eq!(r, 0); + fn test_vgetq_lane_u16() { + let v = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r = vgetq_lane_u16::<1>(v.into()); + assert_eq!(r, 2); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_p8() { - let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = vgetq_lane_p8::<7>(transmute(v)); - assert_eq!(r, 7); - let r = vgetq_lane_p8::<13>(transmute(v)); - assert_eq!(r, 13); - let r = vgetq_lane_p8::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vgetq_lane_p8::<0>(transmute(v)); - assert_eq!(r, 0); + fn test_vget_lane_s8() { + let v = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vget_lane_s8::<2>(v.into()); + assert_eq!(r, 2); + let r = vget_lane_s8::<4>(v.into()); + assert_eq!(r, 4); + let r = vget_lane_s8::<5>(v.into()); + assert_eq!(r, 5); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u8() { - let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = vgetq_lane_u8::<7>(transmute(v)); - assert_eq!(r, 7); - let r = vgetq_lane_u8::<13>(transmute(v)); - assert_eq!(r, 13); - let r = vgetq_lane_u8::<3>(transmute(v)); + fn test_vget_lane_p8() { + let v = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vget_lane_p8::<2>(v.into()); + assert_eq!(r, 2); + let r = vget_lane_p8::<3>(v.into()); assert_eq!(r, 3); - let r = vgetq_lane_u8::<0>(transmute(v)); - assert_eq!(r, 0); + let r = vget_lane_p8::<5>(v.into()); + assert_eq!(r, 5); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_s16() { - let v = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let r = vgetq_lane_s16::<3>(transmute(v)); + fn test_vget_lane_p16() { + let v = u16x4::new(0, 1, 2, 3); + let r = vget_lane_p16::<2>(v.into()); + assert_eq!(r, 2); + let r = vget_lane_p16::<3>(v.into()); assert_eq!(r, 3); - let r = vgetq_lane_s16::<6>(transmute(v)); - assert_eq!(r, 6); - let r = vgetq_lane_s16::<0>(transmute(v)); + let r = vget_lane_p16::<0>(v.into()); assert_eq!(r, 0); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_p16() { - let v = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let r = vgetq_lane_p16::<3>(transmute(v)); - assert_eq!(r, 3); - let r = vgetq_lane_p16::<7>(transmute(v)); - assert_eq!(r, 7); - let r = vgetq_lane_p16::<1>(transmute(v)); + let r = vget_lane_p16::<1>(v.into()); assert_eq!(r, 1); } + #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_f32() { - let v = f32x4::new(0.0, 1.0, 2.0, 3.0); - let r = vgetq_lane_f32::<3>(transmute(v)); - assert_eq!(r, 3.0); - let r = vgetq_lane_f32::<0>(transmute(v)); + fn test_vget_lane_s16() { + let v = i16x4::new(0, 1, 2, 3); + let r = vget_lane_s16::<2>(v.into()); + assert_eq!(r, 2); + let r = vget_lane_s16::<3>(v.into()); + assert_eq!(r, 3); + let r = vget_lane_s16::<0>(v.into()); + assert_eq!(r, 0); + let r = vget_lane_s16::<1>(v.into()); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + fn test_vget_lane_u16() { + let v = u16x4::new(0, 1, 2, 3); + let r = vget_lane_u16::<2>(v.into()); + assert_eq!(r, 2); + let r = vget_lane_u16::<3>(v.into()); + assert_eq!(r, 3); + let r = vget_lane_u16::<0>(v.into()); + assert_eq!(r, 0); + let r = vget_lane_u16::<1>(v.into()); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + fn test_vget_lane_f32() { + let v = f32x2::new(0.0, 1.0); + let r = vget_lane_f32::<1>(v.into()); + assert_eq!(r, 1.0); + let r = vget_lane_f32::<0>(v.into()); assert_eq!(r, 0.0); - let r = vgetq_lane_f32::<2>(transmute(v)); + } + + #[simd_test(enable = "neon")] + fn test_vget_lane_s32() { + let v = i32x2::new(0, 1); + let r = vget_lane_s32::<1>(v.into()); + assert_eq!(r, 1); + let r = vget_lane_s32::<0>(v.into()); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + fn test_vget_lane_u32() { + let v = u32x2::new(0, 1); + let r = vget_lane_u32::<1>(v.into()); + assert_eq!(r, 1); + let r = vget_lane_u32::<0>(v.into()); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + fn test_vget_lane_s64() { + let v = i64x1::new(1); + let r = vget_lane_s64::<0>(v.into()); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + fn test_vget_lane_p64() { + let v = u64x1::new(1); + let r = vget_lane_p64::<0>(v.into()); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_s8() { + let v = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_s8::<7>(v.into()); + assert_eq!(r, 7); + let r = vgetq_lane_s8::<13>(v.into()); + assert_eq!(r, 13); + let r = vgetq_lane_s8::<3>(v.into()); + assert_eq!(r, 3); + let r = vgetq_lane_s8::<0>(v.into()); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_p8() { + let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_p8::<7>(v.into()); + assert_eq!(r, 7); + let r = vgetq_lane_p8::<13>(v.into()); + assert_eq!(r, 13); + let r = vgetq_lane_p8::<3>(v.into()); + assert_eq!(r, 3); + let r = vgetq_lane_p8::<0>(v.into()); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_u8() { + let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_u8::<7>(v.into()); + assert_eq!(r, 7); + let r = vgetq_lane_u8::<13>(v.into()); + assert_eq!(r, 13); + let r = vgetq_lane_u8::<3>(v.into()); + assert_eq!(r, 3); + let r = vgetq_lane_u8::<0>(v.into()); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_s16() { + let v = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vgetq_lane_s16::<3>(v.into()); + assert_eq!(r, 3); + let r = vgetq_lane_s16::<6>(v.into()); + assert_eq!(r, 6); + let r = vgetq_lane_s16::<0>(v.into()); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_p16() { + let v = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vgetq_lane_p16::<3>(v.into()); + assert_eq!(r, 3); + let r = vgetq_lane_p16::<7>(v.into()); + assert_eq!(r, 7); + let r = vgetq_lane_p16::<1>(v.into()); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + fn test_vgetq_lane_f32() { + let v = f32x4::new(0.0, 1.0, 2.0, 3.0); + let r = vgetq_lane_f32::<3>(v.into()); + assert_eq!(r, 3.0); + let r = vgetq_lane_f32::<0>(v.into()); + assert_eq!(r, 0.0); + let r = vgetq_lane_f32::<2>(v.into()); assert_eq!(r, 2.0); - let r = vgetq_lane_f32::<1>(transmute(v)); + let r = vgetq_lane_f32::<1>(v.into()); assert_eq!(r, 1.0); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_s64() { + fn test_vgetq_lane_s64() { let v = i64x2::new(0, 1); - let r = vgetq_lane_s64::<1>(transmute(v)); + let r = vgetq_lane_s64::<1>(v.into()); assert_eq!(r, 1); - let r = vgetq_lane_s64::<0>(transmute(v)); + let r = vgetq_lane_s64::<0>(v.into()); assert_eq!(r, 0); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_p64() { + fn test_vgetq_lane_p64() { let v = u64x2::new(0, 1); - let r = vgetq_lane_p64::<1>(transmute(v)); + let r = vgetq_lane_p64::<1>(v.into()); assert_eq!(r, 1); - let r = vgetq_lane_p64::<0>(transmute(v)); + let r = vgetq_lane_p64::<0>(v.into()); assert_eq!(r, 0); } #[simd_test(enable = "neon")] - unsafe fn test_vext_s64() { + fn test_vext_s64() { let a: i64x1 = i64x1::new(0); let b: i64x1 = i64x1::new(1); let e: i64x1 = i64x1::new(0); - let r: i64x1 = transmute(vext_s64::<0>(transmute(a), transmute(b))); + let r = unsafe { i64x1::from(vext_s64::<0>(a.into(), b.into())) }; assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vext_u64() { + fn test_vext_u64() { let a: u64x1 = u64x1::new(0); let b: u64x1 = u64x1::new(1); let e: u64x1 = u64x1::new(0); - let r: u64x1 = transmute(vext_u64::<0>(transmute(a), transmute(b))); + let r = unsafe { u64x1::from(vext_u64::<0>(a.into(), b.into())) }; assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_s8() { + fn test_vget_high_s8() { let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let e = i8x8::new(9, 10, 11, 12, 13, 14, 15, 16); - let r: i8x8 = transmute(vget_high_s8(transmute(a))); + let r = i8x8::from(vget_high_s8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_s16() { + fn test_vget_high_s16() { let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = i16x4::new(5, 6, 7, 8); - let r: i16x4 = transmute(vget_high_s16(transmute(a))); + let r = i16x4::from(vget_high_s16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_s32() { + fn test_vget_high_s32() { let a = i32x4::new(1, 2, 3, 4); let e = i32x2::new(3, 4); - let r: i32x2 = transmute(vget_high_s32(transmute(a))); + let r = i32x2::from(vget_high_s32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_s64() { + fn test_vget_high_s64() { let a = i64x2::new(1, 2); let e = i64x1::new(2); - let r: i64x1 = transmute(vget_high_s64(transmute(a))); + let r = i64x1::from(vget_high_s64(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_u8() { + fn test_vget_high_u8() { let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let e = u8x8::new(9, 10, 11, 12, 13, 14, 15, 16); - let r: u8x8 = transmute(vget_high_u8(transmute(a))); + let r = u8x8::from(vget_high_u8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_u16() { + fn test_vget_high_u16() { let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = u16x4::new(5, 6, 7, 8); - let r: u16x4 = transmute(vget_high_u16(transmute(a))); + let r = u16x4::from(vget_high_u16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_u32() { + fn test_vget_high_u32() { let a = u32x4::new(1, 2, 3, 4); let e = u32x2::new(3, 4); - let r: u32x2 = transmute(vget_high_u32(transmute(a))); + let r = u32x2::from(vget_high_u32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_u64() { + fn test_vget_high_u64() { let a = u64x2::new(1, 2); let e = u64x1::new(2); - let r: u64x1 = transmute(vget_high_u64(transmute(a))); + let r = u64x1::from(vget_high_u64(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_p8() { + fn test_vget_high_p8() { let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let e = u8x8::new(9, 10, 11, 12, 13, 14, 15, 16); - let r: u8x8 = transmute(vget_high_p8(transmute(a))); + let r = u8x8::from(vget_high_p8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_p16() { + fn test_vget_high_p16() { let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = u16x4::new(5, 6, 7, 8); - let r: u16x4 = transmute(vget_high_p16(transmute(a))); + let r = u16x4::from(vget_high_p16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_high_f32() { + fn test_vget_high_f32() { let a = f32x4::new(1.0, 2.0, 3.0, 4.0); let e = f32x2::new(3.0, 4.0); - let r: f32x2 = transmute(vget_high_f32(transmute(a))); + let r = f32x2::from(vget_high_f32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_s8() { + fn test_vget_low_s8() { let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i8x8 = transmute(vget_low_s8(transmute(a))); + let r = i8x8::from(vget_low_s8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_s16() { + fn test_vget_low_s16() { let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = i16x4::new(1, 2, 3, 4); - let r: i16x4 = transmute(vget_low_s16(transmute(a))); + let r = i16x4::from(vget_low_s16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_s32() { + fn test_vget_low_s32() { let a = i32x4::new(1, 2, 3, 4); let e = i32x2::new(1, 2); - let r: i32x2 = transmute(vget_low_s32(transmute(a))); + let r = i32x2::from(vget_low_s32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_s64() { + fn test_vget_low_s64() { let a = i64x2::new(1, 2); let e = i64x1::new(1); - let r: i64x1 = transmute(vget_low_s64(transmute(a))); + let r = i64x1::from(vget_low_s64(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_u8() { + fn test_vget_low_u8() { let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vget_low_u8(transmute(a))); + let r = u8x8::from(vget_low_u8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_u16() { + fn test_vget_low_u16() { let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vget_low_u16(transmute(a))); + let r = u16x4::from(vget_low_u16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_u32() { + fn test_vget_low_u32() { let a = u32x4::new(1, 2, 3, 4); let e = u32x2::new(1, 2); - let r: u32x2 = transmute(vget_low_u32(transmute(a))); + let r = u32x2::from(vget_low_u32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_u64() { + fn test_vget_low_u64() { let a = u64x2::new(1, 2); let e = u64x1::new(1); - let r: u64x1 = transmute(vget_low_u64(transmute(a))); + let r = u64x1::from(vget_low_u64(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_p8() { + fn test_vget_low_p8() { let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vget_low_p8(transmute(a))); + let r = u8x8::from(vget_low_p8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_p16() { + fn test_vget_low_p16() { let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vget_low_p16(transmute(a))); + let r = u16x4::from(vget_low_p16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_low_f32() { + fn test_vget_low_f32() { let a = f32x4::new(1.0, 2.0, 3.0, 4.0); let e = f32x2::new(1.0, 2.0); - let r: f32x2 = transmute(vget_low_f32(transmute(a))); + let r = f32x2::from(vget_low_f32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_s8() { + fn test_vdupq_n_s8() { let v: i8 = 42; let e = i8x16::new( 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, ); - let r: i8x16 = transmute(vdupq_n_s8(v)); + let r = i8x16::from(vdupq_n_s8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_s16() { + fn test_vdupq_n_s16() { let v: i16 = 64; let e = i16x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: i16x8 = transmute(vdupq_n_s16(v)); + let r = i16x8::from(vdupq_n_s16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_s32() { + fn test_vdupq_n_s32() { let v: i32 = 64; let e = i32x4::new(64, 64, 64, 64); - let r: i32x4 = transmute(vdupq_n_s32(v)); + let r = i32x4::from(vdupq_n_s32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_s64() { + fn test_vdupq_n_s64() { let v: i64 = 64; let e = i64x2::new(64, 64); - let r: i64x2 = transmute(vdupq_n_s64(v)); + let r = i64x2::from(vdupq_n_s64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_u8() { + fn test_vdupq_n_u8() { let v: u8 = 64; let e = u8x16::new( 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, ); - let r: u8x16 = transmute(vdupq_n_u8(v)); + let r = u8x16::from(vdupq_n_u8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_u16() { + fn test_vdupq_n_u16() { let v: u16 = 64; let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u16x8 = transmute(vdupq_n_u16(v)); + let r = u16x8::from(vdupq_n_u16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_u32() { + fn test_vdupq_n_u32() { let v: u32 = 64; let e = u32x4::new(64, 64, 64, 64); - let r: u32x4 = transmute(vdupq_n_u32(v)); + let r = u32x4::from(vdupq_n_u32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_u64() { + fn test_vdupq_n_u64() { let v: u64 = 64; let e = u64x2::new(64, 64); - let r: u64x2 = transmute(vdupq_n_u64(v)); + let r = u64x2::from(vdupq_n_u64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_p8() { + fn test_vdupq_n_p8() { let v: p8 = 64; let e = u8x16::new( 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, ); - let r: u8x16 = transmute(vdupq_n_p8(v)); + let r = u8x16::from(vdupq_n_p8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_p16() { + fn test_vdupq_n_p16() { let v: p16 = 64; let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u16x8 = transmute(vdupq_n_p16(v)); + let r = u16x8::from(vdupq_n_p16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_f32() { + fn test_vdupq_n_f32() { let v: f32 = 64.0; let e = f32x4::new(64.0, 64.0, 64.0, 64.0); - let r: f32x4 = transmute(vdupq_n_f32(v)); + let r = f32x4::from(vdupq_n_f32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_s8() { + fn test_vdup_n_s8() { let v: i8 = 64; let e = i8x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: i8x8 = transmute(vdup_n_s8(v)); + let r = i8x8::from(vdup_n_s8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_s16() { + fn test_vdup_n_s16() { let v: i16 = 64; let e = i16x4::new(64, 64, 64, 64); - let r: i16x4 = transmute(vdup_n_s16(v)); + let r = i16x4::from(vdup_n_s16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_s32() { + fn test_vdup_n_s32() { let v: i32 = 64; let e = i32x2::new(64, 64); - let r: i32x2 = transmute(vdup_n_s32(v)); + let r = i32x2::from(vdup_n_s32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_s64() { + fn test_vdup_n_s64() { let v: i64 = 64; let e = i64x1::new(64); - let r: i64x1 = transmute(vdup_n_s64(v)); + let r = i64x1::from(vdup_n_s64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_u8() { + fn test_vdup_n_u8() { let v: u8 = 64; let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u8x8 = transmute(vdup_n_u8(v)); + let r = u8x8::from(vdup_n_u8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_u16() { + fn test_vdup_n_u16() { let v: u16 = 64; let e = u16x4::new(64, 64, 64, 64); - let r: u16x4 = transmute(vdup_n_u16(v)); + let r = u16x4::from(vdup_n_u16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_u32() { + fn test_vdup_n_u32() { let v: u32 = 64; let e = u32x2::new(64, 64); - let r: u32x2 = transmute(vdup_n_u32(v)); + let r = u32x2::from(vdup_n_u32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_u64() { + fn test_vdup_n_u64() { let v: u64 = 64; let e = u64x1::new(64); - let r: u64x1 = transmute(vdup_n_u64(v)); + let r = u64x1::from(vdup_n_u64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_p8() { + fn test_vdup_n_p8() { let v: p8 = 64; let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u8x8 = transmute(vdup_n_p8(v)); + let r = u8x8::from(vdup_n_p8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_p16() { + fn test_vdup_n_p16() { let v: p16 = 64; let e = u16x4::new(64, 64, 64, 64); - let r: u16x4 = transmute(vdup_n_p16(v)); + let r = u16x4::from(vdup_n_p16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdup_n_f32() { + fn test_vdup_n_f32() { let v: f32 = 64.0; let e = f32x2::new(64.0, 64.0); - let r: f32x2 = transmute(vdup_n_f32(v)); + let r = f32x2::from(vdup_n_f32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vldrq_p128() { + fn test_vldrq_p128() { let v: [p128; 2] = [1, 2]; let e: p128 = 2; - let r: p128 = vldrq_p128(v[1..].as_ptr()); + let r: p128 = unsafe { vldrq_p128(v[1..].as_ptr()) }; assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vstrq_p128() { + fn test_vstrq_p128() { let v: [p128; 2] = [1, 2]; let e: p128 = 2; let mut r: p128 = 1; - vstrq_p128(&mut r, v[1]); + unsafe { + vstrq_p128(&mut r, v[1]); + } assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_s8() { + fn test_vmov_n_s8() { let v: i8 = 64; let e = i8x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: i8x8 = transmute(vmov_n_s8(v)); + let r = i8x8::from(vmov_n_s8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_s16() { + fn test_vmov_n_s16() { let v: i16 = 64; let e = i16x4::new(64, 64, 64, 64); - let r: i16x4 = transmute(vmov_n_s16(v)); + let r = i16x4::from(vmov_n_s16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_s32() { + fn test_vmov_n_s32() { let v: i32 = 64; let e = i32x2::new(64, 64); - let r: i32x2 = transmute(vmov_n_s32(v)); + let r = i32x2::from(vmov_n_s32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_s64() { + fn test_vmov_n_s64() { let v: i64 = 64; let e = i64x1::new(64); - let r: i64x1 = transmute(vmov_n_s64(v)); + let r = i64x1::from(vmov_n_s64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_u8() { + fn test_vmov_n_u8() { let v: u8 = 64; let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u8x8 = transmute(vmov_n_u8(v)); + let r = u8x8::from(vmov_n_u8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_u16() { + fn test_vmov_n_u16() { let v: u16 = 64; let e = u16x4::new(64, 64, 64, 64); - let r: u16x4 = transmute(vmov_n_u16(v)); + let r = u16x4::from(vmov_n_u16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_u32() { + fn test_vmov_n_u32() { let v: u32 = 64; let e = u32x2::new(64, 64); - let r: u32x2 = transmute(vmov_n_u32(v)); + let r = u32x2::from(vmov_n_u32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_u64() { + fn test_vmov_n_u64() { let v: u64 = 64; let e = u64x1::new(64); - let r: u64x1 = transmute(vmov_n_u64(v)); + let r = u64x1::from(vmov_n_u64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_p8() { + fn test_vmov_n_p8() { let v: p8 = 64; let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u8x8 = transmute(vmov_n_p8(v)); + let r = u8x8::from(vmov_n_p8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_p16() { + fn test_vmov_n_p16() { let v: p16 = 64; let e = u16x4::new(64, 64, 64, 64); - let r: u16x4 = transmute(vmov_n_p16(v)); + let r = u16x4::from(vmov_n_p16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmov_n_f32() { + fn test_vmov_n_f32() { let v: f32 = 64.0; let e = f32x2::new(64.0, 64.0); - let r: f32x2 = transmute(vmov_n_f32(v)); + let r = f32x2::from(vmov_n_f32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_s8() { + fn test_vmovq_n_s8() { let v: i8 = 64; let e = i8x16::new( 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, ); - let r: i8x16 = transmute(vmovq_n_s8(v)); + let r = i8x16::from(vmovq_n_s8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_s16() { + fn test_vmovq_n_s16() { let v: i16 = 64; let e = i16x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: i16x8 = transmute(vmovq_n_s16(v)); + let r = i16x8::from(vmovq_n_s16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_s32() { + fn test_vmovq_n_s32() { let v: i32 = 64; let e = i32x4::new(64, 64, 64, 64); - let r: i32x4 = transmute(vmovq_n_s32(v)); + let r = i32x4::from(vmovq_n_s32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_s64() { + fn test_vmovq_n_s64() { let v: i64 = 64; let e = i64x2::new(64, 64); - let r: i64x2 = transmute(vmovq_n_s64(v)); + let r = i64x2::from(vmovq_n_s64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_u8() { + fn test_vmovq_n_u8() { let v: u8 = 64; let e = u8x16::new( 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, ); - let r: u8x16 = transmute(vmovq_n_u8(v)); + let r = u8x16::from(vmovq_n_u8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_u16() { + fn test_vmovq_n_u16() { let v: u16 = 64; let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u16x8 = transmute(vmovq_n_u16(v)); + let r = u16x8::from(vmovq_n_u16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_u32() { + fn test_vmovq_n_u32() { let v: u32 = 64; let e = u32x4::new(64, 64, 64, 64); - let r: u32x4 = transmute(vmovq_n_u32(v)); + let r = u32x4::from(vmovq_n_u32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_u64() { + fn test_vmovq_n_u64() { let v: u64 = 64; let e = u64x2::new(64, 64); - let r: u64x2 = transmute(vmovq_n_u64(v)); + let r = u64x2::from(vmovq_n_u64(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_p8() { + fn test_vmovq_n_p8() { let v: p8 = 64; let e = u8x16::new( 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, ); - let r: u8x16 = transmute(vmovq_n_p8(v)); + let r = u8x16::from(vmovq_n_p8(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_p16() { + fn test_vmovq_n_p16() { let v: p16 = 64; let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); - let r: u16x8 = transmute(vmovq_n_p16(v)); + let r = u16x8::from(vmovq_n_p16(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_f32() { + fn test_vmovq_n_f32() { let v: f32 = 64.0; let e = f32x4::new(64.0, 64.0, 64.0, 64.0); - let r: f32x4 = transmute(vmovq_n_f32(v)); + let r = f32x4::from(vmovq_n_f32(v)); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u64() { - let v = i64x2::new(1, 2); - let r = vgetq_lane_u64::<1>(transmute(v)); + fn test_vgetq_lane_u64() { + let v = u64x2::new(1, 2); + let r = vgetq_lane_u64::<1>(v.into()); assert_eq!(r, 2); } #[simd_test(enable = "neon")] - unsafe fn test_vadd_s8() { + fn test_vadd_s8() { test_ari_s8( |i, j| vadd_s8(i, j), |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s8() { + fn test_vaddq_s8() { testq_ari_s8( |i, j| vaddq_s8(i, j), |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vadd_s16() { + fn test_vadd_s16() { test_ari_s16( |i, j| vadd_s16(i, j), |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s16() { + fn test_vaddq_s16() { testq_ari_s16( |i, j| vaddq_s16(i, j), |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vadd_s32() { + fn test_vadd_s32() { test_ari_s32( |i, j| vadd_s32(i, j), |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s32() { + fn test_vaddq_s32() { testq_ari_s32( |i, j| vaddq_s32(i, j), |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, @@ -2516,42 +2521,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vadd_u8() { + fn test_vadd_u8() { test_ari_u8( |i, j| vadd_u8(i, j), |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u8() { + fn test_vaddq_u8() { testq_ari_u8( |i, j| vaddq_u8(i, j), |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vadd_u16() { + fn test_vadd_u16() { test_ari_u16( |i, j| vadd_u16(i, j), |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u16() { + fn test_vaddq_u16() { testq_ari_u16( |i, j| vaddq_u16(i, j), |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vadd_u32() { + fn test_vadd_u32() { test_ari_u32( |i, j| vadd_u32(i, j), |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u32() { + fn test_vaddq_u32() { testq_ari_u32( |i, j| vaddq_u32(i, j), |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, @@ -2559,142 +2569,143 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vadd_f32() { + fn test_vadd_f32() { test_ari_f32(|i, j| vadd_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vaddq_f32() { + fn test_vaddq_f32() { testq_ari_f32(|i, j| vaddq_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s8() { + fn test_vaddl_s8() { let v = i8::MAX; let a = i8x8::new(v, v, v, v, v, v, v, v); let v = 2 * (v as i16); let e = i16x8::new(v, v, v, v, v, v, v, v); - let r: i16x8 = transmute(vaddl_s8(transmute(a), transmute(a))); + let r = i16x8::from(vaddl_s8(a.into(), a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s16() { + fn test_vaddl_s16() { let v = i16::MAX; let a = i16x4::new(v, v, v, v); let v = 2 * (v as i32); let e = i32x4::new(v, v, v, v); - let r: i32x4 = transmute(vaddl_s16(transmute(a), transmute(a))); + let r = i32x4::from(vaddl_s16(a.into(), a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s32() { + fn test_vaddl_s32() { let v = i32::MAX; let a = i32x2::new(v, v); let v = 2 * (v as i64); let e = i64x2::new(v, v); - let r: i64x2 = transmute(vaddl_s32(transmute(a), transmute(a))); + let r = i64x2::from(vaddl_s32(a.into(), a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u8() { + fn test_vaddl_u8() { let v = u8::MAX; let a = u8x8::new(v, v, v, v, v, v, v, v); let v = 2 * (v as u16); let e = u16x8::new(v, v, v, v, v, v, v, v); - let r: u16x8 = transmute(vaddl_u8(transmute(a), transmute(a))); + let r = u16x8::from(vaddl_u8(a.into(), a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u16() { + fn test_vaddl_u16() { let v = u16::MAX; let a = u16x4::new(v, v, v, v); let v = 2 * (v as u32); let e = u32x4::new(v, v, v, v); - let r: u32x4 = transmute(vaddl_u16(transmute(a), transmute(a))); + let r = u32x4::from(vaddl_u16(a.into(), a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u32() { + fn test_vaddl_u32() { let v = u32::MAX; let a = u32x2::new(v, v); let v = 2 * (v as u64); let e = u64x2::new(v, v); - let r: u64x2 = transmute(vaddl_u32(transmute(a), transmute(a))); + let r = u64x2::from(vaddl_u32(a.into(), a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_s8() { + fn test_vaddl_high_s8() { let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let x = i8::MAX; let b = i8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); let x = x as i16; let e = i16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); - let r: i16x8 = transmute(vaddl_high_s8(transmute(a), transmute(b))); + let r = i16x8::from(vaddl_high_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_s16() { + fn test_vaddl_high_s16() { let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let x = i16::MAX; let b = i16x8::new(x, x, x, x, x, x, x, x); let x = x as i32; let e = i32x4::new(x + 4, x + 5, x + 6, x + 7); - let r: i32x4 = transmute(vaddl_high_s16(transmute(a), transmute(b))); + let r = i32x4::from(vaddl_high_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_s32() { + fn test_vaddl_high_s32() { let a = i32x4::new(0, 1, 2, 3); let x = i32::MAX; let b = i32x4::new(x, x, x, x); let x = x as i64; let e = i64x2::new(x + 2, x + 3); - let r: i64x2 = transmute(vaddl_high_s32(transmute(a), transmute(b))); + let r = i64x2::from(vaddl_high_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_u8() { + fn test_vaddl_high_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let x = u8::MAX; let b = u8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); let x = x as u16; let e = u16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); - let r: u16x8 = transmute(vaddl_high_u8(transmute(a), transmute(b))); + let r = u16x8::from(vaddl_high_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_u16() { + fn test_vaddl_high_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let x = u16::MAX; let b = u16x8::new(x, x, x, x, x, x, x, x); let x = x as u32; let e = u32x4::new(x + 4, x + 5, x + 6, x + 7); - let r: u32x4 = transmute(vaddl_high_u16(transmute(a), transmute(b))); + let r = u32x4::from(vaddl_high_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_u32() { + fn test_vaddl_high_u32() { let a = u32x4::new(0, 1, 2, 3); let x = u32::MAX; let b = u32x4::new(x, x, x, x); let x = x as u64; let e = u64x2::new(x + 2, x + 3); - let r: u64x2 = transmute(vaddl_high_u32(transmute(a), transmute(b))); + let r = u64x2::from(vaddl_high_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_s8() { + fn test_vaddw_s8() { let x = i16::MAX; let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); let y = i8::MAX; @@ -2710,36 +2721,36 @@ mod tests { 6 + y, 7 + y, ); - let r: i16x8 = transmute(vaddw_s8(transmute(a), transmute(b))); + let r = i16x8::from(vaddw_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_s16() { + fn test_vaddw_s16() { let x = i32::MAX; let a = i32x4::new(x, 1, 2, 3); let y = i16::MAX; let b = i16x4::new(y, y, y, y); let y = y as i32; let e = i32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); - let r: i32x4 = transmute(vaddw_s16(transmute(a), transmute(b))); + let r = i32x4::from(vaddw_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_s32() { + fn test_vaddw_s32() { let x = i64::MAX; let a = i64x2::new(x, 1); let y = i32::MAX; let b = i32x2::new(y, y); let y = y as i64; let e = i64x2::new(x.wrapping_add(y), 1 + y); - let r: i64x2 = transmute(vaddw_s32(transmute(a), transmute(b))); + let r = i64x2::from(vaddw_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_u8() { + fn test_vaddw_u8() { let x = u16::MAX; let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); let y = u8::MAX; @@ -2755,36 +2766,36 @@ mod tests { 6 + y, 7 + y, ); - let r: u16x8 = transmute(vaddw_u8(transmute(a), transmute(b))); + let r = u16x8::from(vaddw_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_u16() { + fn test_vaddw_u16() { let x = u32::MAX; let a = u32x4::new(x, 1, 2, 3); let y = u16::MAX; let b = u16x4::new(y, y, y, y); let y = y as u32; let e = u32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); - let r: u32x4 = transmute(vaddw_u16(transmute(a), transmute(b))); + let r = u32x4::from(vaddw_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_u32() { + fn test_vaddw_u32() { let x = u64::MAX; let a = u64x2::new(x, 1); let y = u32::MAX; let b = u32x2::new(y, y); let y = y as u64; let e = u64x2::new(x.wrapping_add(y), 1 + y); - let r: u64x2 = transmute(vaddw_u32(transmute(a), transmute(b))); + let r = u64x2::from(vaddw_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_s8() { + fn test_vaddw_high_s8() { let x = i16::MAX; let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); let y = i8::MAX; @@ -2800,36 +2811,36 @@ mod tests { 6 + y, 7 + y, ); - let r: i16x8 = transmute(vaddw_high_s8(transmute(a), transmute(b))); + let r = i16x8::from(vaddw_high_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_s16() { + fn test_vaddw_high_s16() { let x = i32::MAX; let a = i32x4::new(x, 1, 2, 3); let y = i16::MAX; let b = i16x8::new(0, 0, 0, 0, y, y, y, y); let y = y as i32; let e = i32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); - let r: i32x4 = transmute(vaddw_high_s16(transmute(a), transmute(b))); + let r = i32x4::from(vaddw_high_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_s32() { + fn test_vaddw_high_s32() { let x = i64::MAX; let a = i64x2::new(x, 1); let y = i32::MAX; let b = i32x4::new(0, 0, y, y); let y = y as i64; let e = i64x2::new(x.wrapping_add(y), 1 + y); - let r: i64x2 = transmute(vaddw_high_s32(transmute(a), transmute(b))); + let r = i64x2::from(vaddw_high_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_u8() { + fn test_vaddw_high_u8() { let x = u16::MAX; let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); let y = u8::MAX; @@ -2845,165 +2856,165 @@ mod tests { 6 + y, 7 + y, ); - let r: u16x8 = transmute(vaddw_high_u8(transmute(a), transmute(b))); + let r = u16x8::from(vaddw_high_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_u16() { + fn test_vaddw_high_u16() { let x = u32::MAX; let a = u32x4::new(x, 1, 2, 3); let y = u16::MAX; let b = u16x8::new(0, 0, 0, 0, y, y, y, y); let y = y as u32; let e = u32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); - let r: u32x4 = transmute(vaddw_high_u16(transmute(a), transmute(b))); + let r = u32x4::from(vaddw_high_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_u32() { + fn test_vaddw_high_u32() { let x = u64::MAX; let a = u64x2::new(x, 1); let y = u32::MAX; let b = u32x4::new(0, 0, y, y); let y = y as u64; let e = u64x2::new(x.wrapping_add(y), 1 + y); - let r: u64x2 = transmute(vaddw_high_u32(transmute(a), transmute(b))); + let r = u64x2::from(vaddw_high_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s8() { + fn test_vmvn_s8() { let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let e = i8x8::new(-1, -2, -3, -4, -5, -6, -7, -8); - let r: i8x8 = transmute(vmvn_s8(transmute(a))); + let r = i8x8::from(vmvn_s8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s8() { + fn test_vmvnq_s8() { let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let e = i8x16::new( -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, ); - let r: i8x16 = transmute(vmvnq_s8(transmute(a))); + let r = i8x16::from(vmvnq_s8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s16() { + fn test_vmvn_s16() { let a = i16x4::new(0, 1, 2, 3); let e = i16x4::new(-1, -2, -3, -4); - let r: i16x4 = transmute(vmvn_s16(transmute(a))); + let r = i16x4::from(vmvn_s16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s16() { + fn test_vmvnq_s16() { let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let e = i16x8::new(-1, -2, -3, -4, -5, -6, -7, -8); - let r: i16x8 = transmute(vmvnq_s16(transmute(a))); + let r = i16x8::from(vmvnq_s16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s32() { + fn test_vmvn_s32() { let a = i32x2::new(0, 1); let e = i32x2::new(-1, -2); - let r: i32x2 = transmute(vmvn_s32(transmute(a))); + let r = i32x2::from(vmvn_s32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s32() { + fn test_vmvnq_s32() { let a = i32x4::new(0, 1, 2, 3); let e = i32x4::new(-1, -2, -3, -4); - let r: i32x4 = transmute(vmvnq_s32(transmute(a))); + let r = i32x4::from(vmvnq_s32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u8() { + fn test_vmvn_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); - let r: u8x8 = transmute(vmvn_u8(transmute(a))); + let r = u8x8::from(vmvn_u8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u8() { + fn test_vmvnq_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let e = u8x16::new( 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, ); - let r: u8x16 = transmute(vmvnq_u8(transmute(a))); + let r = u8x16::from(vmvnq_u8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u16() { + fn test_vmvn_u16() { let a = u16x4::new(0, 1, 2, 3); let e = u16x4::new(65_535, 65_534, 65_533, 65_532); - let r: u16x4 = transmute(vmvn_u16(transmute(a))); + let r = u16x4::from(vmvn_u16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u16() { + fn test_vmvnq_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let e = u16x8::new( 65_535, 65_534, 65_533, 65_532, 65_531, 65_530, 65_529, 65_528, ); - let r: u16x8 = transmute(vmvnq_u16(transmute(a))); + let r = u16x8::from(vmvnq_u16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u32() { + fn test_vmvn_u32() { let a = u32x2::new(0, 1); let e = u32x2::new(4_294_967_295, 4_294_967_294); - let r: u32x2 = transmute(vmvn_u32(transmute(a))); + let r = u32x2::from(vmvn_u32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u32() { + fn test_vmvnq_u32() { let a = u32x4::new(0, 1, 2, 3); let e = u32x4::new(4_294_967_295, 4_294_967_294, 4_294_967_293, 4_294_967_292); - let r: u32x4 = transmute(vmvnq_u32(transmute(a))); + let r = u32x4::from(vmvnq_u32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_p8() { + fn test_vmvn_p8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); - let r: u8x8 = transmute(vmvn_p8(transmute(a))); + let r = u8x8::from(vmvn_p8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_p8() { + fn test_vmvnq_p8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let e = u8x16::new( 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, ); - let r: u8x16 = transmute(vmvnq_p8(transmute(a))); + let r = u8x16::from(vmvnq_p8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_s8() { + fn test_vbic_s8() { let a = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); let b = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); let e = i8x8::new(0, -2, -2, -4, -4, -6, -6, -8); - let r: i8x8 = transmute(vbic_s8(transmute(a), transmute(b))); + let r = i8x8::from(vbic_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_s8() { + fn test_vbicq_s8() { let a = i8x16::new( 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, ); @@ -3011,138 +3022,138 @@ mod tests { let e = i8x16::new( 0, -2, -2, -4, -4, -6, -6, -8, -8, -10, -10, -12, -12, -14, -14, -16, ); - let r: i8x16 = transmute(vbicq_s8(transmute(a), transmute(b))); + let r = i8x16::from(vbicq_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_s16() { + fn test_vbic_s16() { let a = i16x4::new(0, -1, -2, -3); let b = i16x4::new(1, 1, 1, 1); let e = i16x4::new(0, -2, -2, -4); - let r: i16x4 = transmute(vbic_s16(transmute(a), transmute(b))); + let r = i16x4::from(vbic_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_s16() { + fn test_vbicq_s16() { let a = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); let b = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); let e = i16x8::new(0, -2, -2, -4, -4, -6, -6, -8); - let r: i16x8 = transmute(vbicq_s16(transmute(a), transmute(b))); + let r = i16x8::from(vbicq_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_s32() { + fn test_vbic_s32() { let a = i32x2::new(0, -1); let b = i32x2::new(1, 1); let e = i32x2::new(0, -2); - let r: i32x2 = transmute(vbic_s32(transmute(a), transmute(b))); + let r = i32x2::from(vbic_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_s32() { + fn test_vbicq_s32() { let a = i32x4::new(0, -1, -2, -3); let b = i32x4::new(1, 1, 1, 1); let e = i32x4::new(0, -2, -2, -4); - let r: i32x4 = transmute(vbicq_s32(transmute(a), transmute(b))); + let r = i32x4::from(vbicq_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_s64() { + fn test_vbic_s64() { let a = i64x1::new(-1); let b = i64x1::new(1); let e = i64x1::new(-2); - let r: i64x1 = transmute(vbic_s64(transmute(a), transmute(b))); + let r = i64x1::from(vbic_s64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_s64() { + fn test_vbicq_s64() { let a = i64x2::new(0, -1); let b = i64x2::new(1, 1); let e = i64x2::new(0, -2); - let r: i64x2 = transmute(vbicq_s64(transmute(a), transmute(b))); + let r = i64x2::from(vbicq_s64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_u8() { + fn test_vbic_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let b = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); let e = u8x8::new(0, 0, 2, 2, 4, 4, 6, 6); - let r: u8x8 = transmute(vbic_u8(transmute(a), transmute(b))); + let r = u8x8::from(vbic_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_u8() { + fn test_vbicq_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let b = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let e = u8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); - let r: u8x16 = transmute(vbicq_u8(transmute(a), transmute(b))); + let r = u8x16::from(vbicq_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_u16() { + fn test_vbic_u16() { let a = u16x4::new(0, 1, 2, 3); let b = u16x4::new(1, 1, 1, 1); let e = u16x4::new(0, 0, 2, 2); - let r: u16x4 = transmute(vbic_u16(transmute(a), transmute(b))); + let r = u16x4::from(vbic_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_u16() { + fn test_vbicq_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let b = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); let e = u16x8::new(0, 0, 2, 2, 4, 4, 6, 6); - let r: u16x8 = transmute(vbicq_u16(transmute(a), transmute(b))); + let r = u16x8::from(vbicq_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_u32() { + fn test_vbic_u32() { let a = u32x2::new(0, 1); let b = u32x2::new(1, 1); let e = u32x2::new(0, 0); - let r: u32x2 = transmute(vbic_u32(transmute(a), transmute(b))); + let r = u32x2::from(vbic_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_u32() { + fn test_vbicq_u32() { let a = u32x4::new(0, 1, 2, 3); let b = u32x4::new(1, 1, 1, 1); let e = u32x4::new(0, 0, 2, 2); - let r: u32x4 = transmute(vbicq_u32(transmute(a), transmute(b))); + let r = u32x4::from(vbicq_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbic_u64() { + fn test_vbic_u64() { let a = u64x1::new(1); let b = u64x1::new(1); let e = u64x1::new(0); - let r: u64x1 = transmute(vbic_u64(transmute(a), transmute(b))); + let r = u64x1::from(vbic_u64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbicq_u64() { + fn test_vbicq_u64() { let a = u64x2::new(0, 1); let b = u64x2::new(1, 1); let e = u64x2::new(0, 0); - let r: u64x2 = transmute(vbicq_u64(transmute(a), transmute(b))); + let r = u64x2::from(vbicq_u64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vbsl_s8() { + fn test_vbsl_s8() { let a = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, 0, u8::MAX, 0); let b = i8x8::new( i8::MAX, @@ -3174,38 +3185,42 @@ mod tests { i8::MAX, i8::MIN, ); - let r: i8x8 = transmute(vbsl_s8(transmute(a), transmute(b), transmute(c))); + let r = i8x8::from(vbsl_s8(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_s16() { + fn test_vbsl_s16() { let a = u16x4::new(u16::MAX, 0, 1, 2); let b = i16x4::new(i16::MAX, i16::MAX, i16::MAX, i16::MAX); let c = i16x4::new(i16::MIN, i16::MIN, i16::MIN, i16::MIN); let e = i16x4::new(i16::MAX, i16::MIN, i16::MIN | 1, i16::MIN | 2); - let r: i16x4 = transmute(vbsl_s16(transmute(a), transmute(b), transmute(c))); + let r = i16x4::from(vbsl_s16(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_s32() { + fn test_vbsl_s32() { let a = u32x2::new(u32::MAX, 1); let b = i32x2::new(i32::MAX, i32::MAX); let c = i32x2::new(i32::MIN, i32::MIN); let e = i32x2::new(i32::MAX, i32::MIN | 1); - let r: i32x2 = transmute(vbsl_s32(transmute(a), transmute(b), transmute(c))); + let r = i32x2::from(vbsl_s32(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_s64() { + fn test_vbsl_s64() { let a = u64x1::new(1); let b = i64x1::new(i64::MAX); let c = i64x1::new(i64::MIN); let e = i64x1::new(i64::MIN | 1); - let r: i64x1 = transmute(vbsl_s64(transmute(a), transmute(b), transmute(c))); + let r = i64x1::from(vbsl_s64(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_u8() { + fn test_vbsl_u8() { let a = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, 0, u8::MAX, 0); let b = u8x8::new( u8::MAX, @@ -3228,47 +3243,52 @@ mod tests { u8::MIN, ); let e = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, u8::MIN, u8::MAX, u8::MIN); - let r: u8x8 = transmute(vbsl_u8(transmute(a), transmute(b), transmute(c))); + let r = u8x8::from(vbsl_u8(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_u16() { + fn test_vbsl_u16() { let a = u16x4::new(u16::MAX, 0, 1, 2); let b = u16x4::new(u16::MAX, u16::MAX, u16::MAX, u16::MAX); let c = u16x4::new(u16::MIN, u16::MIN, u16::MIN, u16::MIN); let e = u16x4::new(u16::MAX, 0, 1, 2); - let r: u16x4 = transmute(vbsl_u16(transmute(a), transmute(b), transmute(c))); + let r = u16x4::from(vbsl_u16(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_u32() { + fn test_vbsl_u32() { let a = u32x2::new(u32::MAX, 2); let b = u32x2::new(u32::MAX, u32::MAX); let c = u32x2::new(u32::MIN, u32::MIN); let e = u32x2::new(u32::MAX, 2); - let r: u32x2 = transmute(vbsl_u32(transmute(a), transmute(b), transmute(c))); + let r = u32x2::from(vbsl_u32(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_u64() { + fn test_vbsl_u64() { let a = u64x1::new(2); let b = u64x1::new(u64::MAX); let c = u64x1::new(u64::MIN); let e = u64x1::new(2); - let r: u64x1 = transmute(vbsl_u64(transmute(a), transmute(b), transmute(c))); + let r = u64x1::from(vbsl_u64(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_f32() { + fn test_vbsl_f32() { let a = u32x2::new(1, 0x80000000); let b = f32x2::new(8388609f32, -1.23f32); let c = f32x2::new(2097152f32, 2.34f32); let e = f32x2::new(2097152.25f32, -2.34f32); - let r: f32x2 = transmute(vbsl_f32(transmute(a), transmute(b), transmute(c))); + let r = f32x2::from(vbsl_f32(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_p8() { + fn test_vbsl_p8() { let a = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, 0, u8::MAX, 0); let b = u8x8::new( u8::MAX, @@ -3291,20 +3311,22 @@ mod tests { u8::MIN, ); let e = u8x8::new(u8::MAX, 1, u8::MAX, 2, u8::MAX, u8::MIN, u8::MAX, u8::MIN); - let r: u8x8 = transmute(vbsl_p8(transmute(a), transmute(b), transmute(c))); + let r = u8x8::from(vbsl_p8(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbsl_p16() { + fn test_vbsl_p16() { let a = u16x4::new(u16::MAX, 0, 1, 2); let b = u16x4::new(u16::MAX, u16::MAX, u16::MAX, u16::MAX); let c = u16x4::new(u16::MIN, u16::MIN, u16::MIN, u16::MIN); let e = u16x4::new(u16::MAX, 0, 1, 2); - let r: u16x4 = transmute(vbsl_p16(transmute(a), transmute(b), transmute(c))); + let r = u16x4::from(vbsl_p16(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_s8() { + fn test_vbslq_s8() { let a = u8x16::new( u8::MAX, 1, @@ -3377,11 +3399,12 @@ mod tests { i8::MAX, i8::MIN, ); - let r: i8x16 = transmute(vbslq_s8(transmute(a), transmute(b), transmute(c))); + let r = i8x16::from(vbslq_s8(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_s16() { + fn test_vbslq_s16() { let a = u16x8::new(u16::MAX, 1, u16::MAX, 2, u16::MAX, 0, u16::MAX, 0); let b = i16x8::new( i16::MAX, @@ -3413,29 +3436,32 @@ mod tests { i16::MAX, i16::MIN, ); - let r: i16x8 = transmute(vbslq_s16(transmute(a), transmute(b), transmute(c))); + let r = i16x8::from(vbslq_s16(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_s32() { + fn test_vbslq_s32() { let a = u32x4::new(u32::MAX, 1, u32::MAX, 2); let b = i32x4::new(i32::MAX, i32::MAX, i32::MAX, i32::MAX); let c = i32x4::new(i32::MIN, i32::MIN, i32::MIN, i32::MIN); let e = i32x4::new(i32::MAX, i32::MIN | 1, i32::MAX, i32::MIN | 2); - let r: i32x4 = transmute(vbslq_s32(transmute(a), transmute(b), transmute(c))); + let r = i32x4::from(vbslq_s32(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_s64() { + fn test_vbslq_s64() { let a = u64x2::new(u64::MAX, 1); let b = i64x2::new(i64::MAX, i64::MAX); let c = i64x2::new(i64::MIN, i64::MIN); let e = i64x2::new(i64::MAX, i64::MIN | 1); - let r: i64x2 = transmute(vbslq_s64(transmute(a), transmute(b), transmute(c))); + let r = i64x2::from(vbslq_s64(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_u8() { + fn test_vbslq_u8() { let a = u8x16::new( u8::MAX, 1, @@ -3508,11 +3534,12 @@ mod tests { u8::MAX, u8::MIN, ); - let r: u8x16 = transmute(vbslq_u8(transmute(a), transmute(b), transmute(c))); + let r = u8x16::from(vbslq_u8(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_u16() { + fn test_vbslq_u16() { let a = u16x8::new(u16::MAX, 1, u16::MAX, 2, u16::MAX, 0, u16::MAX, 0); let b = u16x8::new( u16::MAX, @@ -3544,38 +3571,42 @@ mod tests { u16::MAX, u16::MIN, ); - let r: u16x8 = transmute(vbslq_u16(transmute(a), transmute(b), transmute(c))); + let r = u16x8::from(vbslq_u16(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_u32() { + fn test_vbslq_u32() { let a = u32x4::new(u32::MAX, 1, u32::MAX, 2); let b = u32x4::new(u32::MAX, u32::MAX, u32::MAX, u32::MAX); let c = u32x4::new(u32::MIN, u32::MIN, u32::MIN, u32::MIN); let e = u32x4::new(u32::MAX, 1, u32::MAX, 2); - let r: u32x4 = transmute(vbslq_u32(transmute(a), transmute(b), transmute(c))); + let r = u32x4::from(vbslq_u32(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_u64() { + fn test_vbslq_u64() { let a = u64x2::new(u64::MAX, 1); let b = u64x2::new(u64::MAX, u64::MAX); let c = u64x2::new(u64::MIN, u64::MIN); let e = u64x2::new(u64::MAX, 1); - let r: u64x2 = transmute(vbslq_u64(transmute(a), transmute(b), transmute(c))); + let r = u64x2::from(vbslq_u64(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_f32() { + fn test_vbslq_f32() { let a = u32x4::new(u32::MAX, 0, 1, 0x80000000); let b = f32x4::new(-1.23f32, -1.23f32, 8388609f32, -1.23f32); let c = f32x4::new(2.34f32, 2.34f32, 2097152f32, 2.34f32); let e = f32x4::new(-1.23f32, 2.34f32, 2097152.25f32, -2.34f32); - let r: f32x4 = transmute(vbslq_f32(transmute(a), transmute(b), transmute(c))); + let r = f32x4::from(vbslq_f32(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_p8() { + fn test_vbslq_p8() { let a = u8x16::new( u8::MAX, 1, @@ -3648,11 +3679,12 @@ mod tests { u8::MAX, u8::MIN, ); - let r: u8x16 = transmute(vbslq_p8(transmute(a), transmute(b), transmute(c))); + let r = u8x16::from(vbslq_p8(a.into(), b.into(), c.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vbslq_p16() { + fn test_vbslq_p16() { let a = u16x8::new(u16::MAX, 1, u16::MAX, 2, u16::MAX, 0, u16::MAX, 0); let b = u16x8::new( u16::MAX, @@ -3684,21 +3716,21 @@ mod tests { u16::MAX, u16::MIN, ); - let r: u16x8 = transmute(vbslq_p16(transmute(a), transmute(b), transmute(c))); + let r = u16x8::from(vbslq_p16(a.into(), b.into(), c.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_s8() { + fn test_vorn_s8() { let a = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); let b = i8x8::new(-2, -2, -2, -2, -2, -2, -2, -2); let e = i8x8::new(1, -1, -1, -3, -3, -5, -5, -7); - let r: i8x8 = transmute(vorn_s8(transmute(a), transmute(b))); + let r = i8x8::from(vorn_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_s8() { + fn test_vornq_s8() { let a = i8x16::new( 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, ); @@ -3708,475 +3740,522 @@ mod tests { let e = i8x16::new( 1, -1, -1, -3, -3, -5, -5, -7, -7, -9, -9, -11, -11, -13, -13, -15, ); - let r: i8x16 = transmute(vornq_s8(transmute(a), transmute(b))); + let r = i8x16::from(vornq_s8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_s16() { + fn test_vorn_s16() { let a = i16x4::new(0, -1, -2, -3); let b = i16x4::new(-2, -2, -2, -2); let e = i16x4::new(1, -1, -1, -3); - let r: i16x4 = transmute(vorn_s16(transmute(a), transmute(b))); + let r = i16x4::from(vorn_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_s16() { + fn test_vornq_s16() { let a = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); let b = i16x8::new(-2, -2, -2, -2, -2, -2, -2, -2); let e = i16x8::new(1, -1, -1, -3, -3, -5, -5, -7); - let r: i16x8 = transmute(vornq_s16(transmute(a), transmute(b))); + let r = i16x8::from(vornq_s16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_s32() { + fn test_vorn_s32() { let a = i32x2::new(0, -1); let b = i32x2::new(-2, -2); let e = i32x2::new(1, -1); - let r: i32x2 = transmute(vorn_s32(transmute(a), transmute(b))); + let r = i32x2::from(vorn_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_s32() { + fn test_vornq_s32() { let a = i32x4::new(0, -1, -2, -3); let b = i32x4::new(-2, -2, -2, -2); let e = i32x4::new(1, -1, -1, -3); - let r: i32x4 = transmute(vornq_s32(transmute(a), transmute(b))); + let r = i32x4::from(vornq_s32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_s64() { + fn test_vorn_s64() { let a = i64x1::new(0); let b = i64x1::new(-2); let e = i64x1::new(1); - let r: i64x1 = transmute(vorn_s64(transmute(a), transmute(b))); + let r = i64x1::from(vorn_s64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_s64() { + fn test_vornq_s64() { let a = i64x2::new(0, -1); let b = i64x2::new(-2, -2); let e = i64x2::new(1, -1); - let r: i64x2 = transmute(vornq_s64(transmute(a), transmute(b))); + let r = i64x2::from(vornq_s64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_u8() { + fn test_vorn_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let t = u8::MAX - 1; let b = u8x8::new(t, t, t, t, t, t, t, t); let e = u8x8::new(1, 1, 3, 3, 5, 5, 7, 7); - let r: u8x8 = transmute(vorn_u8(transmute(a), transmute(b))); + let r = u8x8::from(vorn_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_u8() { + fn test_vornq_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let t = u8::MAX - 1; let b = u8x16::new(t, t, t, t, t, t, t, t, t, t, t, t, t, t, t, t); let e = u8x16::new(1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15); - let r: u8x16 = transmute(vornq_u8(transmute(a), transmute(b))); + let r = u8x16::from(vornq_u8(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_u16() { + fn test_vorn_u16() { let a = u16x4::new(0, 1, 2, 3); let t = u16::MAX - 1; let b = u16x4::new(t, t, t, t); let e = u16x4::new(1, 1, 3, 3); - let r: u16x4 = transmute(vorn_u16(transmute(a), transmute(b))); + let r = u16x4::from(vorn_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_u16() { + fn test_vornq_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let t = u16::MAX - 1; let b = u16x8::new(t, t, t, t, t, t, t, t); let e = u16x8::new(1, 1, 3, 3, 5, 5, 7, 7); - let r: u16x8 = transmute(vornq_u16(transmute(a), transmute(b))); + let r = u16x8::from(vornq_u16(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_u32() { + fn test_vorn_u32() { let a = u32x2::new(0, 1); let t = u32::MAX - 1; let b = u32x2::new(t, t); let e = u32x2::new(1, 1); - let r: u32x2 = transmute(vorn_u32(transmute(a), transmute(b))); + let r = u32x2::from(vorn_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_u32() { + fn test_vornq_u32() { let a = u32x4::new(0, 1, 2, 3); let t = u32::MAX - 1; let b = u32x4::new(t, t, t, t); let e = u32x4::new(1, 1, 3, 3); - let r: u32x4 = transmute(vornq_u32(transmute(a), transmute(b))); + let r = u32x4::from(vornq_u32(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vorn_u64() { + fn test_vorn_u64() { let a = u64x1::new(0); let t = u64::MAX - 1; let b = u64x1::new(t); let e = u64x1::new(1); - let r: u64x1 = transmute(vorn_u64(transmute(a), transmute(b))); + let r = u64x1::from(vorn_u64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vornq_u64() { + fn test_vornq_u64() { let a = u64x2::new(0, 1); let t = u64::MAX - 1; let b = u64x2::new(t, t); let e = u64x2::new(1, 1); - let r: u64x2 = transmute(vornq_u64(transmute(a), transmute(b))); + let r = u64x2::from(vornq_u64(a.into(), b.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovn_s16() { + fn test_vmovn_s16() { let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i8x8 = transmute(vmovn_s16(transmute(a))); + let r = i8x8::from(vmovn_s16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovn_s32() { + fn test_vmovn_s32() { let a = i32x4::new(1, 2, 3, 4); let e = i16x4::new(1, 2, 3, 4); - let r: i16x4 = transmute(vmovn_s32(transmute(a))); + let r = i16x4::from(vmovn_s32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovn_s64() { + fn test_vmovn_s64() { let a = i64x2::new(1, 2); let e = i32x2::new(1, 2); - let r: i32x2 = transmute(vmovn_s64(transmute(a))); + let r = i32x2::from(vmovn_s64(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovn_u16() { + fn test_vmovn_u16() { let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vmovn_u16(transmute(a))); + let r = u8x8::from(vmovn_u16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovn_u32() { + fn test_vmovn_u32() { let a = u32x4::new(1, 2, 3, 4); let e = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vmovn_u32(transmute(a))); + let r = u16x4::from(vmovn_u32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovn_u64() { + fn test_vmovn_u64() { let a = u64x2::new(1, 2); let e = u32x2::new(1, 2); - let r: u32x2 = transmute(vmovn_u64(transmute(a))); + let r = u32x2::from(vmovn_u64(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_s8() { + fn test_vmovl_s8() { let e = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i16x8 = transmute(vmovl_s8(transmute(a))); + let r = i16x8::from(vmovl_s8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_s16() { + fn test_vmovl_s16() { let e = i32x4::new(1, 2, 3, 4); let a = i16x4::new(1, 2, 3, 4); - let r: i32x4 = transmute(vmovl_s16(transmute(a))); + let r = i32x4::from(vmovl_s16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_s32() { + fn test_vmovl_s32() { let e = i64x2::new(1, 2); let a = i32x2::new(1, 2); - let r: i64x2 = transmute(vmovl_s32(transmute(a))); + let r = i64x2::from(vmovl_s32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_u8() { + fn test_vmovl_u8() { let e = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u16x8 = transmute(vmovl_u8(transmute(a))); + let r = u16x8::from(vmovl_u8(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_u16() { + fn test_vmovl_u16() { let e = u32x4::new(1, 2, 3, 4); let a = u16x4::new(1, 2, 3, 4); - let r: u32x4 = transmute(vmovl_u16(transmute(a))); + let r = u32x4::from(vmovl_u16(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovl_u32() { + fn test_vmovl_u32() { let e = u64x2::new(1, 2); let a = u32x2::new(1, 2); - let r: u64x2 = transmute(vmovl_u32(transmute(a))); + let r = u64x2::from(vmovl_u32(a.into())); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vand_s8() { + fn test_vand_s8() { test_bit_s8(|i, j| vand_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_s8() { + fn test_vandq_s8() { testq_bit_s8(|i, j| vandq_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vand_s16() { + fn test_vand_s16() { test_bit_s16(|i, j| vand_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_s16() { + fn test_vandq_s16() { testq_bit_s16(|i, j| vandq_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vand_s32() { + fn test_vand_s32() { test_bit_s32(|i, j| vand_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_s32() { + fn test_vandq_s32() { testq_bit_s32(|i, j| vandq_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vand_s64() { + fn test_vand_s64() { test_bit_s64(|i, j| vand_s64(i, j), |a: i64, b: i64| -> i64 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_s64() { + fn test_vandq_s64() { testq_bit_s64(|i, j| vandq_s64(i, j), |a: i64, b: i64| -> i64 { a & b }); } #[simd_test(enable = "neon")] - unsafe fn test_vand_u8() { + fn test_vand_u8() { test_bit_u8(|i, j| vand_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_u8() { + fn test_vandq_u8() { testq_bit_u8(|i, j| vandq_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vand_u16() { + fn test_vand_u16() { test_bit_u16(|i, j| vand_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_u16() { + fn test_vandq_u16() { testq_bit_u16(|i, j| vandq_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vand_u32() { + fn test_vand_u32() { test_bit_u32(|i, j| vand_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_u32() { + fn test_vandq_u32() { testq_bit_u32(|i, j| vandq_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vand_u64() { + fn test_vand_u64() { test_bit_u64(|i, j| vand_u64(i, j), |a: u64, b: u64| -> u64 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vandq_u64() { + fn test_vandq_u64() { testq_bit_u64(|i, j| vandq_u64(i, j), |a: u64, b: u64| -> u64 { a & b }); } #[simd_test(enable = "neon")] - unsafe fn test_vorr_s8() { + fn test_vorr_s8() { test_bit_s8(|i, j| vorr_s8(i, j), |a: i8, b: i8| -> i8 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s8() { + fn test_vorrq_s8() { testq_bit_s8(|i, j| vorrq_s8(i, j), |a: i8, b: i8| -> i8 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorr_s16() { + fn test_vorr_s16() { test_bit_s16(|i, j| vorr_s16(i, j), |a: i16, b: i16| -> i16 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s16() { + fn test_vorrq_s16() { testq_bit_s16(|i, j| vorrq_s16(i, j), |a: i16, b: i16| -> i16 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorr_s32() { + fn test_vorr_s32() { test_bit_s32(|i, j| vorr_s32(i, j), |a: i32, b: i32| -> i32 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s32() { + fn test_vorrq_s32() { testq_bit_s32(|i, j| vorrq_s32(i, j), |a: i32, b: i32| -> i32 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorr_s64() { + fn test_vorr_s64() { test_bit_s64(|i, j| vorr_s64(i, j), |a: i64, b: i64| -> i64 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s64() { + fn test_vorrq_s64() { testq_bit_s64(|i, j| vorrq_s64(i, j), |a: i64, b: i64| -> i64 { a | b }); } #[simd_test(enable = "neon")] - unsafe fn test_vorr_u8() { + fn test_vorr_u8() { test_bit_u8(|i, j| vorr_u8(i, j), |a: u8, b: u8| -> u8 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u8() { + fn test_vorrq_u8() { testq_bit_u8(|i, j| vorrq_u8(i, j), |a: u8, b: u8| -> u8 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorr_u16() { + fn test_vorr_u16() { test_bit_u16(|i, j| vorr_u16(i, j), |a: u16, b: u16| -> u16 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u16() { + fn test_vorrq_u16() { testq_bit_u16(|i, j| vorrq_u16(i, j), |a: u16, b: u16| -> u16 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorr_u32() { + fn test_vorr_u32() { test_bit_u32(|i, j| vorr_u32(i, j), |a: u32, b: u32| -> u32 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u32() { + fn test_vorrq_u32() { testq_bit_u32(|i, j| vorrq_u32(i, j), |a: u32, b: u32| -> u32 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorr_u64() { + fn test_vorr_u64() { test_bit_u64(|i, j| vorr_u64(i, j), |a: u64, b: u64| -> u64 { a | b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u64() { + fn test_vorrq_u64() { testq_bit_u64(|i, j| vorrq_u64(i, j), |a: u64, b: u64| -> u64 { a | b }); } #[simd_test(enable = "neon")] - unsafe fn test_veor_s8() { + fn test_veor_s8() { test_bit_s8(|i, j| veor_s8(i, j), |a: i8, b: i8| -> i8 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_s8() { + fn test_veorq_s8() { testq_bit_s8(|i, j| veorq_s8(i, j), |a: i8, b: i8| -> i8 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veor_s16() { + fn test_veor_s16() { test_bit_s16(|i, j| veor_s16(i, j), |a: i16, b: i16| -> i16 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_s16() { + fn test_veorq_s16() { testq_bit_s16(|i, j| veorq_s16(i, j), |a: i16, b: i16| -> i16 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veor_s32() { + fn test_veor_s32() { test_bit_s32(|i, j| veor_s32(i, j), |a: i32, b: i32| -> i32 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_s32() { + fn test_veorq_s32() { testq_bit_s32(|i, j| veorq_s32(i, j), |a: i32, b: i32| -> i32 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veor_s64() { + fn test_veor_s64() { test_bit_s64(|i, j| veor_s64(i, j), |a: i64, b: i64| -> i64 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_s64() { + fn test_veorq_s64() { testq_bit_s64(|i, j| veorq_s64(i, j), |a: i64, b: i64| -> i64 { a ^ b }); } #[simd_test(enable = "neon")] - unsafe fn test_veor_u8() { + fn test_veor_u8() { test_bit_u8(|i, j| veor_u8(i, j), |a: u8, b: u8| -> u8 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_u8() { + fn test_veorq_u8() { testq_bit_u8(|i, j| veorq_u8(i, j), |a: u8, b: u8| -> u8 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veor_u16() { + fn test_veor_u16() { test_bit_u16(|i, j| veor_u16(i, j), |a: u16, b: u16| -> u16 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_u16() { + fn test_veorq_u16() { testq_bit_u16(|i, j| veorq_u16(i, j), |a: u16, b: u16| -> u16 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veor_u32() { + fn test_veor_u32() { test_bit_u32(|i, j| veor_u32(i, j), |a: u32, b: u32| -> u32 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_u32() { + fn test_veorq_u32() { testq_bit_u32(|i, j| veorq_u32(i, j), |a: u32, b: u32| -> u32 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veor_u64() { + fn test_veor_u64() { test_bit_u64(|i, j| veor_u64(i, j), |a: u64, b: u64| -> u64 { a ^ b }); } + #[simd_test(enable = "neon")] - unsafe fn test_veorq_u64() { + fn test_veorq_u64() { testq_bit_u64(|i, j| veorq_u64(i, j), |a: u64, b: u64| -> u64 { a ^ b }); } #[simd_test(enable = "neon")] - unsafe fn test_vceq_s8() { + fn test_vceq_s8() { test_cmp_s8( |i, j| vceq_s8(i, j), |a: i8, b: i8| -> u8 { if a == b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s8() { + fn test_vceqq_s8() { testq_cmp_s8( |i, j| vceqq_s8(i, j), |a: i8, b: i8| -> u8 { if a == b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceq_s16() { + fn test_vceq_s16() { test_cmp_s16( |i, j| vceq_s16(i, j), |a: i16, b: i16| -> u16 { if a == b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s16() { + fn test_vceqq_s16() { testq_cmp_s16( |i, j| vceqq_s16(i, j), |a: i16, b: i16| -> u16 { if a == b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceq_s32() { + fn test_vceq_s32() { test_cmp_s32( |i, j| vceq_s32(i, j), |a: i32, b: i32| -> u32 { if a == b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s32() { + fn test_vceqq_s32() { testq_cmp_s32( |i, j| vceqq_s32(i, j), |a: i32, b: i32| -> u32 { if a == b { 0xFFFFFFFF } else { 0 } }, @@ -4184,42 +4263,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vceq_u8() { + fn test_vceq_u8() { test_cmp_u8( |i, j| vceq_u8(i, j), |a: u8, b: u8| -> u8 { if a == b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u8() { + fn test_vceqq_u8() { testq_cmp_u8( |i, j| vceqq_u8(i, j), |a: u8, b: u8| -> u8 { if a == b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceq_u16() { + fn test_vceq_u16() { test_cmp_u16( |i, j| vceq_u16(i, j), |a: u16, b: u16| -> u16 { if a == b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u16() { + fn test_vceqq_u16() { testq_cmp_u16( |i, j| vceqq_u16(i, j), |a: u16, b: u16| -> u16 { if a == b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceq_u32() { + fn test_vceq_u32() { test_cmp_u32( |i, j| vceq_u32(i, j), |a: u32, b: u32| -> u32 { if a == b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u32() { + fn test_vceqq_u32() { testq_cmp_u32( |i, j| vceqq_u32(i, j), |a: u32, b: u32| -> u32 { if a == b { 0xFFFFFFFF } else { 0 } }, @@ -4227,14 +4311,15 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vceq_f32() { + fn test_vceq_f32() { test_cmp_f32( |i, j| vcge_f32(i, j), |a: f32, b: f32| -> u32 { if a == b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vceqq_f32() { + fn test_vceqq_f32() { testq_cmp_f32( |i, j| vcgeq_f32(i, j), |a: f32, b: f32| -> u32 { if a == b { 0xFFFFFFFF } else { 0 } }, @@ -4242,42 +4327,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s8() { + fn test_vcgt_s8() { test_cmp_s8( |i, j| vcgt_s8(i, j), |a: i8, b: i8| -> u8 { if a > b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s8() { + fn test_vcgtq_s8() { testq_cmp_s8( |i, j| vcgtq_s8(i, j), |a: i8, b: i8| -> u8 { if a > b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s16() { + fn test_vcgt_s16() { test_cmp_s16( |i, j| vcgt_s16(i, j), |a: i16, b: i16| -> u16 { if a > b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s16() { + fn test_vcgtq_s16() { testq_cmp_s16( |i, j| vcgtq_s16(i, j), |a: i16, b: i16| -> u16 { if a > b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s32() { + fn test_vcgt_s32() { test_cmp_s32( |i, j| vcgt_s32(i, j), |a: i32, b: i32| -> u32 { if a > b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s32() { + fn test_vcgtq_s32() { testq_cmp_s32( |i, j| vcgtq_s32(i, j), |a: i32, b: i32| -> u32 { if a > b { 0xFFFFFFFF } else { 0 } }, @@ -4285,42 +4375,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u8() { + fn test_vcgt_u8() { test_cmp_u8( |i, j| vcgt_u8(i, j), |a: u8, b: u8| -> u8 { if a > b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u8() { + fn test_vcgtq_u8() { testq_cmp_u8( |i, j| vcgtq_u8(i, j), |a: u8, b: u8| -> u8 { if a > b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u16() { + fn test_vcgt_u16() { test_cmp_u16( |i, j| vcgt_u16(i, j), |a: u16, b: u16| -> u16 { if a > b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u16() { + fn test_vcgtq_u16() { testq_cmp_u16( |i, j| vcgtq_u16(i, j), |a: u16, b: u16| -> u16 { if a > b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u32() { + fn test_vcgt_u32() { test_cmp_u32( |i, j| vcgt_u32(i, j), |a: u32, b: u32| -> u32 { if a > b { 0xFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u32() { + fn test_vcgtq_u32() { testq_cmp_u32( |i, j| vcgtq_u32(i, j), |a: u32, b: u32| -> u32 { if a > b { 0xFFFFFFFF } else { 0 } }, @@ -4328,14 +4423,15 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcgt_f32() { + fn test_vcgt_f32() { test_cmp_f32( |i, j| vcgt_f32(i, j), |a: f32, b: f32| -> u32 { if a > b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_f32() { + fn test_vcgtq_f32() { testq_cmp_f32( |i, j| vcgtq_f32(i, j), |a: f32, b: f32| -> u32 { if a > b { 0xFFFFFFFF } else { 0 } }, @@ -4343,42 +4439,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vclt_s8() { + fn test_vclt_s8() { test_cmp_s8( |i, j| vclt_s8(i, j), |a: i8, b: i8| -> u8 { if a < b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s8() { + fn test_vcltq_s8() { testq_cmp_s8( |i, j| vcltq_s8(i, j), |a: i8, b: i8| -> u8 { if a < b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vclt_s16() { + fn test_vclt_s16() { test_cmp_s16( |i, j| vclt_s16(i, j), |a: i16, b: i16| -> u16 { if a < b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s16() { + fn test_vcltq_s16() { testq_cmp_s16( |i, j| vcltq_s16(i, j), |a: i16, b: i16| -> u16 { if a < b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vclt_s32() { + fn test_vclt_s32() { test_cmp_s32( |i, j| vclt_s32(i, j), |a: i32, b: i32| -> u32 { if a < b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s32() { + fn test_vcltq_s32() { testq_cmp_s32( |i, j| vcltq_s32(i, j), |a: i32, b: i32| -> u32 { if a < b { 0xFFFFFFFF } else { 0 } }, @@ -4386,42 +4487,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vclt_u8() { + fn test_vclt_u8() { test_cmp_u8( |i, j| vclt_u8(i, j), |a: u8, b: u8| -> u8 { if a < b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u8() { + fn test_vcltq_u8() { testq_cmp_u8( |i, j| vcltq_u8(i, j), |a: u8, b: u8| -> u8 { if a < b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vclt_u16() { + fn test_vclt_u16() { test_cmp_u16( |i, j| vclt_u16(i, j), |a: u16, b: u16| -> u16 { if a < b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u16() { + fn test_vcltq_u16() { testq_cmp_u16( |i, j| vcltq_u16(i, j), |a: u16, b: u16| -> u16 { if a < b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vclt_u32() { + fn test_vclt_u32() { test_cmp_u32( |i, j| vclt_u32(i, j), |a: u32, b: u32| -> u32 { if a < b { 0xFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u32() { + fn test_vcltq_u32() { testq_cmp_u32( |i, j| vcltq_u32(i, j), |a: u32, b: u32| -> u32 { if a < b { 0xFFFFFFFF } else { 0 } }, @@ -4429,14 +4535,15 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vclt_f32() { + fn test_vclt_f32() { test_cmp_f32( |i, j| vclt_f32(i, j), |a: f32, b: f32| -> u32 { if a < b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcltq_f32() { + fn test_vcltq_f32() { testq_cmp_f32( |i, j| vcltq_f32(i, j), |a: f32, b: f32| -> u32 { if a < b { 0xFFFFFFFF } else { 0 } }, @@ -4444,42 +4551,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcle_s8() { + fn test_vcle_s8() { test_cmp_s8( |i, j| vcle_s8(i, j), |a: i8, b: i8| -> u8 { if a <= b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s8() { + fn test_vcleq_s8() { testq_cmp_s8( |i, j| vcleq_s8(i, j), |a: i8, b: i8| -> u8 { if a <= b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcle_s16() { + fn test_vcle_s16() { test_cmp_s16( |i, j| vcle_s16(i, j), |a: i16, b: i16| -> u16 { if a <= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s16() { + fn test_vcleq_s16() { testq_cmp_s16( |i, j| vcleq_s16(i, j), |a: i16, b: i16| -> u16 { if a <= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcle_s32() { + fn test_vcle_s32() { test_cmp_s32( |i, j| vcle_s32(i, j), |a: i32, b: i32| -> u32 { if a <= b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s32() { + fn test_vcleq_s32() { testq_cmp_s32( |i, j| vcleq_s32(i, j), |a: i32, b: i32| -> u32 { if a <= b { 0xFFFFFFFF } else { 0 } }, @@ -4487,42 +4599,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcle_u8() { + fn test_vcle_u8() { test_cmp_u8( |i, j| vcle_u8(i, j), |a: u8, b: u8| -> u8 { if a <= b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u8() { + fn test_vcleq_u8() { testq_cmp_u8( |i, j| vcleq_u8(i, j), |a: u8, b: u8| -> u8 { if a <= b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcle_u16() { + fn test_vcle_u16() { test_cmp_u16( |i, j| vcle_u16(i, j), |a: u16, b: u16| -> u16 { if a <= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u16() { + fn test_vcleq_u16() { testq_cmp_u16( |i, j| vcleq_u16(i, j), |a: u16, b: u16| -> u16 { if a <= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcle_u32() { + fn test_vcle_u32() { test_cmp_u32( |i, j| vcle_u32(i, j), |a: u32, b: u32| -> u32 { if a <= b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u32() { + fn test_vcleq_u32() { testq_cmp_u32( |i, j| vcleq_u32(i, j), |a: u32, b: u32| -> u32 { if a <= b { 0xFFFFFFFF } else { 0 } }, @@ -4530,14 +4647,15 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcle_f32() { + fn test_vcle_f32() { test_cmp_f32( |i, j| vcle_f32(i, j), |a: f32, b: f32| -> u32 { if a <= b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcleq_f32() { + fn test_vcleq_f32() { testq_cmp_f32( |i, j| vcleq_f32(i, j), |a: f32, b: f32| -> u32 { if a <= b { 0xFFFFFFFF } else { 0 } }, @@ -4545,42 +4663,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcge_s8() { + fn test_vcge_s8() { test_cmp_s8( |i, j| vcge_s8(i, j), |a: i8, b: i8| -> u8 { if a >= b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s8() { + fn test_vcgeq_s8() { testq_cmp_s8( |i, j| vcgeq_s8(i, j), |a: i8, b: i8| -> u8 { if a >= b { 0xFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcge_s16() { + fn test_vcge_s16() { test_cmp_s16( |i, j| vcge_s16(i, j), |a: i16, b: i16| -> u16 { if a >= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s16() { + fn test_vcgeq_s16() { testq_cmp_s16( |i, j| vcgeq_s16(i, j), |a: i16, b: i16| -> u16 { if a >= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcge_s32() { + fn test_vcge_s32() { test_cmp_s32( |i, j| vcge_s32(i, j), |a: i32, b: i32| -> u32 { if a >= b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s32() { + fn test_vcgeq_s32() { testq_cmp_s32( |i, j| vcgeq_s32(i, j), |a: i32, b: i32| -> u32 { if a >= b { 0xFFFFFFFF } else { 0 } }, @@ -4588,42 +4711,45 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcge_u8() { + fn test_vcge_u8() { test_cmp_u8( |i, j| vcge_u8(i, j), |a: u8, b: u8| -> u8 { if a >= b { 0xFF } else { 0 } }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u8() { + fn test_vcgeq_u8() { testq_cmp_u8( |i, j| vcgeq_u8(i, j), |a: u8, b: u8| -> u8 { if a >= b { 0xFF } else { 0 } }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vcge_u16() { + fn test_vcge_u16() { test_cmp_u16( |i, j| vcge_u16(i, j), |a: u16, b: u16| -> u16 { if a >= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u16() { + fn test_vcgeq_u16() { testq_cmp_u16( |i, j| vcgeq_u16(i, j), |a: u16, b: u16| -> u16 { if a >= b { 0xFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcge_u32() { + fn test_vcge_u32() { test_cmp_u32( |i, j| vcge_u32(i, j), |a: u32, b: u32| -> u32 { if a >= b { 0xFFFFFFFF } else { 0 } }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u32() { + fn test_vcgeq_u32() { testq_cmp_u32( |i, j| vcgeq_u32(i, j), |a: u32, b: u32| -> u32 { if a >= b { 0xFFFFFFFF } else { 0 } }, @@ -4631,14 +4757,14 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vcge_f32() { + fn test_vcge_f32() { test_cmp_f32( |i, j| vcge_f32(i, j), |a: f32, b: f32| -> u32 { if a >= b { 0xFFFFFFFF } else { 0 } }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_f32() { + fn test_vcgeq_f32() { testq_cmp_f32( |i, j| vcgeq_f32(i, j), |a: f32, b: f32| -> u32 { if a >= b { 0xFFFFFFFF } else { 0 } }, @@ -4646,42 +4772,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vqsub_s8() { + fn test_vqsub_s8() { test_ari_s8( |i, j| vqsub_s8(i, j), |a: i8, b: i8| -> i8 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_s8() { + fn test_vqsubq_s8() { testq_ari_s8( |i, j| vqsubq_s8(i, j), |a: i8, b: i8| -> i8 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsub_s16() { + fn test_vqsub_s16() { test_ari_s16( |i, j| vqsub_s16(i, j), |a: i16, b: i16| -> i16 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_s16() { + fn test_vqsubq_s16() { testq_ari_s16( |i, j| vqsubq_s16(i, j), |a: i16, b: i16| -> i16 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsub_s32() { + fn test_vqsub_s32() { test_ari_s32( |i, j| vqsub_s32(i, j), |a: i32, b: i32| -> i32 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_s32() { + fn test_vqsubq_s32() { testq_ari_s32( |i, j| vqsubq_s32(i, j), |a: i32, b: i32| -> i32 { a.saturating_sub(b) }, @@ -4689,42 +4820,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vqsub_u8() { + fn test_vqsub_u8() { test_ari_u8( |i, j| vqsub_u8(i, j), |a: u8, b: u8| -> u8 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_u8() { + fn test_vqsubq_u8() { testq_ari_u8( |i, j| vqsubq_u8(i, j), |a: u8, b: u8| -> u8 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsub_u16() { + fn test_vqsub_u16() { test_ari_u16( |i, j| vqsub_u16(i, j), |a: u16, b: u16| -> u16 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_u16() { + fn test_vqsubq_u16() { testq_ari_u16( |i, j| vqsubq_u16(i, j), |a: u16, b: u16| -> u16 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsub_u32() { + fn test_vqsub_u32() { test_ari_u32( |i, j| vqsub_u32(i, j), |a: u32, b: u32| -> u32 { a.saturating_sub(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_u32() { + fn test_vqsubq_u32() { testq_ari_u32( |i, j| vqsubq_u32(i, j), |a: u32, b: u32| -> u32 { a.saturating_sub(b) }, @@ -4732,142 +4868,166 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vhadd_s8() { + fn test_vhadd_s8() { test_ari_s8(|i, j| vhadd_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_s8() { + fn test_vhaddq_s8() { testq_ari_s8(|i, j| vhaddq_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhadd_s16() { + fn test_vhadd_s16() { test_ari_s16(|i, j| vhadd_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_s16() { + fn test_vhaddq_s16() { testq_ari_s16(|i, j| vhaddq_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhadd_s32() { + fn test_vhadd_s32() { test_ari_s32(|i, j| vhadd_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_s32() { + fn test_vhaddq_s32() { testq_ari_s32(|i, j| vhaddq_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); } #[simd_test(enable = "neon")] - unsafe fn test_vhadd_u8() { + fn test_vhadd_u8() { test_ari_u8(|i, j| vhadd_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_u8() { + fn test_vhaddq_u8() { testq_ari_u8(|i, j| vhaddq_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhadd_u16() { + fn test_vhadd_u16() { test_ari_u16(|i, j| vhadd_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_u16() { + fn test_vhaddq_u16() { testq_ari_u16(|i, j| vhaddq_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhadd_u32() { + fn test_vhadd_u32() { test_ari_u32(|i, j| vhadd_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_u32() { + fn test_vhaddq_u32() { testq_ari_u32(|i, j| vhaddq_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); } #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_s8() { + fn test_vrhadd_s8() { test_ari_s8(|i, j| vrhadd_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_s8() { + fn test_vrhaddq_s8() { testq_ari_s8(|i, j| vrhaddq_s8(i, j), |a: i8, b: i8| -> i8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_s16() { + fn test_vrhadd_s16() { test_ari_s16(|i, j| vrhadd_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_s16() { + fn test_vrhaddq_s16() { testq_ari_s16(|i, j| vrhaddq_s16(i, j), |a: i16, b: i16| -> i16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_s32() { + fn test_vrhadd_s32() { test_ari_s32(|i, j| vrhadd_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_s32() { + fn test_vrhaddq_s32() { testq_ari_s32(|i, j| vrhaddq_s32(i, j), |a: i32, b: i32| -> i32 { a & b }); } #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_u8() { + fn test_vrhadd_u8() { test_ari_u8(|i, j| vrhadd_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_u8() { + fn test_vrhaddq_u8() { testq_ari_u8(|i, j| vrhaddq_u8(i, j), |a: u8, b: u8| -> u8 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_u16() { + fn test_vrhadd_u16() { test_ari_u16(|i, j| vrhadd_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_u16() { + fn test_vrhaddq_u16() { testq_ari_u16(|i, j| vrhaddq_u16(i, j), |a: u16, b: u16| -> u16 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_u32() { + fn test_vrhadd_u32() { test_ari_u32(|i, j| vrhadd_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_u32() { + fn test_vrhaddq_u32() { testq_ari_u32(|i, j| vrhaddq_u32(i, j), |a: u32, b: u32| -> u32 { a & b }); } #[simd_test(enable = "neon")] - unsafe fn test_vqadd_s8() { + fn test_vqadd_s8() { test_ari_s8( |i, j| vqadd_s8(i, j), |a: i8, b: i8| -> i8 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_s8() { + fn test_vqaddq_s8() { testq_ari_s8( |i, j| vqaddq_s8(i, j), |a: i8, b: i8| -> i8 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqadd_s16() { + fn test_vqadd_s16() { test_ari_s16( |i, j| vqadd_s16(i, j), |a: i16, b: i16| -> i16 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_s16() { + fn test_vqaddq_s16() { testq_ari_s16( |i, j| vqaddq_s16(i, j), |a: i16, b: i16| -> i16 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqadd_s32() { + fn test_vqadd_s32() { test_ari_s32( |i, j| vqadd_s32(i, j), |a: i32, b: i32| -> i32 { a.saturating_add(b) }, ); } #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_s32() { + fn test_vqaddq_s32() { testq_ari_s32( |i, j| vqaddq_s32(i, j), |a: i32, b: i32| -> i32 { a.saturating_add(b) }, @@ -4875,42 +5035,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vqadd_u8() { + fn test_vqadd_u8() { test_ari_u8( |i, j| vqadd_u8(i, j), |a: u8, b: u8| -> u8 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_u8() { + fn test_vqaddq_u8() { testq_ari_u8( |i, j| vqaddq_u8(i, j), |a: u8, b: u8| -> u8 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqadd_u16() { + fn test_vqadd_u16() { test_ari_u16( |i, j| vqadd_u16(i, j), |a: u16, b: u16| -> u16 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_u16() { + fn test_vqaddq_u16() { testq_ari_u16( |i, j| vqaddq_u16(i, j), |a: u16, b: u16| -> u16 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqadd_u32() { + fn test_vqadd_u32() { test_ari_u32( |i, j| vqadd_u32(i, j), |a: u32, b: u32| -> u32 { a.saturating_add(b) }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_u32() { + fn test_vqaddq_u32() { testq_ari_u32( |i, j| vqaddq_u32(i, j), |a: u32, b: u32| -> u32 { a.saturating_add(b) }, @@ -4918,42 +5083,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vmul_s8() { + fn test_vmul_s8() { test_ari_s8( |i, j| vmul_s8(i, j), |a: i8, b: i8| -> i8 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_s8() { + fn test_vmulq_s8() { testq_ari_s8( |i, j| vmulq_s8(i, j), |a: i8, b: i8| -> i8 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmul_s16() { + fn test_vmul_s16() { test_ari_s16( |i, j| vmul_s16(i, j), |a: i16, b: i16| -> i16 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_s16() { + fn test_vmulq_s16() { testq_ari_s16( |i, j| vmulq_s16(i, j), |a: i16, b: i16| -> i16 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmul_s32() { + fn test_vmul_s32() { test_ari_s32( |i, j| vmul_s32(i, j), |a: i32, b: i32| -> i32 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_s32() { + fn test_vmulq_s32() { testq_ari_s32( |i, j| vmulq_s32(i, j), |a: i32, b: i32| -> i32 { a.overflowing_mul(b).0 }, @@ -4961,42 +5131,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vmul_u8() { + fn test_vmul_u8() { test_ari_u8( |i, j| vmul_u8(i, j), |a: u8, b: u8| -> u8 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_u8() { + fn test_vmulq_u8() { testq_ari_u8( |i, j| vmulq_u8(i, j), |a: u8, b: u8| -> u8 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmul_u16() { + fn test_vmul_u16() { test_ari_u16( |i, j| vmul_u16(i, j), |a: u16, b: u16| -> u16 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_u16() { + fn test_vmulq_u16() { testq_ari_u16( |i, j| vmulq_u16(i, j), |a: u16, b: u16| -> u16 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmul_u32() { + fn test_vmul_u32() { test_ari_u32( |i, j| vmul_u32(i, j), |a: u32, b: u32| -> u32 { a.overflowing_mul(b).0 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_u32() { + fn test_vmulq_u32() { testq_ari_u32( |i, j| vmulq_u32(i, j), |a: u32, b: u32| -> u32 { a.overflowing_mul(b).0 }, @@ -5004,110 +5179,127 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vmul_f32() { + fn test_vmul_f32() { test_ari_f32(|i, j| vmul_f32(i, j), |a: f32, b: f32| -> f32 { a * b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vmulq_f32() { + fn test_vmulq_f32() { testq_ari_f32(|i, j| vmulq_f32(i, j), |a: f32, b: f32| -> f32 { a * b }); } #[simd_test(enable = "neon")] - unsafe fn test_vsub_s8() { + fn test_vsub_s8() { test_ari_s8(|i, j| vsub_s8(i, j), |a: i8, b: i8| -> i8 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s8() { + fn test_vsubq_s8() { testq_ari_s8(|i, j| vsubq_s8(i, j), |a: i8, b: i8| -> i8 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsub_s16() { + fn test_vsub_s16() { test_ari_s16(|i, j| vsub_s16(i, j), |a: i16, b: i16| -> i16 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s16() { + fn test_vsubq_s16() { testq_ari_s16(|i, j| vsubq_s16(i, j), |a: i16, b: i16| -> i16 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsub_s32() { + fn test_vsub_s32() { test_ari_s32(|i, j| vsub_s32(i, j), |a: i32, b: i32| -> i32 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s32() { + fn test_vsubq_s32() { testq_ari_s32(|i, j| vsubq_s32(i, j), |a: i32, b: i32| -> i32 { a - b }); } #[simd_test(enable = "neon")] - unsafe fn test_vsub_u8() { + fn test_vsub_u8() { test_ari_u8(|i, j| vsub_u8(i, j), |a: u8, b: u8| -> u8 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u8() { + fn test_vsubq_u8() { testq_ari_u8(|i, j| vsubq_u8(i, j), |a: u8, b: u8| -> u8 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsub_u16() { + fn test_vsub_u16() { test_ari_u16(|i, j| vsub_u16(i, j), |a: u16, b: u16| -> u16 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u16() { + fn test_vsubq_u16() { testq_ari_u16(|i, j| vsubq_u16(i, j), |a: u16, b: u16| -> u16 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsub_u32() { + fn test_vsub_u32() { test_ari_u32(|i, j| vsub_u32(i, j), |a: u32, b: u32| -> u32 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u32() { + fn test_vsubq_u32() { testq_ari_u32(|i, j| vsubq_u32(i, j), |a: u32, b: u32| -> u32 { a - b }); } #[simd_test(enable = "neon")] - unsafe fn test_vsub_f32() { + fn test_vsub_f32() { test_ari_f32(|i, j| vsub_f32(i, j), |a: f32, b: f32| -> f32 { a - b }); } + #[simd_test(enable = "neon")] - unsafe fn test_vsubq_f32() { + fn test_vsubq_f32() { testq_ari_f32(|i, j| vsubq_f32(i, j), |a: f32, b: f32| -> f32 { a - b }); } #[simd_test(enable = "neon")] - unsafe fn test_vhsub_s8() { + fn test_vhsub_s8() { test_ari_s8( |i, j| vhsub_s8(i, j), |a: i8, b: i8| -> i8 { (((a as i16) - (b as i16)) / 2) as i8 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_s8() { + fn test_vhsubq_s8() { testq_ari_s8( |i, j| vhsubq_s8(i, j), |a: i8, b: i8| -> i8 { (((a as i16) - (b as i16)) / 2) as i8 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsub_s16() { + fn test_vhsub_s16() { test_ari_s16( |i, j| vhsub_s16(i, j), |a: i16, b: i16| -> i16 { (((a as i32) - (b as i32)) / 2) as i16 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_s16() { + fn test_vhsubq_s16() { testq_ari_s16( |i, j| vhsubq_s16(i, j), |a: i16, b: i16| -> i16 { (((a as i32) - (b as i32)) / 2) as i16 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsub_s32() { + fn test_vhsub_s32() { test_ari_s32( |i, j| vhsub_s32(i, j), |a: i32, b: i32| -> i32 { (((a as i64) - (b as i64)) / 2) as i32 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_s32() { + fn test_vhsubq_s32() { testq_ari_s32( |i, j| vhsubq_s32(i, j), |a: i32, b: i32| -> i32 { (((a as i64) - (b as i64)) / 2) as i32 }, @@ -5115,42 +5307,47 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vhsub_u8() { + fn test_vhsub_u8() { test_ari_u8( |i, j| vhsub_u8(i, j), |a: u8, b: u8| -> u8 { (((a as u16) - (b as u16)) / 2) as u8 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_u8() { + fn test_vhsubq_u8() { testq_ari_u8( |i, j| vhsubq_u8(i, j), |a: u8, b: u8| -> u8 { (((a as u16) - (b as u16)) / 2) as u8 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsub_u16() { + fn test_vhsub_u16() { test_ari_u16( |i, j| vhsub_u16(i, j), |a: u16, b: u16| -> u16 { (((a as u16) - (b as u16)) / 2) as u16 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_u16() { + fn test_vhsubq_u16() { testq_ari_u16( |i, j| vhsubq_u16(i, j), |a: u16, b: u16| -> u16 { (((a as u16) - (b as u16)) / 2) as u16 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsub_u32() { + fn test_vhsub_u32() { test_ari_u32( |i, j| vhsub_u32(i, j), |a: u32, b: u32| -> u32 { (((a as u64) - (b as u64)) / 2) as u32 }, ); } + #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_u32() { + fn test_vhsubq_u32() { testq_ari_u32( |i, j| vhsubq_u32(i, j), |a: u32, b: u32| -> u32 { (((a as u64) - (b as u64)) / 2) as u32 }, @@ -5158,368 +5355,414 @@ mod tests { } #[simd_test(enable = "neon")] - unsafe fn test_vaba_s8() { + fn test_vaba_s8() { let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); let b = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); let c = i8x8::new(10, 9, 8, 7, 6, 5, 4, 3); - let r: i8x8 = transmute(vaba_s8(transmute(a), transmute(b), transmute(c))); + let r = i8x8::from(vaba_s8(a.into(), b.into(), c.into())); let e = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vaba_s16() { + fn test_vaba_s16() { let a = i16x4::new(1, 2, 3, 4); let b = i16x4::new(1, 1, 1, 1); let c = i16x4::new(10, 9, 8, 7); - let r: i16x4 = transmute(vaba_s16(transmute(a), transmute(b), transmute(c))); + let r = i16x4::from(vaba_s16(a.into(), b.into(), c.into())); let e = i16x4::new(10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vaba_s32() { + fn test_vaba_s32() { let a = i32x2::new(1, 2); let b = i32x2::new(1, 1); let c = i32x2::new(10, 9); - let r: i32x2 = transmute(vaba_s32(transmute(a), transmute(b), transmute(c))); + let r = i32x2::from(vaba_s32(a.into(), b.into(), c.into())); let e = i32x2::new(10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vaba_u8() { + fn test_vaba_u8() { let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); let b = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); let c = u8x8::new(10, 9, 8, 7, 6, 5, 4, 3); - let r: u8x8 = transmute(vaba_u8(transmute(a), transmute(b), transmute(c))); + let r = u8x8::from(vaba_u8(a.into(), b.into(), c.into())); let e = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vaba_u16() { + fn test_vaba_u16() { let a = u16x4::new(1, 2, 3, 4); let b = u16x4::new(1, 1, 1, 1); let c = u16x4::new(10, 9, 8, 7); - let r: u16x4 = transmute(vaba_u16(transmute(a), transmute(b), transmute(c))); + let r = u16x4::from(vaba_u16(a.into(), b.into(), c.into())); let e = u16x4::new(10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vaba_u32() { + fn test_vaba_u32() { let a = u32x2::new(1, 2); let b = u32x2::new(1, 1); let c = u32x2::new(10, 9); - let r: u32x2 = transmute(vaba_u32(transmute(a), transmute(b), transmute(c))); + let r = u32x2::from(vaba_u32(a.into(), b.into(), c.into())); let e = u32x2::new(10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vabaq_s8() { + fn test_vabaq_s8() { let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2); let b = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let c = i8x16::new(10, 9, 8, 7, 6, 5, 4, 3, 12, 13, 14, 15, 16, 17, 18, 19); - let r: i8x16 = transmute(vabaq_s8(transmute(a), transmute(b), transmute(c))); + let r = i8x16::from(vabaq_s8(a.into(), b.into(), c.into())); let e = i8x16::new( 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, ); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vabaq_s16() { + fn test_vabaq_s16() { let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let b = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); let c = i16x8::new(10, 9, 8, 7, 6, 5, 4, 3); - let r: i16x8 = transmute(vabaq_s16(transmute(a), transmute(b), transmute(c))); + let r = i16x8::from(vabaq_s16(a.into(), b.into(), c.into())); let e = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vabaq_s32() { + fn test_vabaq_s32() { let a = i32x4::new(1, 2, 3, 4); let b = i32x4::new(1, 1, 1, 1); let c = i32x4::new(10, 9, 8, 7); - let r: i32x4 = transmute(vabaq_s32(transmute(a), transmute(b), transmute(c))); + let r = i32x4::from(vabaq_s32(a.into(), b.into(), c.into())); let e = i32x4::new(10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vabaq_u8() { + fn test_vabaq_u8() { let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2); let b = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let c = u8x16::new(10, 9, 8, 7, 6, 5, 4, 3, 12, 13, 14, 15, 16, 17, 18, 19); - let r: u8x16 = transmute(vabaq_u8(transmute(a), transmute(b), transmute(c))); + let r = u8x16::from(vabaq_u8(a.into(), b.into(), c.into())); let e = u8x16::new( 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, ); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vabaq_u16() { + fn test_vabaq_u16() { let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); let b = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); let c = u16x8::new(10, 9, 8, 7, 6, 5, 4, 3); - let r: u16x8 = transmute(vabaq_u16(transmute(a), transmute(b), transmute(c))); + let r = u16x8::from(vabaq_u16(a.into(), b.into(), c.into())); let e = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vabaq_u32() { + fn test_vabaq_u32() { let a = u32x4::new(1, 2, 3, 4); let b = u32x4::new(1, 1, 1, 1); let c = u32x4::new(10, 9, 8, 7); - let r: u32x4 = transmute(vabaq_u32(transmute(a), transmute(b), transmute(c))); + let r = u32x4::from(vabaq_u32(a.into(), b.into(), c.into())); let e = u32x4::new(10, 10, 10, 10); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vrev16_s8() { + fn test_vrev16_s8() { let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = i8x8::new(1, 0, 3, 2, 5, 4, 7, 6); - let e: i8x8 = transmute(vrev16_s8(transmute(a))); + let e = i8x8::from(vrev16_s8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev16q_s8() { + fn test_vrev16q_s8() { let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = i8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - let e: i8x16 = transmute(vrev16q_s8(transmute(a))); + let e = i8x16::from(vrev16q_s8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev16_u8() { + fn test_vrev16_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u8x8::new(1, 0, 3, 2, 5, 4, 7, 6); - let e: u8x8 = transmute(vrev16_u8(transmute(a))); + let e = u8x8::from(vrev16_u8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev16q_u8() { + fn test_vrev16q_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = u8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - let e: u8x16 = transmute(vrev16q_u8(transmute(a))); + let e = u8x16::from(vrev16q_u8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev16_p8() { - let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let r = i8x8::new(1, 0, 3, 2, 5, 4, 7, 6); - let e: i8x8 = transmute(vrev16_p8(transmute(a))); + fn test_vrev16_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u8x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e = u8x8::from(vrev16_p8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev16q_p8() { + fn test_vrev16q_p8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = u8x16::new(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - let e: u8x16 = transmute(vrev16q_p8(transmute(a))); + let e = u8x16::from(vrev16q_p8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32_s8() { + fn test_vrev32_s8() { let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = i8x8::new(3, 2, 1, 0, 7, 6, 5, 4); - let e: i8x8 = transmute(vrev32_s8(transmute(a))); + let e = i8x8::from(vrev32_s8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32q_s8() { + fn test_vrev32q_s8() { let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = i8x16::new(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); - let e: i8x16 = transmute(vrev32q_s8(transmute(a))); + let e = i8x16::from(vrev32q_s8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32_u8() { + fn test_vrev32_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u8x8::new(3, 2, 1, 0, 7, 6, 5, 4); - let e: u8x8 = transmute(vrev32_u8(transmute(a))); + let e = u8x8::from(vrev32_u8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32q_u8() { + fn test_vrev32q_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = u8x16::new(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); - let e: u8x16 = transmute(vrev32q_u8(transmute(a))); + let e = u8x16::from(vrev32q_u8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32_s16() { + fn test_vrev32_s16() { let a = i16x4::new(0, 1, 2, 3); let r = i16x4::new(1, 0, 3, 2); - let e: i16x4 = transmute(vrev32_s16(transmute(a))); + let e = i16x4::from(vrev32_s16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32q_s16() { + fn test_vrev32q_s16() { let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); - let e: i16x8 = transmute(vrev32q_s16(transmute(a))); + let e = i16x8::from(vrev32q_s16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32_p16() { - let a = i16x4::new(0, 1, 2, 3); - let r = i16x4::new(1, 0, 3, 2); - let e: i16x4 = transmute(vrev32_p16(transmute(a))); - assert_eq!(r, e); - } - #[simd_test(enable = "neon")] - unsafe fn test_vrev32q_p16() { - let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let r = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); - let e: i16x8 = transmute(vrev32q_p16(transmute(a))); - assert_eq!(r, e); - } - #[simd_test(enable = "neon")] - unsafe fn test_vrev32_u16() { + fn test_vrev32_p16() { let a = u16x4::new(0, 1, 2, 3); let r = u16x4::new(1, 0, 3, 2); - let e: u16x4 = transmute(vrev32_u16(transmute(a))); + let e = u16x4::from(vrev32_p16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32q_u16() { + fn test_vrev32q_p16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u16x8::new(1, 0, 3, 2, 5, 4, 7, 6); - let e: u16x8 = transmute(vrev32q_u16(transmute(a))); + let e = u16x8::from(vrev32q_p16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32_p8() { + fn test_vrev32_u16() { + let a = u16x4::new(0, 1, 2, 3); + let r = u16x4::new(1, 0, 3, 2); + let e = u16x4::from(vrev32_u16(a.into())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + fn test_vrev32q_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = u16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e = u16x8::from(vrev32q_u16(a.into())); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + fn test_vrev32_p8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u8x8::new(3, 2, 1, 0, 7, 6, 5, 4); - let e: u8x8 = transmute(vrev32_p8(transmute(a))); + let e = u8x8::from(vrev32_p8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev32q_p8() { + fn test_vrev32q_p8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = u8x16::new(3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); - let e: u8x16 = transmute(vrev32q_p8(transmute(a))); + let e = u8x16::from(vrev32q_p8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_s8() { + fn test_vrev64_s8() { let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = i8x8::new(7, 6, 5, 4, 3, 2, 1, 0); - let e: i8x8 = transmute(vrev64_s8(transmute(a))); + let e = i8x8::from(vrev64_s8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_s8() { + fn test_vrev64q_s8() { let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = i8x16::new(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); - let e: i8x16 = transmute(vrev64q_s8(transmute(a))); + let e = i8x16::from(vrev64q_s8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_s16() { + fn test_vrev64_s16() { let a = i16x4::new(0, 1, 2, 3); let r = i16x4::new(3, 2, 1, 0); - let e: i16x4 = transmute(vrev64_s16(transmute(a))); + let e = i16x4::from(vrev64_s16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_s16() { + fn test_vrev64q_s16() { let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = i16x8::new(3, 2, 1, 0, 7, 6, 5, 4); - let e: i16x8 = transmute(vrev64q_s16(transmute(a))); + let e = i16x8::from(vrev64q_s16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_s32() { + fn test_vrev64_s32() { let a = i32x2::new(0, 1); let r = i32x2::new(1, 0); - let e: i32x2 = transmute(vrev64_s32(transmute(a))); + let e = i32x2::from(vrev64_s32(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_s32() { + fn test_vrev64q_s32() { let a = i32x4::new(0, 1, 2, 3); let r = i32x4::new(1, 0, 3, 2); - let e: i32x4 = transmute(vrev64q_s32(transmute(a))); + let e = i32x4::from(vrev64q_s32(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_u8() { + fn test_vrev64_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u8x8::new(7, 6, 5, 4, 3, 2, 1, 0); - let e: u8x8 = transmute(vrev64_u8(transmute(a))); + let e = u8x8::from(vrev64_u8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_u8() { + fn test_vrev64q_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = u8x16::new(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); - let e: u8x16 = transmute(vrev64q_u8(transmute(a))); + let e = u8x16::from(vrev64q_u8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_u16() { + fn test_vrev64_u16() { let a = u16x4::new(0, 1, 2, 3); let r = u16x4::new(3, 2, 1, 0); - let e: u16x4 = transmute(vrev64_u16(transmute(a))); + let e = u16x4::from(vrev64_u16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_u16() { + fn test_vrev64q_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u16x8::new(3, 2, 1, 0, 7, 6, 5, 4); - let e: u16x8 = transmute(vrev64q_u16(transmute(a))); + let e = u16x8::from(vrev64q_u16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_u32() { + fn test_vrev64_u32() { let a = u32x2::new(0, 1); let r = u32x2::new(1, 0); - let e: u32x2 = transmute(vrev64_u32(transmute(a))); + let e = u32x2::from(vrev64_u32(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_u32() { + fn test_vrev64q_u32() { let a = u32x4::new(0, 1, 2, 3); let r = u32x4::new(1, 0, 3, 2); - let e: u32x4 = transmute(vrev64q_u32(transmute(a))); + let e = u32x4::from(vrev64q_u32(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_f32() { + fn test_vrev64_f32() { let a = f32x2::new(1.0, 2.0); let r = f32x2::new(2.0, 1.0); - let e: f32x2 = transmute(vrev64_f32(transmute(a))); + let e = f32x2::from(vrev64_f32(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_f32() { + fn test_vrev64q_f32() { let a = f32x4::new(1.0, 2.0, -2.0, -1.0); let r = f32x4::new(2.0, 1.0, -1.0, -2.0); - let e: f32x4 = transmute(vrev64q_f32(transmute(a))); + let e = f32x4::from(vrev64q_f32(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_p8() { + fn test_vrev64_p8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u8x8::new(7, 6, 5, 4, 3, 2, 1, 0); - let e: u8x8 = transmute(vrev64_p8(transmute(a))); + let e = u8x8::from(vrev64_p8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_p8() { + fn test_vrev64q_p8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = u8x16::new(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); - let e: u8x16 = transmute(vrev64q_p8(transmute(a))); + let e = u8x16::from(vrev64q_p8(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64_p16() { + fn test_vrev64_p16() { let a = u16x4::new(0, 1, 2, 3); let r = u16x4::new(3, 2, 1, 0); - let e: u16x4 = transmute(vrev64_p16(transmute(a))); + let e = u16x4::from(vrev64_p16(a.into())); assert_eq!(r, e); } + #[simd_test(enable = "neon")] - unsafe fn test_vrev64q_p16() { + fn test_vrev64q_p16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = u16x8::new(3, 2, 1, 0, 7, 6, 5, 4); - let e: u16x8 = transmute(vrev64q_p16(transmute(a))); + let e = u16x8::from(vrev64q_p16(a.into())); assert_eq!(r, e); } @@ -5527,13 +5770,13 @@ mod tests { ($test_id:ident => $fn_id:ident ([$($a:expr),*], [$($b:expr),*])) => { #[allow(unused_assignments)] #[simd_test(enable = "neon")] - unsafe fn $test_id() { - let a = [$($a),*]; - let b = [$($b),*]; - let e = [$($a),* $(, $b)*]; - let c = $fn_id(transmute(a), transmute(b)); + fn $test_id() { + let a = Simd::from_array([$($a),*]); + let b = Simd::from_array([$($b),*]); + let e = Simd::from_array([$($a),* $(, $b)*]); + let c = $fn_id(a.into(), b.into()); let mut d = e; - d = transmute(c); + d = c.into(); assert_eq!(d, e); } } @@ -5546,11 +5789,19 @@ mod tests { test_vcombine!(test_vcombine_s16 => vcombine_s16([3_i16, -4, 5, -6], [13_i16, -14, 15, -16])); test_vcombine!(test_vcombine_u16 => vcombine_u16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16])); test_vcombine!(test_vcombine_p16 => vcombine_p16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16])); + #[cfg(not(target_arch = "arm64ec"))] mod fp16 { use super::*; - test_vcombine!(test_vcombine_f16 => vcombine_f16([3_f16, 4., 5., 6.], - [13_f16, 14., 15., 16.])); + #[cfg_attr(target_arch = "arm", simd_test(enable = "neon,fp16"))] + #[cfg_attr(not(target_arch = "arm"), simd_test(enable = "neon"))] + fn test_vcombine_f16() { + let a = f16x4::from_array([3_f16, 4., 5., 6.]); + let b = f16x4::from_array([13_f16, 14., 15., 16.]); + let e = f16x8::from_array([3_f16, 4., 5., 6., 13_f16, 14., 15., 16.]); + let c = f16x8::from(vcombine_f16(a.into(), b.into())); + assert_eq!(c, e); + } } test_vcombine!(test_vcombine_s32 => vcombine_s32([3_i32, -4], [13_i32, -14])); diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs index 6b5d4a19ad57..2b10b38f2dd8 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs @@ -14,11 +14,13 @@ use crate::core_arch::simd::*; use stdarch_test::simd_test; #[simd_test(enable = "neon")] -unsafe fn test_vst1_s8() { +fn test_vst1_s8() { let mut vals = [0_i8; 9]; let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - vst1_s8(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_s8(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -32,11 +34,13 @@ unsafe fn test_vst1_s8() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_s8() { +fn test_vst1q_s8() { let mut vals = [0_i8; 17]; let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - vst1q_s8(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_s8(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -58,11 +62,13 @@ unsafe fn test_vst1q_s8() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_s16() { +fn test_vst1_s16() { let mut vals = [0_i16; 5]; let a = i16x4::new(1, 2, 3, 4); - vst1_s16(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_s16(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -72,11 +78,13 @@ unsafe fn test_vst1_s16() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_s16() { +fn test_vst1q_s16() { let mut vals = [0_i16; 9]; let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - vst1q_s16(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_s16(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -90,11 +98,13 @@ unsafe fn test_vst1q_s16() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_s32() { +fn test_vst1_s32() { let mut vals = [0_i32; 3]; let a = i32x2::new(1, 2); - vst1_s32(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_s32(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -102,11 +112,13 @@ unsafe fn test_vst1_s32() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_s32() { +fn test_vst1q_s32() { let mut vals = [0_i32; 5]; let a = i32x4::new(1, 2, 3, 4); - vst1q_s32(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_s32(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -116,22 +128,26 @@ unsafe fn test_vst1q_s32() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_s64() { +fn test_vst1_s64() { let mut vals = [0_i64; 2]; let a = i64x1::new(1); - vst1_s64(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_s64(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_s64() { +fn test_vst1q_s64() { let mut vals = [0_i64; 3]; let a = i64x2::new(1, 2); - vst1q_s64(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_s64(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -139,11 +155,13 @@ unsafe fn test_vst1q_s64() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_u8() { +fn test_vst1_u8() { let mut vals = [0_u8; 9]; let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - vst1_u8(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_u8(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -157,11 +175,13 @@ unsafe fn test_vst1_u8() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_u8() { +fn test_vst1q_u8() { let mut vals = [0_u8; 17]; let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - vst1q_u8(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_u8(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -183,11 +203,13 @@ unsafe fn test_vst1q_u8() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_u16() { +fn test_vst1_u16() { let mut vals = [0_u16; 5]; let a = u16x4::new(1, 2, 3, 4); - vst1_u16(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_u16(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -197,11 +219,13 @@ unsafe fn test_vst1_u16() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_u16() { +fn test_vst1q_u16() { let mut vals = [0_u16; 9]; let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - vst1q_u16(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_u16(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -215,11 +239,13 @@ unsafe fn test_vst1q_u16() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_u32() { +fn test_vst1_u32() { let mut vals = [0_u32; 3]; let a = u32x2::new(1, 2); - vst1_u32(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_u32(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -227,11 +253,13 @@ unsafe fn test_vst1_u32() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_u32() { +fn test_vst1q_u32() { let mut vals = [0_u32; 5]; let a = u32x4::new(1, 2, 3, 4); - vst1q_u32(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_u32(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -241,22 +269,26 @@ unsafe fn test_vst1q_u32() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_u64() { +fn test_vst1_u64() { let mut vals = [0_u64; 2]; let a = u64x1::new(1); - vst1_u64(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_u64(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_u64() { +fn test_vst1q_u64() { let mut vals = [0_u64; 3]; let a = u64x2::new(1, 2); - vst1q_u64(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_u64(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -264,11 +296,13 @@ unsafe fn test_vst1q_u64() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_p8() { +fn test_vst1_p8() { let mut vals = [0_u8; 9]; let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - vst1_p8(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_p8(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -282,11 +316,13 @@ unsafe fn test_vst1_p8() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_p8() { +fn test_vst1q_p8() { let mut vals = [0_u8; 17]; let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - vst1q_p8(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_p8(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -308,11 +344,13 @@ unsafe fn test_vst1q_p8() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_p16() { +fn test_vst1_p16() { let mut vals = [0_u16; 5]; let a = u16x4::new(1, 2, 3, 4); - vst1_p16(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_p16(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -322,11 +360,13 @@ unsafe fn test_vst1_p16() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_p16() { +fn test_vst1q_p16() { let mut vals = [0_u16; 9]; let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - vst1q_p16(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_p16(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -340,22 +380,26 @@ unsafe fn test_vst1q_p16() { } #[simd_test(enable = "neon,aes")] -unsafe fn test_vst1_p64() { +fn test_vst1_p64() { let mut vals = [0_u64; 2]; let a = u64x1::new(1); - vst1_p64(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_p64(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); } #[simd_test(enable = "neon,aes")] -unsafe fn test_vst1q_p64() { +fn test_vst1q_p64() { let mut vals = [0_u64; 3]; let a = u64x2::new(1, 2); - vst1q_p64(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_p64(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0); assert_eq!(vals[1], 1); @@ -363,11 +407,13 @@ unsafe fn test_vst1q_p64() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1_f32() { +fn test_vst1_f32() { let mut vals = [0_f32; 3]; let a = f32x2::new(1., 2.); - vst1_f32(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1_f32(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0.); assert_eq!(vals[1], 1.); @@ -375,11 +421,13 @@ unsafe fn test_vst1_f32() { } #[simd_test(enable = "neon")] -unsafe fn test_vst1q_f32() { +fn test_vst1q_f32() { let mut vals = [0_f32; 5]; let a = f32x4::new(1., 2., 3., 4.); - vst1q_f32(vals[1..].as_mut_ptr(), transmute(a)); + unsafe { + vst1q_f32(vals[1..].as_mut_ptr(), a.into()); + } assert_eq!(vals[0], 0.); assert_eq!(vals[1], 1.); diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs index 9403855f00e0..1e0333444b94 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs @@ -21,19 +21,19 @@ macro_rules! test_vtbl { ) => { #[cfg(target_endian = "little")] #[simd_test(enable = "neon")] - unsafe fn $test_name() { + fn $test_name() { // create table as array, and transmute it to // arm's table type - let table: $table_t = mem::transmute([$($table_v),*]); + let table: $table_t = unsafe { mem::transmute([$($table_v),*]) }; // For each control vector, perform a table lookup and // verify the result: $( { - let ctrl: $ctrl_t = mem::transmute([$($ctrl_v),*]); - let result = $fn_id(table, mem::transmute(ctrl)); - let result: $ctrl_t = mem::transmute(result); - let expected: $ctrl_t = mem::transmute([$($exp_v),*]); + let ctrl: $ctrl_t = unsafe { mem::transmute([$($ctrl_v),*]) }; + let result = $fn_id(table, unsafe { mem::transmute(ctrl) }); + let result: $ctrl_t = unsafe { mem::transmute(result) }; + let expected: $ctrl_t = unsafe { mem::transmute([$($exp_v),*]) }; assert_eq!(result, expected); } )* @@ -171,20 +171,19 @@ macro_rules! test_vtbx { ) => { #[cfg(target_endian = "little")] #[simd_test(enable = "neon")] - unsafe fn $test_name() { + fn $test_name() { // create table as array, and transmute it to // arm's table type - let table: $table_t = mem::transmute([$($table_v),*]); - let ext: $ext_t = mem::transmute([$($ext_v),*]); - + let table: $table_t = unsafe { mem::transmute([$($table_v),*]) }; + let ext: $ext_t = unsafe { mem::transmute([$($ext_v),*]) }; // For each control vector, perform a table lookup and // verify the result: $( { - let ctrl: $ctrl_t = mem::transmute([$($ctrl_v),*]); - let result = $fn_id(ext, table, mem::transmute(ctrl)); - let result: $ctrl_t = mem::transmute(result); - let expected: $ctrl_t = mem::transmute([$($exp_v),*]); + let ctrl: $ctrl_t = unsafe { mem::transmute([$($ctrl_v),*]) }; + let result = $fn_id(ext, table, unsafe { mem::transmute(ctrl) }); + let result: $ctrl_t = unsafe { mem::transmute(result) }; + let expected: $ctrl_t = unsafe { mem::transmute([$($exp_v),*]) }; assert_eq!(result, expected); } )* diff --git a/library/stdarch/crates/core_arch/src/arm_shared/test_support.rs b/library/stdarch/crates/core_arch/src/arm_shared/test_support.rs index e2828f85561d..8117b81cd9a3 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/test_support.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/test_support.rs @@ -111,13 +111,13 @@ macro_rules! V_f32 { macro_rules! to64 { ($t : ident) => { - |v: $t| -> u64 { transmute(v) } + |v: $t| -> u64 { unsafe { transmute(v) } } }; } macro_rules! to128 { ($t : ident) => { - |v: $t| -> u128 { transmute(v) } + |v: $t| -> u128 { unsafe { transmute(v) } } }; } @@ -158,9 +158,7 @@ pub(crate) fn test( macro_rules! gen_test_fn { ($n: ident, $t: ident, $u: ident, $v: ident, $w: ident, $x: ident, $vals: expr, $fill1: expr, $fill2: expr, $cast: expr) => { pub(crate) fn $n(test_fun: fn($v, $v) -> $w, verify_fun: fn($t, $t) -> $u) { - unsafe { - test::<$t, $u, $v, $w, $x>($vals, $fill1, $fill2, $cast, test_fun, verify_fun) - }; + test::<$t, $u, $v, $w, $x>($vals, $fill1, $fill2, $cast, test_fun, verify_fun); } }; } diff --git a/library/stdarch/crates/simd-test-macro/src/lib.rs b/library/stdarch/crates/simd-test-macro/src/lib.rs index b8bb874480a0..92bb40946e1f 100644 --- a/library/stdarch/crates/simd-test-macro/src/lib.rs +++ b/library/stdarch/crates/simd-test-macro/src/lib.rs @@ -17,6 +17,15 @@ pub fn simd_test( item: proc_macro::TokenStream, ) -> proc_macro::TokenStream { let tokens = TokenStream::from(attr).into_iter().collect::>(); + + let target = env::var("TARGET").expect( + "TARGET environment variable should be set for rustc (e.g. TARGET=x86_64-apple-darwin cargo test)" + ); + let target_arch = target + .split('-') + .next() + .unwrap_or_else(|| panic!("target triple contained no \"-\": {target}")); + let (target_features, target_feature_attr) = match &tokens[..] { [] => (Vec::new(), TokenStream::new()), [ @@ -24,13 +33,20 @@ pub fn simd_test( TokenTree::Punct(equals), TokenTree::Literal(literal), ] if enable == "enable" && equals.as_char() == '=' => { - let enable_feature = literal.to_string(); - let enable_feature = enable_feature.trim_start_matches('"').trim_end_matches('"'); + let mut enable_feature = literal + .to_string() + .trim_start_matches('"') + .trim_end_matches('"') + .to_string(); let target_features: Vec<_> = enable_feature .replace('+', "") .split(',') .map(String::from) .collect(); + // Allows using `#[simd_test(enable = "neon")]` on aarch64/armv7 shared tests. + if target_arch == "armv7" && target_features.iter().any(|feat| feat == "neon") { + enable_feature.push_str(",v7"); + } ( target_features, @@ -46,14 +62,7 @@ pub fn simd_test( let item_attrs = std::mem::take(&mut item.attrs); let name = &item.sig.ident; - let target = env::var("TARGET").expect( - "TARGET environment variable should be set for rustc (e.g. TARGET=x86_64-apple-darwin cargo test)" - ); - let macro_test = match target - .split('-') - .next() - .unwrap_or_else(|| panic!("target triple contained no \"-\": {target}")) - { + let macro_test = match target_arch { "i686" | "x86_64" | "i586" => "is_x86_feature_detected", "arm" | "armv7" | "thumbv7neon" => "is_arm_feature_detected", "aarch64" | "arm64ec" | "aarch64_be" => "is_aarch64_feature_detected", @@ -85,10 +94,20 @@ pub fn simd_test( let mut detect_missing_features = TokenStream::new(); for feature in target_features { - let q = quote_spanned! { - proc_macro2::Span::call_site() => - if !::std::arch::#macro_test!(#feature) { - missing_features.push(#feature); + let q = if target_arch == "armv7" && feature == "fp16" { + // "fp16" cannot be checked at runtime + quote_spanned! { + proc_macro2::Span::call_site() => + if !cfg!(target_feature = #feature) { + missing_features.push(#feature); + } + } + } else { + quote_spanned! { + proc_macro2::Span::call_site() => + if !::std::arch::#macro_test!(#feature) { + missing_features.push(#feature); + } } }; q.to_tokens(&mut detect_missing_features); From 57e44f5046058d467b9f8731f4c49885a8664d8e Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 2 Jan 2026 18:39:06 +0100 Subject: [PATCH 0077/1843] skip codegen for intrinsics with big fallback bodies if backend does not need them --- compiler/rustc_codegen_llvm/src/lib.rs | 6 +++++- compiler/rustc_codegen_ssa/src/traits/backend.rs | 6 ++++++ compiler/rustc_interface/src/interface.rs | 1 + compiler/rustc_monomorphize/src/collector.rs | 9 +++++---- compiler/rustc_session/src/session.rs | 7 ++++++- tests/codegen-llvm/intrinsics/carrying_mul_add.rs | 5 ++--- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 095274744993..875b60ef6ddb 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -42,7 +42,7 @@ use rustc_middle::ty::TyCtxt; use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; -use rustc_span::Symbol; +use rustc_span::{Symbol, sym}; use rustc_target::spec::{RelocModel, TlsModel}; use crate::llvm::ToLlvmBool; @@ -333,6 +333,10 @@ impl CodegenBackend for LlvmCodegenBackend { target_config(sess) } + fn replaced_intrinsics(&self) -> Vec { + vec![sym::unchecked_funnel_shl, sym::unchecked_funnel_shr, sym::carrying_mul_add] + } + fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Box { Box::new(rustc_codegen_ssa::base::codegen_crate( LlvmCodegenBackend(()), diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index cb74e2e46d65..2ca24836d046 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -78,6 +78,12 @@ pub trait CodegenBackend { fn print_version(&self) {} + /// Returns a list of all intrinsics that this backend definitely + /// replaces, which means their fallback bodies do not need to be monomorphized. + fn replaced_intrinsics(&self) -> Vec { + vec![] + } + /// Value printed by `--print=backend-has-zstd`. /// /// Used by compiletest to determine whether tests involving zstd compression diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index c0f8f33692e8..c60cf9b8992a 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -496,6 +496,7 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se ); codegen_backend.init(&sess); + sess.replaced_intrinsics = FxHashSet::from_iter(codegen_backend.replaced_intrinsics()); let cfg = parse_cfg(sess.dcx(), config.crate_cfg); let mut cfg = config::build_configuration(&sess, cfg); diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 4b2f8e03afc1..622fdeb4e81c 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -1001,11 +1001,12 @@ fn visit_instance_use<'tcx>( if tcx.should_codegen_locally(panic_instance) { output.push(create_fn_mono_item(tcx, panic_instance, source)); } - } else if !intrinsic.must_be_overridden { + } else if !intrinsic.must_be_overridden + && !tcx.sess.replaced_intrinsics.contains(&intrinsic.name) + { // Codegen the fallback body of intrinsics with fallback bodies. - // We explicitly skip this otherwise to ensure we get a linker error - // if anyone tries to call this intrinsic and the codegen backend did not - // override the implementation. + // We have to skip this otherwise as there's no body to codegen. + // We also skip intrinsics the backend handles, to reduce monomorphizations. let instance = ty::Instance::new_raw(instance.def_id(), instance.args); if tcx.should_codegen_locally(instance) { output.push(create_fn_mono_item(tcx, instance, source)); diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 1a0ec600af47..b3027d902b4c 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -8,7 +8,7 @@ use std::{env, io}; use rand::{RngCore, rng}; use rustc_data_structures::base_n::{CASE_INSENSITIVE, ToBaseN}; use rustc_data_structures::flock; -use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_data_structures::profiling::{SelfProfiler, SelfProfilerRef}; use rustc_data_structures::sync::{DynSend, DynSync, Lock, MappedReadGuard, ReadGuard, RwLock}; use rustc_errors::annotate_snippet_emitter_writer::AnnotateSnippetEmitter; @@ -154,6 +154,10 @@ pub struct Session { /// preserved with a flag like `-C save-temps`, since these files may be /// hard linked. pub invocation_temp: Option, + + /// The names of intrinsics that the current codegen backend replaces + /// with its own implementations. + pub replaced_intrinsics: FxHashSet, } #[derive(Clone, Copy)] @@ -1091,6 +1095,7 @@ pub fn build_session( target_filesearch, host_filesearch, invocation_temp, + replaced_intrinsics: FxHashSet::default(), // filled by `run_compiler` }; validate_commandline_args_with_session_available(&sess); diff --git a/tests/codegen-llvm/intrinsics/carrying_mul_add.rs b/tests/codegen-llvm/intrinsics/carrying_mul_add.rs index 21fb49a3786a..844f4f6cff64 100644 --- a/tests/codegen-llvm/intrinsics/carrying_mul_add.rs +++ b/tests/codegen-llvm/intrinsics/carrying_mul_add.rs @@ -11,10 +11,9 @@ use std::intrinsics::{carrying_mul_add, fallback}; -// The fallbacks are emitted even when they're never used, but optimize out. +// The fallbacks should not be emitted. -// RAW: wide_mul_u128 -// OPT-NOT: wide_mul_u128 +// NOT: wide_mul_u128 // CHECK-LABEL: @cma_u8 #[no_mangle] From ebd4bc9224a5de37ce151745b00cb8039fef4385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20K=C3=BCber?= Date: Sat, 27 Dec 2025 20:56:37 +0000 Subject: [PATCH 0078/1843] Support long diff conflict markers git can be configured to use more than 7 characters for conflict markers, and jj automatically uses longer conflict markers when the text contains any char sequence that could be confused with conflict markers. Ensure that we only point at markers that are consistent with the start marker's length. Ensure that we only consider char sequences at the beginning of a line as a diff marker. --- .../rustc_parse/src/parser/diagnostics.rs | 54 +++++++++++++++---- .../diff-markers/long-conflict-markers.rs | 11 ++++ .../diff-markers/long-conflict-markers.stderr | 20 +++++++ 3 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 tests/ui/parser/diff-markers/long-conflict-markers.rs create mode 100644 tests/ui/parser/diff-markers/long-conflict-markers.stderr diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 70ec80a50812..9c4895e391d2 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -3037,27 +3037,53 @@ impl<'a> Parser<'a> { long_kind: &TokenKind, short_kind: &TokenKind, ) -> bool { - (0..3).all(|i| self.look_ahead(i, |tok| tok == long_kind)) - && self.look_ahead(3, |tok| tok == short_kind) + if long_kind == short_kind { + // For conflict marker chars like `%` and `\`. + (0..7).all(|i| self.look_ahead(i, |tok| tok == long_kind)) + } else { + // For conflict marker chars like `<` and `|`. + (0..3).all(|i| self.look_ahead(i, |tok| tok == long_kind)) + && self.look_ahead(3, |tok| tok == short_kind || tok == long_kind) + } } - fn conflict_marker(&mut self, long_kind: &TokenKind, short_kind: &TokenKind) -> Option { + fn conflict_marker( + &mut self, + long_kind: &TokenKind, + short_kind: &TokenKind, + expected: Option, + ) -> Option<(Span, usize)> { if self.is_vcs_conflict_marker(long_kind, short_kind) { let lo = self.token.span; - for _ in 0..4 { - self.bump(); + if self.psess.source_map().span_to_margin(lo) != Some(0) { + return None; } - return Some(lo.to(self.prev_token.span)); + let mut len = 0; + while self.token.kind == *long_kind || self.token.kind == *short_kind { + if self.token.kind.break_two_token_op(1).is_some() { + len += 2; + } else { + len += 1; + } + self.bump(); + if expected == Some(len) { + break; + } + } + if expected.is_some() && expected != Some(len) { + return None; + } + return Some((lo.to(self.prev_token.span), len)); } None } pub(super) fn recover_vcs_conflict_marker(&mut self) { // <<<<<<< - let Some(start) = self.conflict_marker(&TokenKind::Shl, &TokenKind::Lt) else { + let Some((start, len)) = self.conflict_marker(&TokenKind::Shl, &TokenKind::Lt, None) else { return; }; - let mut spans = Vec::with_capacity(3); + let mut spans = Vec::with_capacity(2); spans.push(start); // ||||||| let mut middlediff3 = None; @@ -3069,13 +3095,19 @@ impl<'a> Parser<'a> { if self.token == TokenKind::Eof { break; } - if let Some(span) = self.conflict_marker(&TokenKind::OrOr, &TokenKind::Or) { + if let Some((span, _)) = + self.conflict_marker(&TokenKind::OrOr, &TokenKind::Or, Some(len)) + { middlediff3 = Some(span); } - if let Some(span) = self.conflict_marker(&TokenKind::EqEq, &TokenKind::Eq) { + if let Some((span, _)) = + self.conflict_marker(&TokenKind::EqEq, &TokenKind::Eq, Some(len)) + { middle = Some(span); } - if let Some(span) = self.conflict_marker(&TokenKind::Shr, &TokenKind::Gt) { + if let Some((span, _)) = + self.conflict_marker(&TokenKind::Shr, &TokenKind::Gt, Some(len)) + { spans.push(span); end = Some(span); break; diff --git a/tests/ui/parser/diff-markers/long-conflict-markers.rs b/tests/ui/parser/diff-markers/long-conflict-markers.rs new file mode 100644 index 000000000000..f92615fb75b4 --- /dev/null +++ b/tests/ui/parser/diff-markers/long-conflict-markers.rs @@ -0,0 +1,11 @@ +enum E { + Foo { +<<<<<<<<< HEAD //~ ERROR encountered diff marker + x: u8, +||||||| + z: (), +========= + y: i8, +>>>>>>>>> branch + } +} diff --git a/tests/ui/parser/diff-markers/long-conflict-markers.stderr b/tests/ui/parser/diff-markers/long-conflict-markers.stderr new file mode 100644 index 000000000000..c83aaf16b741 --- /dev/null +++ b/tests/ui/parser/diff-markers/long-conflict-markers.stderr @@ -0,0 +1,20 @@ +error: encountered diff marker + --> $DIR/long-conflict-markers.rs:3:1 + | +LL | <<<<<<<<< HEAD + | ^^^^^^^^^ between this marker and `=======` is the code that you are merging into +... +LL | ========= + | --------- between this marker and `>>>>>>>` is the incoming code +LL | y: i8, +LL | >>>>>>>>> branch + | ^^^^^^^^^ this marker concludes the conflict region + | + = note: conflict markers indicate that a merge was started but could not be completed due to merge conflicts + to resolve a conflict, keep only the code you want and then delete the lines containing conflict markers + = help: if you are in a merge, the top section is the code you already had checked out and the bottom section is the new code + if you are in a rebase, the top section is the code being rebased onto and the bottom section is the code you had checked out which is being rebased + = note: for an explanation on these markers from the `git` documentation, visit + +error: aborting due to 1 previous error + From f091a8acad5c52378be2f1b6f29cc2d21ff7bf5f Mon Sep 17 00:00:00 2001 From: A4-Tacks Date: Sun, 4 Jan 2026 20:54:19 +0800 Subject: [PATCH 0079/1843] Fix complete semicolon in array expression Example --- ```rust fn foo() {} fn bar() { let _ = [fo$0]; } ``` **Before this PR** ```rust fn foo() {} fn bar() { let _ = [foo();$0]; } ``` **After this PR** ```rust fn foo() {} fn bar() { let _ = [foo()$0]; } ``` --- .../crates/ide-completion/src/context.rs | 5 ++++- .../ide-completion/src/render/function.rs | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/context.rs b/src/tools/rust-analyzer/crates/ide-completion/src/context.rs index d116f665adbd..7f3e69c8ab30 100644 --- a/src/tools/rust-analyzer/crates/ide-completion/src/context.rs +++ b/src/tools/rust-analyzer/crates/ide-completion/src/context.rs @@ -821,7 +821,10 @@ impl<'db> CompletionContext<'db> { CompleteSemicolon::DoNotComplete } else if let Some(term_node) = sema.token_ancestors_with_macros(token.clone()).find(|node| { - matches!(node.kind(), BLOCK_EXPR | MATCH_ARM | CLOSURE_EXPR | ARG_LIST | PAREN_EXPR) + matches!( + node.kind(), + BLOCK_EXPR | MATCH_ARM | CLOSURE_EXPR | ARG_LIST | PAREN_EXPR | ARRAY_EXPR + ) }) { let next_token = iter::successors(token.next_token(), |it| it.next_token()) diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/render/function.rs b/src/tools/rust-analyzer/crates/ide-completion/src/render/function.rs index 4713b1f1afa7..475e00dfcf29 100644 --- a/src/tools/rust-analyzer/crates/ide-completion/src/render/function.rs +++ b/src/tools/rust-analyzer/crates/ide-completion/src/render/function.rs @@ -905,6 +905,25 @@ fn baz(_: impl FnOnce()) {} fn bar() { baz(foo()$0); } +"#, + ); + } + + #[test] + fn no_semicolon_in_array() { + check_edit( + r#"foo"#, + r#" +fn foo() {} +fn bar() { + let _ = [fo$0]; +} +"#, + r#" +fn foo() {} +fn bar() { + let _ = [foo()$0]; +} "#, ); } From 65c17223c1fa4792be28f533c17f680d9eef0e66 Mon Sep 17 00:00:00 2001 From: is57primenumber <58158444+is57primenumber@users.noreply.github.com> Date: Wed, 17 Dec 2025 02:30:19 +0900 Subject: [PATCH 0080/1843] add CSE optimization tests for iterating over slice --- tests/codegen-llvm/slice_cse_optimization.rs | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/codegen-llvm/slice_cse_optimization.rs diff --git a/tests/codegen-llvm/slice_cse_optimization.rs b/tests/codegen-llvm/slice_cse_optimization.rs new file mode 100644 index 000000000000..2b1851d8ae44 --- /dev/null +++ b/tests/codegen-llvm/slice_cse_optimization.rs @@ -0,0 +1,46 @@ +//! Various iterating method over slice correctly optimized using common subexpression elimination. +//! Checks function has memory(argmem: read) attribute. +//! Regression test for . +//@ compile-flags: -O + +#![crate_type = "lib"] +// CHECK-LABEL: @has_zero_iter +// CHECK-SAME: #[[ATTR:[0-9]+]] +#[inline(never)] +#[unsafe(no_mangle)] +pub fn has_zero_iter(xs: &[u8]) -> bool { + xs.iter().any(|&x| x == 0) +} + +// CHECK-LABEL: @has_zero_ptr +// CHECK-SAME: #[[ATTR]] +#[inline(never)] +#[unsafe(no_mangle)] +fn has_zero_ptr(xs: &[u8]) -> bool { + let range = xs.as_ptr_range(); + let mut start = range.start; + let end = range.end; + while start < end { + unsafe { + if *start == 0 { + return true; + } + start = start.add(1); + } + } + false +} +// CHECK-LABEL: @has_zero_for +// CHECK-SAME: #[[ATTR]] +#[inline(never)] +#[unsafe(no_mangle)] +fn has_zero_for(xs: &[u8]) -> bool { + for x in xs { + if *x == 0 { + return true; + } + } + false +} + +// CHECK: attributes #[[ATTR]] = { {{.*}}memory(argmem: read){{.*}} } From de505d24db776bf399b9932f673f3f3de8c591a0 Mon Sep 17 00:00:00 2001 From: Martin Nordholts Date: Mon, 5 Jan 2026 08:08:02 +0100 Subject: [PATCH 0081/1843] tests/debuginfo/basic-stepping.rs: Don't mix `compile-flags` with `ignore-*` We want directives nice and tidy. --- tests/debuginfo/basic-stepping.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/debuginfo/basic-stepping.rs b/tests/debuginfo/basic-stepping.rs index ffb5d87710dc..72d4e354a6f1 100644 --- a/tests/debuginfo/basic-stepping.rs +++ b/tests/debuginfo/basic-stepping.rs @@ -5,9 +5,11 @@ //@ ignore-aarch64: Doesn't work yet. //@ ignore-loongarch64: Doesn't work yet. //@ ignore-riscv64: Doesn't work yet. -//@ compile-flags: -g //@ ignore-backends: gcc +// Debugger tests need debuginfo +//@ compile-flags: -g + //@ gdb-command: run // FIXME(#97083): Should we be able to break on initialization of zero-sized types? // FIXME(#97083): Right now the first breakable line is: From 622572f6df87a8b374159a8ee95a1195a600754b Mon Sep 17 00:00:00 2001 From: Martin Nordholts Date: Tue, 7 Oct 2025 06:41:09 +0200 Subject: [PATCH 0082/1843] tests/debuginfo/basic-stepping.rs: Add revisions `default-mir-passes`, `no-SingleUseConsts-mir-pass` To prevent regressions our test must cover the code both inside and outside of the `SingleUseConsts` MIR pass. Use revisions for that. --- tests/debuginfo/basic-stepping.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/debuginfo/basic-stepping.rs b/tests/debuginfo/basic-stepping.rs index 72d4e354a6f1..744f8f5dd654 100644 --- a/tests/debuginfo/basic-stepping.rs +++ b/tests/debuginfo/basic-stepping.rs @@ -10,6 +10,10 @@ // Debugger tests need debuginfo //@ compile-flags: -g +// FIXME(#128945): SingleUseConsts shouldn't need to be disabled. +//@ revisions: default-mir-passes no-SingleUseConsts-mir-pass +//@ [no-SingleUseConsts-mir-pass] compile-flags: -Zmir-enable-passes=-SingleUseConsts + //@ gdb-command: run // FIXME(#97083): Should we be able to break on initialization of zero-sized types? // FIXME(#97083): Right now the first breakable line is: @@ -17,12 +21,12 @@ //@ gdb-command: next //@ gdb-check: let d = c = 99; //@ gdb-command: next -// FIXME(#33013): gdb-check: let e = "hi bob"; -// FIXME(#33013): gdb-command: next -// FIXME(#33013): gdb-check: let f = b"hi bob"; -// FIXME(#33013): gdb-command: next -// FIXME(#33013): gdb-check: let g = b'9'; -// FIXME(#33013): gdb-command: next +//@ [no-SingleUseConsts-mir-pass] gdb-check: let e = "hi bob"; +//@ [no-SingleUseConsts-mir-pass] gdb-command: next +//@ [no-SingleUseConsts-mir-pass] gdb-check: let f = b"hi bob"; +//@ [no-SingleUseConsts-mir-pass] gdb-command: next +//@ [no-SingleUseConsts-mir-pass] gdb-check: let g = b'9'; +//@ [no-SingleUseConsts-mir-pass] gdb-command: next //@ gdb-check: let h = ["whatever"; 8]; //@ gdb-command: next //@ gdb-check: let i = [1,2,3,4]; From c0ecf1ad40a953f01d8ebf5c035eef268cc001f8 Mon Sep 17 00:00:00 2001 From: Till Adam Date: Fri, 2 Jan 2026 21:57:02 +0100 Subject: [PATCH 0083/1843] Implement Span::line() and Span::column() for proc-macro server Add proper line/column resolution for proc-macro spans via a callback mechanism. Previously these methods returned hardcoded 1 values. The implementation adds: - SubRequest::LineColumn and SubResponse::LineColumnResult to the bidirectional protocol - ProcMacroClientInterface::line_column() method - Callback handling in load-cargo using LineIndex - Server implementation in RaSpanServer that uses the callback - a test for Span::line() and Span::column() in proc-macro server Add fn_like_span_line_column test proc-macro that exercises the new line/column API, and a corresponding test with a mock callback. --- src/tools/rust-analyzer/Cargo.lock | 1 + .../crates/load-cargo/src/lib.rs | 42 +++++++++--- .../src/bidirectional_protocol/msg.rs | 18 ++++- .../proc-macro-srv-cli/src/main_loop.rs | 14 ++++ .../crates/proc-macro-srv/Cargo.toml | 1 + .../proc-macro-test/imp/src/lib.rs | 10 +++ .../crates/proc-macro-srv/src/lib.rs | 2 + .../src/server_impl/rust_analyzer_span.rs | 10 ++- .../crates/proc-macro-srv/src/tests/mod.rs | 15 +++++ .../crates/proc-macro-srv/src/tests/utils.rs | 66 ++++++++++++++++++- 10 files changed, 161 insertions(+), 18 deletions(-) diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock index 42eaeb01f1f2..8188fbf96064 100644 --- a/src/tools/rust-analyzer/Cargo.lock +++ b/src/tools/rust-analyzer/Cargo.lock @@ -1864,6 +1864,7 @@ dependencies = [ "intern", "libc", "libloading", + "line-index 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "memmap2", "object", "paths", diff --git a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs index e8d98b1ce661..33468a5003c3 100644 --- a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs +++ b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs @@ -554,14 +554,12 @@ impl ProcMacroExpander for Expander { Ok(SubResponse::LocalFilePathResult { name }) } SubRequest::SourceText { file_id, ast_id, start, end } => { - let ast_id = span::ErasedFileAstId::from_raw(ast_id); - let editioned_file_id = span::EditionedFileId::from_raw(file_id); - let span = Span { - range: TextRange::new(TextSize::from(start), TextSize::from(end)), - anchor: SpanAnchor { file_id: editioned_file_id, ast_id }, - ctx: SyntaxContext::root(editioned_file_id.edition()), - }; - let range = db.resolve_span(span); + let range = resolve_sub_span( + db, + file_id, + ast_id, + TextRange::new(TextSize::from(start), TextSize::from(end)), + ); let source = db.file_text(range.file_id.file_id(db)).text(db); let text = source .get(usize::from(range.range.start())..usize::from(range.range.end())) @@ -569,6 +567,18 @@ impl ProcMacroExpander for Expander { Ok(SubResponse::SourceTextResult { text }) } + SubRequest::LineColumn { file_id, ast_id, offset } => { + let range = + resolve_sub_span(db, file_id, ast_id, TextRange::empty(TextSize::from(offset))); + let source = db.file_text(range.file_id.file_id(db)).text(db); + let line_index = ide_db::line_index::LineIndex::new(source); + let (line, column) = line_index + .try_line_col(range.range.start()) + .map(|lc| (lc.line + 1, lc.col + 1)) + .unwrap_or((1, 1)); + // proc_macro::Span line/column are 1-based + Ok(SubResponse::LineColumnResult { line, column }) + } SubRequest::FilePath { file_id } => { let file_id = FileId::from_raw(file_id); let source_root_id = db.file_source_root(file_id).source_root_id(db); @@ -603,6 +613,22 @@ impl ProcMacroExpander for Expander { } } +fn resolve_sub_span( + db: &dyn ExpandDatabase, + file_id: u32, + ast_id: u32, + range: TextRange, +) -> hir_expand::FileRange { + let ast_id = span::ErasedFileAstId::from_raw(ast_id); + let editioned_file_id = span::EditionedFileId::from_raw(file_id); + let span = Span { + range, + anchor: SpanAnchor { file_id: editioned_file_id, ast_id }, + ctx: SyntaxContext::root(editioned_file_id.edition()), + }; + db.resolve_span(span) +} + #[cfg(test)] mod tests { use ide_db::base_db::RootQueryDb; diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs index e41f8a5d7da7..0e3b700dcc5a 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs @@ -13,13 +13,25 @@ pub enum SubRequest { FilePath { file_id: u32 }, SourceText { file_id: u32, ast_id: u32, start: u32, end: u32 }, LocalFilePath { file_id: u32 }, + LineColumn { file_id: u32, ast_id: u32, offset: u32 }, } #[derive(Debug, Serialize, Deserialize)] pub enum SubResponse { - FilePathResult { name: String }, - SourceTextResult { text: Option }, - LocalFilePathResult { name: Option }, + FilePathResult { + name: String, + }, + SourceTextResult { + text: Option, + }, + LocalFilePathResult { + name: Option, + }, + /// Line and column are 1-based. + LineColumnResult { + line: u32, + column: u32, + }, } #[derive(Debug, Serialize, Deserialize)] diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs index b2f4b96bd255..22536a4e52b1 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs @@ -220,6 +220,20 @@ impl proc_macro_srv::ProcMacroClientInterface for ProcMacroClientHandl _ => None, } } + + fn line_column(&mut self, span: proc_macro_srv::span::Span) -> Option<(u32, u32)> { + let proc_macro_srv::span::Span { range, anchor, ctx: _ } = span; + match self.roundtrip(bidirectional::SubRequest::LineColumn { + file_id: anchor.file_id.as_u32(), + ast_id: anchor.ast_id.into_raw(), + offset: range.start().into(), + }) { + Some(bidirectional::BidirectionalMessage::SubResponse( + bidirectional::SubResponse::LineColumnResult { line, column }, + )) => Some((line, column)), + _ => None, + } + } } fn handle_expand_ra( diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml b/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml index 361017178409..8e5617f8a20e 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml @@ -31,6 +31,7 @@ libc.workspace = true [dev-dependencies] expect-test.workspace = true +line-index.workspace = true # used as proc macro test targets proc-macro-test.path = "./proc-macro-test" diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs index b4fac26d6e72..06c76b6d0381 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/proc-macro-test/imp/src/lib.rs @@ -79,6 +79,16 @@ pub fn fn_like_span_ops(args: TokenStream) -> TokenStream { TokenStream::from_iter(vec![first, second, third]) } +/// Returns the line and column of the first token's span as two integer literals. +#[proc_macro] +pub fn fn_like_span_line_column(args: TokenStream) -> TokenStream { + let first = args.into_iter().next().unwrap(); + let span = first.span(); + let line = Literal::usize_unsuffixed(span.line()); + let column = Literal::usize_unsuffixed(span.column()); + TokenStream::from_iter(vec![TokenTree::Literal(line), TokenTree::Literal(column)]) +} + #[proc_macro_attribute] pub fn attr_noop(_args: TokenStream, item: TokenStream) -> TokenStream { item diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs index f2d1dfbba4cc..c1ef49a7176b 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs @@ -98,6 +98,8 @@ pub trait ProcMacroClientInterface { fn file(&mut self, file_id: span::FileId) -> String; fn source_text(&mut self, span: Span) -> Option; fn local_file(&mut self, file_id: span::FileId) -> Option; + /// Line and column are 1-based. + fn line_column(&mut self, span: Span) -> Option<(u32, u32)>; } const EXPANDER_STACK_SIZE: usize = 8 * 1024 * 1024; diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index 32725afc5527..3a25391b573b 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -257,14 +257,12 @@ impl server::Span for RaSpanServer<'_> { Span { range: TextRange::empty(span.range.start()), ..span } } - fn line(&mut self, _span: Self::Span) -> usize { - // FIXME requires db to resolve line index, THIS IS NOT INCREMENTAL - 1 + fn line(&mut self, span: Self::Span) -> usize { + self.callback.as_mut().and_then(|cb| cb.line_column(span)).map_or(1, |(l, _)| l as usize) } - fn column(&mut self, _span: Self::Span) -> usize { - // FIXME requires db to resolve line index, THIS IS NOT INCREMENTAL - 1 + fn column(&mut self, span: Self::Span) -> usize { + self.callback.as_mut().and_then(|cb| cb.line_column(span)).map_or(1, |(_, c)| c as usize) } } diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/mod.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/mod.rs index 20507a6def54..ebef9a9a519a 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/mod.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/mod.rs @@ -703,6 +703,7 @@ fn list_test_macros() { fn_like_mk_idents [Bang] fn_like_span_join [Bang] fn_like_span_ops [Bang] + fn_like_span_line_column [Bang] attr_noop [Attr] attr_panic [Attr] attr_error [Attr] @@ -712,3 +713,17 @@ fn list_test_macros() { DeriveError [CustomDerive]"#]] .assert_eq(&res); } + +#[test] +fn test_fn_like_span_line_column() { + assert_expand_with_callback( + "fn_like_span_line_column", + // Input text with known position: "hello" starts at offset 1 (line 2, column 1 in 1-based) + " +hello", + expect![[r#" + LITER 42:Root[0000, 0]@0..100#ROOT2024 Integer 2 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Integer 1 + "#]], + ); +} diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs index 61fcd810b1d9..81ff1965d68b 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs @@ -6,7 +6,8 @@ use span::{ }; use crate::{ - EnvSnapshot, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, token_stream::TokenStream, + EnvSnapshot, ProcMacroClientInterface, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, + token_stream::TokenStream, }; fn parse_string(call_site: SpanId, src: &str) -> TokenStream { @@ -109,3 +110,66 @@ pub(crate) fn list() -> Vec { let res = srv.list_macros(&dylib_path).unwrap(); res.into_iter().map(|(name, kind)| format!("{name} [{kind:?}]")).collect() } + +/// A mock callback for testing that computes line/column from the input text. +struct MockCallback<'a> { + text: &'a str, +} + +impl ProcMacroClientInterface for MockCallback<'_> { + fn source_text(&mut self, span: Span) -> Option { + self.text + .get(usize::from(span.range.start())..usize::from(span.range.end())) + .map(ToOwned::to_owned) + } + + fn file(&mut self, _file_id: FileId) -> String { + String::new() + } + + fn local_file(&mut self, _file_id: FileId) -> Option { + None + } + + fn line_column(&mut self, span: Span) -> Option<(u32, u32)> { + let line_index = line_index::LineIndex::new(self.text); + let line_col = line_index.try_line_col(span.range.start())?; + // proc_macro uses 1-based line/column + Some((line_col.line as u32 + 1, line_col.col as u32 + 1)) + } +} + +pub fn assert_expand_with_callback( + macro_name: &str, + #[rust_analyzer::rust_fixture] ra_fixture: &str, + expect_spanned: Expect, +) { + let path = proc_macro_test_dylib_path(); + let expander = dylib::Expander::new(&temp_dir::TempDir::new().unwrap(), &path).unwrap(); + + let def_site = Span { + range: TextRange::new(0.into(), 150.into()), + anchor: SpanAnchor { + file_id: EditionedFileId::current_edition(FileId::from_raw(41)), + ast_id: ROOT_ERASED_FILE_AST_ID, + }, + ctx: SyntaxContext::root(span::Edition::CURRENT), + }; + let call_site = Span { + range: TextRange::new(0.into(), 100.into()), + anchor: SpanAnchor { + file_id: EditionedFileId::current_edition(FileId::from_raw(42)), + ast_id: ROOT_ERASED_FILE_AST_ID, + }, + ctx: SyntaxContext::root(span::Edition::CURRENT), + }; + let mixed_site = call_site; + + let fixture = parse_string_spanned(call_site.anchor, call_site.ctx, ra_fixture); + + let mut callback = MockCallback { text: ra_fixture }; + let res = expander + .expand(macro_name, fixture, None, def_site, call_site, mixed_site, Some(&mut callback)) + .unwrap(); + expect_spanned.assert_eq(&format!("{res:?}")); +} From 4fbc52085fda746d8128a79b878a5008e94a81fe Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 5 Jan 2026 11:30:51 +0100 Subject: [PATCH 0084/1843] perf: Re-use scratch allocations for `try_evaluate_obligations` --- .../crates/hir-ty/src/next_solver/fulfill.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/fulfill.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/fulfill.rs index 0fe073297279..a8bff44a0258 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/fulfill.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/fulfill.rs @@ -48,6 +48,7 @@ pub struct FulfillmentCtxt<'db> { /// use the context in exactly this snapshot. #[expect(unused)] usable_in_snapshot: usize, + try_evaluate_obligations_scratch: PendingObligations<'db>, } #[derive(Default, Debug, Clone)] @@ -115,6 +116,7 @@ impl<'db> FulfillmentCtxt<'db> { FulfillmentCtxt { obligations: Default::default(), usable_in_snapshot: infcx.num_open_snapshots(), + try_evaluate_obligations_scratch: Default::default(), } } } @@ -162,12 +164,12 @@ impl<'db> FulfillmentCtxt<'db> { // and select. They should use a different `ObligationCtxt` instead. Then we'll be also able // to not put the obligations queue in `InferenceTable`'s snapshots. // assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots()); + self.try_evaluate_obligations_scratch.clear(); let mut errors = Vec::new(); - let mut obligations = Vec::new(); loop { let mut any_changed = false; - obligations.extend(self.obligations.drain_pending(|_| true)); - for (mut obligation, stalled_on) in obligations.drain(..) { + self.try_evaluate_obligations_scratch.extend(self.obligations.drain_pending(|_| true)); + for (mut obligation, stalled_on) in self.try_evaluate_obligations_scratch.drain(..) { if obligation.recursion_depth >= infcx.interner.recursion_limit() { self.obligations.on_fulfillment_overflow(infcx); // Only return true errors that we have accumulated while processing. From c5f67830c2a1395da5589a87e84facbcfe30ea54 Mon Sep 17 00:00:00 2001 From: Rune Tynan Date: Fri, 19 Dec 2025 10:27:38 -0800 Subject: [PATCH 0085/1843] Convert Windows to use check_shim_sig instead of check_shim_sig_lenient --- src/tools/miri/src/shims/sig.rs | 11 +- .../miri/src/shims/windows/foreign_items.rs | 605 ++++++++++++++---- src/tools/miri/tests/pass-dep/getrandom.rs | 2 + src/tools/miri/tests/pass/tls/windows-tls.rs | 8 +- 4 files changed, 497 insertions(+), 129 deletions(-) diff --git a/src/tools/miri/src/shims/sig.rs b/src/tools/miri/src/shims/sig.rs index bc5e7f584f50..a49689d11fd7 100644 --- a/src/tools/miri/src/shims/sig.rs +++ b/src/tools/miri/src/shims/sig.rs @@ -17,11 +17,14 @@ pub struct ShimSig<'tcx, const ARGS: usize> { /// Construct a `ShimSig` with convenient syntax: /// ```rust,ignore -/// shim_sig!(this, extern "C" fn (*const T, i32) -> usize) +/// shim_sig!(extern "C" fn (*const T, i32) -> usize) /// ``` +/// +/// In type position, `winapi::` can be used as a shorthand for the full path used by +/// `windows_ty_layout`. #[macro_export] macro_rules! shim_sig { - (extern $abi:literal fn($($arg:ty),*) -> $ret:ty) => { + (extern $abi:literal fn($($arg:ty),* $(,)?) -> $ret:ty) => { |this| $crate::shims::sig::ShimSig { abi: std::str::FromStr::from_str($abi).expect("incorrect abi specified"), args: [$(shim_sig_arg!(this, $arg)),*], @@ -53,7 +56,9 @@ macro_rules! shim_sig_arg { "*const _" => $this.machine.layouts.const_raw_ptr.ty, "*mut _" => $this.machine.layouts.mut_raw_ptr.ty, ty if let Some(libc_ty) = ty.strip_prefix("libc::") => $this.libc_ty_layout(libc_ty).ty, - ty => panic!("unsupported signature type {ty:?}"), + ty if let Some(win_ty) = ty.strip_prefix("winapi::") => + $this.windows_ty_layout(win_ty).ty, + ty => helpers::path_ty_layout($this, &ty.split("::").collect::>()).ty, } }}; } diff --git a/src/tools/miri/src/shims/windows/foreign_items.rs b/src/tools/miri/src/shims/windows/foreign_items.rs index 3bc52dddfe8b..1905fb22e26a 100644 --- a/src/tools/miri/src/shims/windows/foreign_items.rs +++ b/src/tools/miri/src/shims/windows/foreign_items.rs @@ -2,11 +2,11 @@ use std::ffi::OsStr; use std::path::{self, Path, PathBuf}; use std::{io, iter, str}; -use rustc_abi::{Align, CanonAbi, Size, X86Call}; +use rustc_abi::{Align, Size}; use rustc_middle::ty::Ty; use rustc_span::Symbol; use rustc_target::callconv::FnAbi; -use rustc_target::spec::{Arch, Env}; +use rustc_target::spec::Env; use self::shims::windows::handle::{Handle, PseudoHandle}; use crate::shims::os_str::bytes_to_os_str; @@ -137,65 +137,93 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ) -> InterpResult<'tcx, EmulateItemResult> { let this = self.eval_context_mut(); - // According to - // https://github.com/rust-lang/rust/blob/fb00adbdb69266f10df95a4527b767b0ad35ea48/compiler/rustc_target/src/spec/mod.rs#L2766-L2768, - // x86-32 Windows uses a different calling convention than other Windows targets - // for the "system" ABI. - let sys_conv = if this.tcx.sess.target.arch == Arch::X86 { - CanonAbi::X86(X86Call::Stdcall) - } else { - CanonAbi::C - }; - // See `fn emulate_foreign_item_inner` in `shims/foreign_items.rs` for the general pattern. // Windows API stubs. - // HANDLE = isize - // NTSTATUS = LONH = i32 + // HANDLE = *mut c_void (formerly: isize) + // NTSTATUS = LONG = i32 // DWORD = ULONG = u32 // BOOL = i32 // BOOLEAN = u8 match link_name.as_str() { // Environment related shims "GetEnvironmentVariableW" => { - let [name, buf, size] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [name, buf, size] = this.check_shim_sig( + shim_sig!(extern "system" fn(*const _, *mut _, u32) -> u32), + link_name, + abi, + args, + )?; let result = this.GetEnvironmentVariableW(name, buf, size)?; this.write_scalar(result, dest)?; } "SetEnvironmentVariableW" => { - let [name, value] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [name, value] = this.check_shim_sig( + shim_sig!(extern "system" fn(*const _, *const _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let result = this.SetEnvironmentVariableW(name, value)?; this.write_scalar(result, dest)?; } "GetEnvironmentStringsW" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> *mut _), + link_name, + abi, + args, + )?; let result = this.GetEnvironmentStringsW()?; this.write_pointer(result, dest)?; } "FreeEnvironmentStringsW" => { - let [env_block] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [env_block] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let result = this.FreeEnvironmentStringsW(env_block)?; this.write_scalar(result, dest)?; } "GetCurrentDirectoryW" => { - let [size, buf] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [size, buf] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32, *mut _) -> u32), + link_name, + abi, + args, + )?; let result = this.GetCurrentDirectoryW(size, buf)?; this.write_scalar(result, dest)?; } "SetCurrentDirectoryW" => { - let [path] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [path] = this.check_shim_sig( + shim_sig!(extern "system" fn(*const _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let result = this.SetCurrentDirectoryW(path)?; this.write_scalar(result, dest)?; } "GetUserProfileDirectoryW" => { - let [token, buf, size] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [token, buf, size] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, *mut _, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let result = this.GetUserProfileDirectoryW(token, buf, size)?; this.write_scalar(result, dest)?; } "GetCurrentProcessId" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> u32), + link_name, + abi, + args, + )?; let result = this.GetCurrentProcessId()?; this.write_scalar(result, dest)?; } @@ -212,7 +240,24 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { n, byte_offset, key, - ] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + ] = this.check_shim_sig( + shim_sig!( + extern "system" fn( + winapi::HANDLE, + winapi::HANDLE, + *mut _, + *mut _, + *mut _, + *mut _, + u32, + *mut _, + *mut _, + ) -> i32 + ), + link_name, + abi, + args, + )?; this.NtWriteFile( handle, event, @@ -237,7 +282,24 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { n, byte_offset, key, - ] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + ] = this.check_shim_sig( + shim_sig!( + extern "system" fn( + winapi::HANDLE, + winapi::HANDLE, + *mut _, + *mut _, + *mut _, + *mut _, + u32, + *mut _, + *mut _, + ) -> i32 + ), + link_name, + abi, + args, + )?; this.NtReadFile( handle, event, @@ -252,8 +314,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { )?; } "GetFullPathNameW" => { - let [filename, size, buffer, filepart] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [filename, size, buffer, filepart] = this.check_shim_sig( + shim_sig!(extern "system" fn(*const _, u32, *mut _, *mut _) -> u32), + link_name, + abi, + args, + )?; this.check_no_isolation("`GetFullPathNameW`")?; let filename = this.read_pointer(filename)?; @@ -290,7 +356,22 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { creation_disposition, flags_and_attributes, template_file, - ] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + ] = this.check_shim_sig( + shim_sig!( + extern "system" fn( + *const _, + u32, + u32, + *mut _, + u32, + u32, + winapi::HANDLE, + ) -> winapi::HANDLE + ), + link_name, + abi, + args, + )?; let handle = this.CreateFileW( file_name, desired_access, @@ -303,29 +384,60 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(handle.to_scalar(this), dest)?; } "GetFileInformationByHandle" => { - let [handle, info] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, info] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let res = this.GetFileInformationByHandle(handle, info)?; this.write_scalar(res, dest)?; } "SetFileInformationByHandle" => { - let [handle, class, info, size] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, class, info, size] = this.check_shim_sig( + shim_sig!( + extern "system" fn( + winapi::HANDLE, + winapi::FILE_INFO_BY_HANDLE_CLASS, + *mut _, + u32, + ) -> winapi::BOOL + ), + link_name, + abi, + args, + )?; let res = this.SetFileInformationByHandle(handle, class, info, size)?; this.write_scalar(res, dest)?; } "FlushFileBuffers" => { - let [handle] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE) -> winapi::BOOL), + link_name, + abi, + args, + )?; let res = this.FlushFileBuffers(handle)?; this.write_scalar(res, dest)?; } "DeleteFileW" => { - let [file_name] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [file_name] = this.check_shim_sig( + shim_sig!(extern "system" fn(*const _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let res = this.DeleteFileW(file_name)?; this.write_scalar(res, dest)?; } "SetFilePointerEx" => { - let [file, distance_to_move, new_file_pointer, move_method] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [file, distance_to_move, new_file_pointer, move_method] = this.check_shim_sig( + // i64 is actually a LARGE_INTEGER union of {u32, i32} and {i64} + shim_sig!(extern "system" fn(winapi::HANDLE, i64, *mut _, u32) -> winapi::BOOL), + link_name, + abi, + args, + )?; let res = this.SetFilePointerEx(file, distance_to_move, new_file_pointer, move_method)?; this.write_scalar(res, dest)?; @@ -333,8 +445,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Allocation "HeapAlloc" => { - let [handle, flags, size] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, flags, size] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, u32, usize) -> *mut _), + link_name, + abi, + args, + )?; this.read_target_isize(handle)?; let flags = this.read_scalar(flags)?.to_u32()?; let size = this.read_target_usize(size)?; @@ -356,8 +472,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_pointer(ptr, dest)?; } "HeapFree" => { - let [handle, flags, ptr] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, flags, ptr] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, u32, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; this.read_target_isize(handle)?; this.read_scalar(flags)?.to_u32()?; let ptr = this.read_pointer(ptr)?; @@ -369,8 +489,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Scalar::from_i32(1), dest)?; } "HeapReAlloc" => { - let [handle, flags, old_ptr, size] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, flags, old_ptr, size] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, u32, *mut _, usize) -> *mut _), + link_name, + abi, + args, + )?; this.read_target_isize(handle)?; this.read_scalar(flags)?.to_u32()?; let old_ptr = this.read_pointer(old_ptr)?; @@ -390,7 +514,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_pointer(new_ptr, dest)?; } "LocalFree" => { - let [ptr] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HLOCAL) -> winapi::HLOCAL), + link_name, + abi, + args, + )?; let ptr = this.read_pointer(ptr)?; // "If the hMem parameter is NULL, LocalFree ignores the parameter and returns NULL." // (https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-localfree) @@ -402,17 +531,32 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // errno "SetLastError" => { - let [error] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [error] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32) -> ()), + link_name, + abi, + args, + )?; let error = this.read_scalar(error)?; this.set_last_error(error)?; } "GetLastError" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> u32), + link_name, + abi, + args, + )?; let last_error = this.get_last_error()?; this.write_scalar(last_error, dest)?; } "RtlNtStatusToDosError" => { - let [status] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [status] = this.check_shim_sig( + shim_sig!(extern "system" fn(i32) -> u32), + link_name, + abi, + args, + )?; let status = this.read_scalar(status)?.to_u32()?; let err = match status { // STATUS_MEDIA_WRITE_PROTECTED => ERROR_WRITE_PROTECT @@ -434,7 +578,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Querying system information "GetSystemInfo" => { // Also called from `page_size` crate. - let [system_info] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [system_info] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> ()), + link_name, + abi, + args, + )?; let system_info = this.deref_pointer_as(system_info, this.windows_ty_layout("SYSTEM_INFO"))?; // Initialize with `0`. @@ -457,19 +606,34 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // This just creates a key; Windows does not natively support TLS destructors. // Create key and return it. - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> u32), + link_name, + abi, + args, + )?; let key = this.machine.tls.create_tls_key(None, dest.layout.size)?; this.write_scalar(Scalar::from_uint(key, dest.layout.size), dest)?; } "TlsGetValue" => { - let [key] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [key] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32) -> *mut _), + link_name, + abi, + args, + )?; let key = u128::from(this.read_scalar(key)?.to_u32()?); let active_thread = this.active_thread(); let ptr = this.machine.tls.load_tls(key, active_thread, this)?; this.write_scalar(ptr, dest)?; } "TlsSetValue" => { - let [key, new_ptr] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [key, new_ptr] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let key = u128::from(this.read_scalar(key)?.to_u32()?); let active_thread = this.active_thread(); let new_data = this.read_scalar(new_ptr)?; @@ -479,7 +643,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_int(1, dest)?; } "TlsFree" => { - let [key] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [key] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32) -> winapi::BOOL), + link_name, + abi, + args, + )?; let key = u128::from(this.read_scalar(key)?.to_u32()?); this.machine.tls.delete_tls_key(key)?; @@ -489,7 +658,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Access to command-line arguments "GetCommandLineW" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> *mut _), + link_name, + abi, + args, + )?; this.write_pointer( this.machine.cmd_line.expect("machine must be initialized"), dest, @@ -498,31 +672,51 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Time related shims "GetSystemTimeAsFileTime" | "GetSystemTimePreciseAsFileTime" => { - #[allow(non_snake_case)] - let [LPFILETIME] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; - this.GetSystemTimeAsFileTime(link_name.as_str(), LPFILETIME)?; + let [filetime] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> ()), + link_name, + abi, + args, + )?; + this.GetSystemTimeAsFileTime(link_name.as_str(), filetime)?; } "QueryPerformanceCounter" => { - #[allow(non_snake_case)] - let [lpPerformanceCount] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; - let result = this.QueryPerformanceCounter(lpPerformanceCount)?; + let [performance_count] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; + let result = this.QueryPerformanceCounter(performance_count)?; this.write_scalar(result, dest)?; } "QueryPerformanceFrequency" => { - #[allow(non_snake_case)] - let [lpFrequency] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; - let result = this.QueryPerformanceFrequency(lpFrequency)?; + let [frequency] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; + let result = this.QueryPerformanceFrequency(frequency)?; this.write_scalar(result, dest)?; } "Sleep" => { - let [timeout] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [timeout] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32) -> ()), + link_name, + abi, + args, + )?; this.Sleep(timeout)?; } "CreateWaitableTimerExW" => { - let [attributes, name, flags, access] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [attributes, name, flags, access] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _, *const _, u32, u32) -> winapi::HANDLE), + link_name, + abi, + args, + )?; this.read_pointer(attributes)?; this.read_pointer(name)?; this.read_scalar(flags)?.to_u32()?; @@ -535,40 +729,66 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Synchronization primitives "InitOnceBeginInitialize" => { - let [ptr, flags, pending, context] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr, flags, pending, context] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _, u32, *mut _, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; this.InitOnceBeginInitialize(ptr, flags, pending, context, dest)?; } "InitOnceComplete" => { - let [ptr, flags, context] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr, flags, context] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _, u32, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; let result = this.InitOnceComplete(ptr, flags, context)?; this.write_scalar(result, dest)?; } "WaitOnAddress" => { - let [ptr_op, compare_op, size_op, timeout_op] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr_op, compare_op, size_op, timeout_op] = this.check_shim_sig( + // First pointer is volatile + shim_sig!(extern "system" fn(*mut _, *mut _, usize, u32) -> winapi::BOOL), + link_name, + abi, + args, + )?; this.WaitOnAddress(ptr_op, compare_op, size_op, timeout_op, dest)?; } "WakeByAddressSingle" => { - let [ptr_op] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr_op] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> ()), + link_name, + abi, + args, + )?; this.WakeByAddressSingle(ptr_op)?; } "WakeByAddressAll" => { - let [ptr_op] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr_op] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> ()), + link_name, + abi, + args, + )?; this.WakeByAddressAll(ptr_op)?; } // Dynamic symbol loading "GetProcAddress" => { - #[allow(non_snake_case)] - let [hModule, lpProcName] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; - this.read_target_isize(hModule)?; - let name = this.read_c_str(this.read_pointer(lpProcName)?)?; + let [module, proc_name] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HMODULE, *const _) -> winapi::FARPROC), + link_name, + abi, + args, + )?; + this.read_target_isize(module)?; + let name = this.read_c_str(this.read_pointer(proc_name)?)?; if let Ok(name) = str::from_utf8(name) && is_dyn_sym(name) { @@ -581,8 +801,21 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Threading "CreateThread" => { - let [security, stacksize, start, arg, flags, thread] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [security, stacksize, start, arg, flags, thread] = this.check_shim_sig( + shim_sig!( + extern "system" fn( + *mut _, + usize, + *mut _, + *mut _, + u32, + *mut _, + ) -> winapi::HANDLE + ), + link_name, + abi, + args, + )?; let thread_id = this.CreateThread(security, stacksize, start, arg, flags, thread)?; @@ -590,13 +823,22 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Handle::Thread(thread_id).to_scalar(this), dest)?; } "WaitForSingleObject" => { - let [handle, timeout] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, timeout] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, u32) -> u32), + link_name, + abi, + args, + )?; this.WaitForSingleObject(handle, timeout, dest)?; } "GetCurrentProcess" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> winapi::HANDLE), + link_name, + abi, + args, + )?; this.write_scalar( Handle::Pseudo(PseudoHandle::CurrentProcess).to_scalar(this), @@ -604,7 +846,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { )?; } "GetCurrentThread" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> winapi::HANDLE), + link_name, + abi, + args, + )?; this.write_scalar( Handle::Pseudo(PseudoHandle::CurrentThread).to_scalar(this), @@ -612,7 +859,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { )?; } "SetThreadDescription" => { - let [handle, name] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, name] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, *const _) -> i32), + link_name, + abi, + args, + )?; let handle = this.read_handle(handle, "SetThreadDescription")?; let name = this.read_wide_str(this.read_pointer(name)?)?; @@ -627,8 +879,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Scalar::from_u32(0), dest)?; } "GetThreadDescription" => { - let [handle, name_ptr] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, name_ptr] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, *mut _) -> i32), + link_name, + abi, + args, + )?; let handle = this.read_handle(handle, "GetThreadDescription")?; let name_ptr = this.deref_pointer_as(name_ptr, this.machine.layouts.mut_raw_ptr)?; // the pointer where we should store the ptr to the name @@ -651,7 +907,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(res, dest)?; } "GetThreadId" => { - let [handle] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE) -> u32), + link_name, + abi, + args, + )?; let handle = this.read_handle(handle, "GetThreadId")?; let thread = match handle { Handle::Thread(thread) => thread, @@ -662,7 +923,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Scalar::from_u32(tid), dest)?; } "GetCurrentThreadId" => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> u32), + link_name, + abi, + args, + )?; let thread = this.active_thread(); let tid = this.get_tid(thread); this.write_scalar(Scalar::from_u32(tid), dest)?; @@ -670,7 +936,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Miscellaneous "ExitProcess" => { - let [code] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [code] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32) -> ()), + link_name, + abi, + args, + )?; // Windows technically uses u32, but we unify everything to a Unix-style i32. let code = this.read_scalar(code)?.to_i32()?; throw_machine_stop!(TerminationInfo::Exit { code, leak_check: false }); @@ -678,7 +949,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { "SystemFunction036" => { // used by getrandom 0.1 // This is really 'RtlGenRandom'. - let [ptr, len] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr, len] = this.check_shim_sig( + // Returns winapi::BOOLEAN, which is a byte + shim_sig!(extern "system" fn(*mut _, u32) -> u8), + link_name, + abi, + args, + )?; let ptr = this.read_pointer(ptr)?; let len = this.read_scalar(len)?.to_u32()?; this.gen_random(ptr, len.into())?; @@ -686,7 +963,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "ProcessPrng" => { // used by `std` - let [ptr, len] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [ptr, len] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _, usize) -> winapi::BOOL), + link_name, + abi, + args, + )?; let ptr = this.read_pointer(ptr)?; let len = this.read_target_usize(len)?; this.gen_random(ptr, len)?; @@ -694,8 +976,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "BCryptGenRandom" => { // used by getrandom 0.2 - let [algorithm, ptr, len, flags] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [algorithm, ptr, len, flags] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _, *mut _, u32, u32) -> i32), + link_name, + abi, + args, + )?; let algorithm = this.read_scalar(algorithm)?; let algorithm = algorithm.to_target_usize(this)?; let ptr = this.read_pointer(ptr)?; @@ -729,8 +1015,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "GetConsoleScreenBufferInfo" => { // `term` needs this, so we fake it. - let [console, buffer_info] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [console, buffer_info] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; this.read_target_isize(console)?; // FIXME: this should use deref_pointer_as, but CONSOLE_SCREEN_BUFFER_INFO is not in std this.deref_pointer(buffer_info)?; @@ -739,13 +1029,33 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_null(dest)?; } "GetStdHandle" => { - let [which] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [which] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32) -> winapi::HANDLE), + link_name, + abi, + args, + )?; let res = this.GetStdHandle(which)?; this.write_scalar(res, dest)?; } "DuplicateHandle" => { let [src_proc, src_handle, target_proc, target_handle, access, inherit, options] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + this.check_shim_sig( + shim_sig!( + extern "system" fn( + winapi::HANDLE, + winapi::HANDLE, + winapi::HANDLE, + *mut _, + u32, + winapi::BOOL, + u32, + ) -> winapi::BOOL + ), + link_name, + abi, + args, + )?; let res = this.DuplicateHandle( src_proc, src_handle, @@ -758,15 +1068,24 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(res, dest)?; } "CloseHandle" => { - let [handle] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE) -> winapi::BOOL), + link_name, + abi, + args, + )?; let ret = this.CloseHandle(handle)?; this.write_scalar(ret, dest)?; } "GetModuleFileNameW" => { - let [handle, filename, size] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [handle, filename, size] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HMODULE, *mut _, u32) -> u32), + link_name, + abi, + args, + )?; this.check_no_isolation("`GetModuleFileNameW`")?; let handle = this.read_handle(handle, "GetModuleFileNameW")?; @@ -799,8 +1118,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } } "FormatMessageW" => { - let [flags, module, message_id, language_id, buffer, size, arguments] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [flags, module, message_id, language_id, buffer, size, arguments] = this + .check_shim_sig( + shim_sig!( + extern "system" fn(u32, *const _, u32, u32, *mut _, u32, *mut _) -> u32 + ), + link_name, + abi, + args, + )?; let flags = this.read_scalar(flags)?.to_u32()?; let _module = this.read_pointer(module)?; // seems to contain a module name @@ -843,7 +1169,12 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ); } // This function looks and behaves excatly like miri_start_unwind. - let [payload] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?; + let [payload] = this.check_shim_sig( + shim_sig!(extern "C" fn(*mut _) -> unwind::libunwind::_Unwind_Reason_Code), + link_name, + abi, + args, + )?; this.handle_miri_start_unwind(payload)?; return interp_ok(EmulateItemResult::NeedsUnwind); } @@ -851,56 +1182,86 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Incomplete shims that we "stub out" just to get pre-main initialization code to work. // These shims are enabled only when the caller is in the standard library. "GetProcessHeap" if this.frame_in_std() => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> winapi::HANDLE), + link_name, + abi, + args, + )?; // Just fake a HANDLE // It's fine to not use the Handle type here because its a stub this.write_int(1, dest)?; } "GetModuleHandleA" if this.frame_in_std() => { - #[allow(non_snake_case)] - let [_lpModuleName] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [_module_name] = this.check_shim_sig( + shim_sig!(extern "system" fn(*const _) -> winapi::HMODULE), + link_name, + abi, + args, + )?; // We need to return something non-null here to make `compat_fn!` work. this.write_int(1, dest)?; } "SetConsoleTextAttribute" if this.frame_in_std() => { - #[allow(non_snake_case)] - let [_hConsoleOutput, _wAttribute] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [_console_output, _attribute] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, u16) -> winapi::BOOL), + link_name, + abi, + args, + )?; // Pretend these does not exist / nothing happened, by returning zero. this.write_null(dest)?; } "GetConsoleMode" if this.frame_in_std() => { - let [console, mode] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [console, mode] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE, *mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; this.read_target_isize(console)?; this.deref_pointer_as(mode, this.machine.layouts.u32)?; // Indicate an error. this.write_null(dest)?; } "GetFileType" if this.frame_in_std() => { - #[allow(non_snake_case)] - let [_hFile] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [_file] = this.check_shim_sig( + shim_sig!(extern "system" fn(winapi::HANDLE) -> u32), + link_name, + abi, + args, + )?; // Return unknown file type. this.write_null(dest)?; } "AddVectoredExceptionHandler" if this.frame_in_std() => { - #[allow(non_snake_case)] - let [_First, _Handler] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [_first, _handler] = this.check_shim_sig( + shim_sig!(extern "system" fn(u32, *mut _) -> *mut _), + link_name, + abi, + args, + )?; // Any non zero value works for the stdlib. This is just used for stack overflows anyway. this.write_int(1, dest)?; } "SetThreadStackGuarantee" if this.frame_in_std() => { - #[allow(non_snake_case)] - let [_StackSizeInBytes] = - this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [_stack_size_in_bytes] = this.check_shim_sig( + shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL), + link_name, + abi, + args, + )?; // Any non zero value works for the stdlib. This is just used for stack overflows anyway. this.write_int(1, dest)?; } // this is only callable from std because we know that std ignores the return value "SwitchToThread" if this.frame_in_std() => { - let [] = this.check_shim_sig_lenient(abi, sys_conv, link_name, args)?; + let [] = this.check_shim_sig( + shim_sig!(extern "system" fn() -> winapi::BOOL), + link_name, + abi, + args, + )?; this.yield_active_thread(); diff --git a/src/tools/miri/tests/pass-dep/getrandom.rs b/src/tools/miri/tests/pass-dep/getrandom.rs index d359730e7f97..9adb48fb1c3c 100644 --- a/src/tools/miri/tests/pass-dep/getrandom.rs +++ b/src/tools/miri/tests/pass-dep/getrandom.rs @@ -12,6 +12,8 @@ fn main() { #[cfg(not(target_os = "solaris"))] getrandom_01::getrandom(&mut data).unwrap(); + // On Windows, getrandom 0.2 uses the wrong return type for BCryptGenRandom + #[cfg(not(target_os = "windows"))] getrandom_02::getrandom(&mut data).unwrap(); getrandom_03::fill(&mut data).unwrap(); diff --git a/src/tools/miri/tests/pass/tls/windows-tls.rs b/src/tools/miri/tests/pass/tls/windows-tls.rs index 58131be19037..9c5e8eea4d3f 100644 --- a/src/tools/miri/tests/pass/tls/windows-tls.rs +++ b/src/tools/miri/tests/pass/tls/windows-tls.rs @@ -5,14 +5,14 @@ use std::ptr; extern "system" { fn TlsAlloc() -> u32; - fn TlsSetValue(key: u32, val: *mut c_void) -> bool; + fn TlsSetValue(key: u32, val: *mut c_void) -> i32; fn TlsGetValue(key: u32) -> *mut c_void; - fn TlsFree(key: u32) -> bool; + fn TlsFree(key: u32) -> i32; } fn main() { let key = unsafe { TlsAlloc() }; - assert!(unsafe { TlsSetValue(key, ptr::without_provenance_mut(1)) }); + assert!(unsafe { TlsSetValue(key, ptr::without_provenance_mut(1)) != 0 }); assert_eq!(unsafe { TlsGetValue(key).addr() }, 1); - assert!(unsafe { TlsFree(key) }); + assert!(unsafe { TlsFree(key) != 0 }); } From 00693a8b2523ff78fd8d0f62bad17ccaa5f7e422 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 5 Jan 2026 15:10:44 +0100 Subject: [PATCH 0086/1843] tweak shim_sig --- src/tools/miri/src/shims/sig.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tools/miri/src/shims/sig.rs b/src/tools/miri/src/shims/sig.rs index a49689d11fd7..e88cc66b6ea5 100644 --- a/src/tools/miri/src/shims/sig.rs +++ b/src/tools/miri/src/shims/sig.rs @@ -20,8 +20,12 @@ pub struct ShimSig<'tcx, const ARGS: usize> { /// shim_sig!(extern "C" fn (*const T, i32) -> usize) /// ``` /// -/// In type position, `winapi::` can be used as a shorthand for the full path used by -/// `windows_ty_layout`. +/// The following types are supported: +/// - primitive integer types +/// - `()` +/// - (thin) raw pointers, written `*const _` and `*mut _` since the pointee type is irrelevant +/// - `$crate::$mod::...::$ty` for a type from the given crate (most commonly that is `libc`) +/// - `winapi::$ty` for a type from `std::sys::pal::windows::c` #[macro_export] macro_rules! shim_sig { (extern $abi:literal fn($($arg:ty),* $(,)?) -> $ret:ty) => { @@ -55,10 +59,11 @@ macro_rules! shim_sig_arg { "()" => $this.tcx.types.unit, "*const _" => $this.machine.layouts.const_raw_ptr.ty, "*mut _" => $this.machine.layouts.mut_raw_ptr.ty, - ty if let Some(libc_ty) = ty.strip_prefix("libc::") => $this.libc_ty_layout(libc_ty).ty, ty if let Some(win_ty) = ty.strip_prefix("winapi::") => $this.windows_ty_layout(win_ty).ty, - ty => helpers::path_ty_layout($this, &ty.split("::").collect::>()).ty, + ty if ty.contains("::") => + helpers::path_ty_layout($this, &ty.split("::").collect::>()).ty, + ty => panic!("unsupported signature type {ty:?}"), } }}; } From 4699fdc2887bd9a28921ca9e73193aa38f94d0c4 Mon Sep 17 00:00:00 2001 From: A4-Tacks Date: Tue, 6 Jan 2026 16:53:34 +0800 Subject: [PATCH 0087/1843] Fix loses exists guard for move_guard Example --- ```rust fn main() { let cond = true; match 92 { 3 => true, x if cond => if x $0> 10 { false } else if x > 5 { true } else if x > 4 || x < -2 { false } else { true }, } } ``` **Before this PR** ```rust fn main() { let cond = true; match 92 { 3 => true, x if x > 10 => false, x if x > 5 => true, x if x > 4 || x < -2 => false, x => true, } } ``` **After this PR** ```rust fn main() { let cond = true; match 92 { 3 => true, x if cond && x > 10 => false, x if cond && x > 5 => true, x if cond && (x > 4 || x < -2) => false, x if cond => true, } } ``` --- .../ide-assists/src/handlers/move_guard.rs | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs index 1c0c6e43d53b..31baa63372ff 100644 --- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs +++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs @@ -3,7 +3,7 @@ use syntax::{ SyntaxKind::WHITESPACE, ast::{ AstNode, BlockExpr, ElseBranch, Expr, IfExpr, MatchArm, Pat, edit::AstNodeEdit, make, - syntax_factory::SyntaxFactory, + prec::ExprPrecedence, syntax_factory::SyntaxFactory, }, syntax_editor::Element, }; @@ -109,6 +109,7 @@ pub(crate) fn move_arm_cond_to_match_guard( let match_arm: MatchArm = ctx.find_node_at_offset::()?; let match_pat = match_arm.pat()?; let arm_body = match_arm.expr()?; + let arm_guard = match_arm.guard().and_then(|it| it.condition()); let mut replace_node = None; let if_expr: IfExpr = IfExpr::cast(arm_body.syntax().clone()).or_else(|| { @@ -149,6 +150,25 @@ pub(crate) fn move_arm_cond_to_match_guard( 0 }; let indent_level = match_arm.indent_level(); + let make_guard = |cond: Option| { + let condition = match (arm_guard.clone(), cond) { + (None, None) => return None, + (None, Some(it)) | (Some(it), None) => it, + (Some(lhs), Some(rhs)) => { + let op_expr = |expr: Expr| { + if expr.precedence().needs_parentheses_in(ExprPrecedence::LAnd) { + make.expr_paren(expr).into() + } else { + expr + } + }; + let op = syntax::ast::BinaryOp::LogicOp(syntax::ast::LogicOp::And); + let expr_bin = make.expr_bin(op_expr(lhs), op, op_expr(rhs)); + expr_bin.into() + } + }; + Some(make.match_guard(condition)) + }; for (cond, block) in conds_blocks { let only_expr = block.statements().next().is_none(); @@ -156,8 +176,7 @@ pub(crate) fn move_arm_cond_to_match_guard( Some(then_expr) if only_expr => then_expr, _ => block.dedent(dedent.into()).into(), }; - let guard = make.match_guard(cond); - let new_arm = make.match_arm(match_pat.clone(), Some(guard), expr); + let new_arm = make.match_arm(match_pat.clone(), make_guard(Some(cond)), expr); replace_arms.push(new_arm); } if let Some(block) = tail { @@ -170,7 +189,7 @@ pub(crate) fn move_arm_cond_to_match_guard( } _ => block.dedent(dedent.into()).into(), }; - let new_arm = make.match_arm(match_pat, None, expr); + let new_arm = make.match_arm(match_pat, make_guard(None), expr); replace_arms.push(new_arm); } else { // There's no else branch. Add a pattern without guard, unless the following match @@ -185,7 +204,7 @@ pub(crate) fn move_arm_cond_to_match_guard( } _ => { let block_expr = make.expr_empty_block().into(); - replace_arms.push(make.match_arm(match_pat, None, block_expr)); + replace_arms.push(make.match_arm(match_pat, make_guard(None), block_expr)); } } } @@ -1081,6 +1100,42 @@ fn main() { x => {} } } +"#, + ) + } + + #[test] + fn move_arm_cond_to_match_guard_elseif_exist_guard() { + check_assist( + move_arm_cond_to_match_guard, + r#" +fn main() { + let cond = true; + match 92 { + 3 => true, + x if cond => if x $0> 10 { + false + } else if x > 5 { + true + } else if x > 4 || x < -2 { + false + } else { + true + }, + } +} +"#, + r#" +fn main() { + let cond = true; + match 92 { + 3 => true, + x if cond && x > 10 => false, + x if cond && x > 5 => true, + x if cond && (x > 4 || x < -2) => false, + x if cond => true, + } +} "#, ) } From ce012da28e4d69cc24a4ecb6afb262564183f2bc Mon Sep 17 00:00:00 2001 From: andjsrk Date: Tue, 6 Jan 2026 20:14:52 +0900 Subject: [PATCH 0088/1843] accept test changes related to binary ops --- tests/ui/manual_clamp.fixed | 3 +- tests/ui/manual_clamp.rs | 3 +- tests/ui/manual_clamp.stderr | 70 +++++++++++++-------------- tests/ui/needless_bitwise_bool.fixed | 4 +- tests/ui/needless_bitwise_bool.rs | 2 + tests/ui/needless_bitwise_bool.stderr | 8 ++- 6 files changed, 51 insertions(+), 39 deletions(-) diff --git a/tests/ui/manual_clamp.fixed b/tests/ui/manual_clamp.fixed index 2450a4f4c611..b279a413bc17 100644 --- a/tests/ui/manual_clamp.fixed +++ b/tests/ui/manual_clamp.fixed @@ -4,7 +4,8 @@ dead_code, clippy::unnecessary_operation, clippy::no_effect, - clippy::if_same_then_else + clippy::if_same_then_else, + clippy::needless_match )] use std::cmp::{max as cmp_max, min as cmp_min}; diff --git a/tests/ui/manual_clamp.rs b/tests/ui/manual_clamp.rs index ee341d50768f..7fda41cd4c35 100644 --- a/tests/ui/manual_clamp.rs +++ b/tests/ui/manual_clamp.rs @@ -4,7 +4,8 @@ dead_code, clippy::unnecessary_operation, clippy::no_effect, - clippy::if_same_then_else + clippy::if_same_then_else, + clippy::needless_match )] use std::cmp::{max as cmp_max, min as cmp_min}; diff --git a/tests/ui/manual_clamp.stderr b/tests/ui/manual_clamp.stderr index 4a0e4fa51646..775a16418eba 100644 --- a/tests/ui/manual_clamp.stderr +++ b/tests/ui/manual_clamp.stderr @@ -1,5 +1,5 @@ error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:212:5 + --> tests/ui/manual_clamp.rs:213:5 | LL | / if x9 < CONST_MIN { LL | | @@ -15,7 +15,7 @@ LL | | } = help: to override `-D warnings` add `#[allow(clippy::manual_clamp)]` error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:230:5 + --> tests/ui/manual_clamp.rs:231:5 | LL | / if x11 > CONST_MAX { LL | | @@ -29,7 +29,7 @@ LL | | } = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:240:5 + --> tests/ui/manual_clamp.rs:241:5 | LL | / if CONST_MIN > x12 { LL | | @@ -43,7 +43,7 @@ LL | | } = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:250:5 + --> tests/ui/manual_clamp.rs:251:5 | LL | / if CONST_MAX < x13 { LL | | @@ -57,7 +57,7 @@ LL | | } = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:363:5 + --> tests/ui/manual_clamp.rs:364:5 | LL | / if CONST_MAX < x35 { LL | | @@ -71,7 +71,7 @@ LL | | } = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:144:14 + --> tests/ui/manual_clamp.rs:145:14 | LL | let x0 = if CONST_MAX < input { | ______________^ @@ -86,7 +86,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:154:14 + --> tests/ui/manual_clamp.rs:155:14 | LL | let x1 = if input > CONST_MAX { | ______________^ @@ -101,7 +101,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:164:14 + --> tests/ui/manual_clamp.rs:165:14 | LL | let x2 = if input < CONST_MIN { | ______________^ @@ -116,7 +116,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:174:14 + --> tests/ui/manual_clamp.rs:175:14 | LL | let x3 = if CONST_MIN > input { | ______________^ @@ -131,7 +131,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:184:14 + --> tests/ui/manual_clamp.rs:185:14 | LL | let x4 = input.max(CONST_MIN).min(CONST_MAX); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -139,7 +139,7 @@ LL | let x4 = input.max(CONST_MIN).min(CONST_MAX); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:187:14 + --> tests/ui/manual_clamp.rs:188:14 | LL | let x5 = input.min(CONST_MAX).max(CONST_MIN); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -147,7 +147,7 @@ LL | let x5 = input.min(CONST_MAX).max(CONST_MIN); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:190:14 + --> tests/ui/manual_clamp.rs:191:14 | LL | let x6 = match input { | ______________^ @@ -161,7 +161,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:197:14 + --> tests/ui/manual_clamp.rs:198:14 | LL | let x7 = match input { | ______________^ @@ -175,7 +175,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:204:14 + --> tests/ui/manual_clamp.rs:205:14 | LL | let x8 = match input { | ______________^ @@ -189,7 +189,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:221:15 + --> tests/ui/manual_clamp.rs:222:15 | LL | let x10 = match input { | _______________^ @@ -203,7 +203,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:259:15 + --> tests/ui/manual_clamp.rs:260:15 | LL | let x14 = if input > CONST_MAX { | _______________^ @@ -218,7 +218,7 @@ LL | | }; = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:270:19 + --> tests/ui/manual_clamp.rs:271:19 | LL | let x15 = if input > CONST_F64_MAX { | ___________________^ @@ -234,7 +234,7 @@ LL | | }; = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:283:19 + --> tests/ui/manual_clamp.rs:284:19 | LL | let x16 = cmp_max(cmp_min(input, CONST_MAX), CONST_MIN); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -242,7 +242,7 @@ LL | let x16 = cmp_max(cmp_min(input, CONST_MAX), CONST_MIN); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:286:19 + --> tests/ui/manual_clamp.rs:287:19 | LL | let x17 = cmp_min(cmp_max(input, CONST_MIN), CONST_MAX); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -250,7 +250,7 @@ LL | let x17 = cmp_min(cmp_max(input, CONST_MIN), CONST_MAX); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:289:19 + --> tests/ui/manual_clamp.rs:290:19 | LL | let x18 = cmp_max(CONST_MIN, cmp_min(input, CONST_MAX)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -258,7 +258,7 @@ LL | let x18 = cmp_max(CONST_MIN, cmp_min(input, CONST_MAX)); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:292:19 + --> tests/ui/manual_clamp.rs:293:19 | LL | let x19 = cmp_min(CONST_MAX, cmp_max(input, CONST_MIN)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -266,7 +266,7 @@ LL | let x19 = cmp_min(CONST_MAX, cmp_max(input, CONST_MIN)); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:295:19 + --> tests/ui/manual_clamp.rs:296:19 | LL | let x20 = cmp_max(cmp_min(CONST_MAX, input), CONST_MIN); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -274,7 +274,7 @@ LL | let x20 = cmp_max(cmp_min(CONST_MAX, input), CONST_MIN); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:298:19 + --> tests/ui/manual_clamp.rs:299:19 | LL | let x21 = cmp_min(cmp_max(CONST_MIN, input), CONST_MAX); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -282,7 +282,7 @@ LL | let x21 = cmp_min(cmp_max(CONST_MIN, input), CONST_MAX); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:301:19 + --> tests/ui/manual_clamp.rs:302:19 | LL | let x22 = cmp_max(CONST_MIN, cmp_min(CONST_MAX, input)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -290,7 +290,7 @@ LL | let x22 = cmp_max(CONST_MIN, cmp_min(CONST_MAX, input)); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:304:19 + --> tests/ui/manual_clamp.rs:305:19 | LL | let x23 = cmp_min(CONST_MAX, cmp_max(CONST_MIN, input)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_MIN, CONST_MAX)` @@ -298,7 +298,7 @@ LL | let x23 = cmp_min(CONST_MAX, cmp_max(CONST_MIN, input)); = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:308:19 + --> tests/ui/manual_clamp.rs:309:19 | LL | let x24 = f64::max(f64::min(input, CONST_F64_MAX), CONST_F64_MIN); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -307,7 +307,7 @@ LL | let x24 = f64::max(f64::min(input, CONST_F64_MAX), CONST_F64_MIN); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:311:19 + --> tests/ui/manual_clamp.rs:312:19 | LL | let x25 = f64::min(f64::max(input, CONST_F64_MIN), CONST_F64_MAX); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -316,7 +316,7 @@ LL | let x25 = f64::min(f64::max(input, CONST_F64_MIN), CONST_F64_MAX); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:314:19 + --> tests/ui/manual_clamp.rs:315:19 | LL | let x26 = f64::max(CONST_F64_MIN, f64::min(input, CONST_F64_MAX)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -325,7 +325,7 @@ LL | let x26 = f64::max(CONST_F64_MIN, f64::min(input, CONST_F64_MAX)); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:317:19 + --> tests/ui/manual_clamp.rs:318:19 | LL | let x27 = f64::min(CONST_F64_MAX, f64::max(input, CONST_F64_MIN)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -334,7 +334,7 @@ LL | let x27 = f64::min(CONST_F64_MAX, f64::max(input, CONST_F64_MIN)); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:320:19 + --> tests/ui/manual_clamp.rs:321:19 | LL | let x28 = f64::max(f64::min(CONST_F64_MAX, input), CONST_F64_MIN); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -343,7 +343,7 @@ LL | let x28 = f64::max(f64::min(CONST_F64_MAX, input), CONST_F64_MIN); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:323:19 + --> tests/ui/manual_clamp.rs:324:19 | LL | let x29 = f64::min(f64::max(CONST_F64_MIN, input), CONST_F64_MAX); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -352,7 +352,7 @@ LL | let x29 = f64::min(f64::max(CONST_F64_MIN, input), CONST_F64_MAX); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:326:19 + --> tests/ui/manual_clamp.rs:327:19 | LL | let x30 = f64::max(CONST_F64_MIN, f64::min(CONST_F64_MAX, input)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -361,7 +361,7 @@ LL | let x30 = f64::max(CONST_F64_MIN, f64::min(CONST_F64_MAX, input)); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:329:19 + --> tests/ui/manual_clamp.rs:330:19 | LL | let x31 = f64::min(CONST_F64_MAX, f64::max(CONST_F64_MIN, input)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with clamp: `input.clamp(CONST_F64_MIN, CONST_F64_MAX)` @@ -370,7 +370,7 @@ LL | let x31 = f64::min(CONST_F64_MAX, f64::max(CONST_F64_MIN, input)); = note: clamp returns NaN if the input is NaN error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:333:5 + --> tests/ui/manual_clamp.rs:334:5 | LL | / if x32 < CONST_MIN { LL | | @@ -384,7 +384,7 @@ LL | | } = note: clamp will panic if max < min error: clamp-like pattern without using clamp function - --> tests/ui/manual_clamp.rs:525:13 + --> tests/ui/manual_clamp.rs:526:13 | LL | let _ = if input > CONST_MAX { | _____________^ diff --git a/tests/ui/needless_bitwise_bool.fixed b/tests/ui/needless_bitwise_bool.fixed index 89a3c1474f25..751d3d257000 100644 --- a/tests/ui/needless_bitwise_bool.fixed +++ b/tests/ui/needless_bitwise_bool.fixed @@ -34,7 +34,9 @@ fn main() { println!("true") // This is a const method call } - if y & (0 < 1) { + // Resolved + if y && (0 < 1) { + //~^ needless_bitwise_bool println!("true") // This is a BinOp with no side effects } } diff --git a/tests/ui/needless_bitwise_bool.rs b/tests/ui/needless_bitwise_bool.rs index f5aa7a9f3d9e..5d3ff3b2079c 100644 --- a/tests/ui/needless_bitwise_bool.rs +++ b/tests/ui/needless_bitwise_bool.rs @@ -34,7 +34,9 @@ fn main() { println!("true") // This is a const method call } + // Resolved if y & (0 < 1) { + //~^ needless_bitwise_bool println!("true") // This is a BinOp with no side effects } } diff --git a/tests/ui/needless_bitwise_bool.stderr b/tests/ui/needless_bitwise_bool.stderr index 9f14646c3e5a..4f64c7136916 100644 --- a/tests/ui/needless_bitwise_bool.stderr +++ b/tests/ui/needless_bitwise_bool.stderr @@ -7,5 +7,11 @@ LL | if y & !x { = note: `-D clippy::needless-bitwise-bool` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::needless_bitwise_bool)]` -error: aborting due to 1 previous error +error: use of bitwise operator instead of lazy operator between booleans + --> tests/ui/needless_bitwise_bool.rs:38:8 + | +LL | if y & (0 < 1) { + | ^^^^^^^^^^^ help: try: `y && (0 < 1)` + +error: aborting due to 2 previous errors From a978fdcdace3c9bee4d11e03ee3ea2be5792025c Mon Sep 17 00:00:00 2001 From: Hendrik Lind Date: Tue, 6 Jan 2026 20:49:14 +0100 Subject: [PATCH 0089/1843] fix: use crates where ADT was defined in deref_chain of trait_applicable_items --- .../ide-completion/src/tests/flyimport.rs | 48 +++++++++++++++++++ .../ide-db/src/imports/import_assets.rs | 14 +++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/tests/flyimport.rs b/src/tools/rust-analyzer/crates/ide-completion/src/tests/flyimport.rs index 797df3f163da..d7db896679df 100644 --- a/src/tools/rust-analyzer/crates/ide-completion/src/tests/flyimport.rs +++ b/src/tools/rust-analyzer/crates/ide-completion/src/tests/flyimport.rs @@ -1976,3 +1976,51 @@ fn main() { "#]], ); } + +#[test] +fn trait_method_import_across_multiple_crates() { + let fixture = r#" + //- /lib.rs crate:test-trait + pub trait TestTrait { + fn test_function(&self) -> u32; + } + + //- /lib.rs crate:test-implementation deps:test-trait + pub struct TestStruct(pub usize); + + impl test_trait::TestTrait for TestStruct { + fn test_function(&self) -> u32 { + 1 + } + } + + //- /main.rs crate:main deps:test-implementation,test-trait + use test_implementation::TestStruct; + + fn main() { + let test = TestStruct(42); + test.test_f$0 + } + "#; + + check( + fixture, + expect![[r#" + me test_function() (use test_trait::TestTrait) fn(&self) -> u32 + "#]], + ); + + check_edit( + "test_function", + fixture, + r#" +use test_implementation::TestStruct; +use test_trait::TestTrait; + +fn main() { + let test = TestStruct(42); + test.test_function()$0 +} +"#, + ); +} diff --git a/src/tools/rust-analyzer/crates/ide-db/src/imports/import_assets.rs b/src/tools/rust-analyzer/crates/ide-db/src/imports/import_assets.rs index 90e3bb61f44d..35579eb2590d 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/imports/import_assets.rs +++ b/src/tools/rust-analyzer/crates/ide-db/src/imports/import_assets.rs @@ -600,7 +600,19 @@ fn trait_applicable_items<'db>( } deref_chain .into_iter() - .filter_map(|ty| Some((ty.krate(db).into(), ty.fingerprint_for_trait_impl()?))) + .flat_map(|ty| { + let fingerprint = ty.fingerprint_for_trait_impl()?; + let mut crates = vec![]; + + if let Some(adt) = ty.as_adt() { + // Push crate where ADT was defined + crates.push((adt.krate(db).into(), fingerprint)); + } + // Always include environment crate + crates.push((ty.krate(db).into(), fingerprint)); + Some(crates) + }) + .flatten() .unique() .collect::>() }; From 6b11237e7dca81a57340ac4ab5ca2437b2c59a93 Mon Sep 17 00:00:00 2001 From: mu001999 Date: Tue, 6 Jan 2026 20:23:31 +0800 Subject: [PATCH 0090/1843] Add span field for ConstArg --- clippy_lints/src/large_stack_arrays.rs | 2 +- clippy_lints/src/utils/author.rs | 1 + clippy_utils/src/consts.rs | 2 +- clippy_utils/src/hir_utils.rs | 16 ++++++++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/clippy_lints/src/large_stack_arrays.rs b/clippy_lints/src/large_stack_arrays.rs index 620e27fa67c6..261b03abba17 100644 --- a/clippy_lints/src/large_stack_arrays.rs +++ b/clippy_lints/src/large_stack_arrays.rs @@ -126,7 +126,7 @@ fn might_be_expanded<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'tcx>) -> bool { let ExprKind::Repeat(_, len_ct) = expr.kind else { return false; }; - !expr.span.contains(len_ct.span()) + !expr.span.contains(len_ct.span) } expr.span.from_expansion() || is_from_proc_macro(cx, expr) || repeat_expr_might_be_expanded(expr) diff --git a/clippy_lints/src/utils/author.rs b/clippy_lints/src/utils/author.rs index f515f9987a80..455f76edc904 100644 --- a/clippy_lints/src/utils/author.rs +++ b/clippy_lints/src/utils/author.rs @@ -321,6 +321,7 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { }, ConstArgKind::Struct(..) => chain!(self, "let ConstArgKind::Struct(..) = {const_arg}.kind"), ConstArgKind::TupleCall(..) => chain!(self, "let ConstArgKind::TupleCall(..) = {const_arg}.kind"), + ConstArgKind::Tup(..) => chain!(self, "let ConstArgKind::Tup(..) = {const_arg}.kind"), ConstArgKind::Infer(..) => chain!(self, "let ConstArgKind::Infer(..) = {const_arg}.kind"), ConstArgKind::Error(..) => chain!(self, "let ConstArgKind::Error(..) = {const_arg}.kind"), } diff --git a/clippy_utils/src/consts.rs b/clippy_utils/src/consts.rs index a44cd31dc123..46b87fd5df96 100644 --- a/clippy_utils/src/consts.rs +++ b/clippy_utils/src/consts.rs @@ -1140,7 +1140,7 @@ pub fn const_item_rhs_to_expr<'tcx>(tcx: TyCtxt<'tcx>, ct_rhs: ConstItemRhs<'tcx ConstItemRhs::Body(body_id) => Some(tcx.hir_body(body_id).value), ConstItemRhs::TypeConst(const_arg) => match const_arg.kind { ConstArgKind::Anon(anon) => Some(tcx.hir_body(anon.body).value), - ConstArgKind::Struct(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) | ConstArgKind::Infer(..) => { + ConstArgKind::Struct(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Tup(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) | ConstArgKind::Infer(..) => { None }, }, diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs index f1ee534c500d..57c896c97172 100644 --- a/clippy_utils/src/hir_utils.rs +++ b/clippy_utils/src/hir_utils.rs @@ -661,6 +661,10 @@ impl HirEqInterExpr<'_, '_, '_> { } fn eq_const_arg(&mut self, left: &ConstArg<'_>, right: &ConstArg<'_>) -> bool { + if !self.check_ctxt(left.span.ctxt(), right.span.ctxt()) { + return false; + } + match (&left.kind, &right.kind) { (ConstArgKind::Path(l_p), ConstArgKind::Path(r_p)) => self.eq_qpath(l_p, r_p), (ConstArgKind::Anon(l_an), ConstArgKind::Anon(r_an)) => self.eq_body(l_an.body, r_an.body), @@ -679,11 +683,18 @@ impl HirEqInterExpr<'_, '_, '_> { .zip(*args_b) .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) } + (ConstArgKind::Tup(args_a), ConstArgKind::Tup(args_b)) => { + args_a + .iter() + .zip(*args_b) + .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) + } // Use explicit match for now since ConstArg is undergoing flux. ( ConstArgKind::Path(..) | ConstArgKind::Anon(..) | ConstArgKind::TupleCall(..) + | ConstArgKind::Tup(..) | ConstArgKind::Infer(..) | ConstArgKind::Struct(..) | ConstArgKind::Error(..), @@ -1560,6 +1571,11 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> { self.hash_const_arg(arg); } }, + ConstArgKind::Tup(args) => { + for arg in *args { + self.hash_const_arg(arg); + } + }, ConstArgKind::Infer(..) | ConstArgKind::Error(..) => {}, } } From 6cc9e5ccd3728842f23502dc9d1f9de08056a340 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 7 Jan 2026 08:15:27 +0100 Subject: [PATCH 0091/1843] Document `Query` --- .../crates/ide-db/src/symbol_index.rs | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs index eb0529d6b5e7..06e1f6bb4560 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs +++ b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs @@ -40,15 +40,61 @@ use salsa::Update; use crate::RootDatabase; +/// A query for searching symbols in the workspace or dependencies. +/// +/// This struct configures how symbol search is performed, including the search text, +/// matching strategy, and filtering options. It is used by [`world_symbols`] to find +/// symbols across the codebase. +/// +/// # Example +/// ```ignore +/// let mut query = Query::new("MyStruct".to_string()); +/// query.only_types(); // Only search for type definitions +/// query.libs(); // Include library dependencies +/// query.exact(); // Use exact matching instead of fuzzy +/// ``` #[derive(Debug, Clone)] pub struct Query { + /// The original search query string as provided by the user. + /// Used for the final matching check via [`SearchMode::check`]. query: String, + /// Lowercase version of [`Self::query`], pre-computed for efficiency. + /// Used to build FST automata for case-insensitive index lookups. lowercased: String, + /// The search strategy to use when matching symbols. + /// - [`SearchMode::Exact`]: Symbol name must exactly match the query. + /// - [`SearchMode::Fuzzy`]: Symbol name must contain all query characters in order (subsequence match). + /// - [`SearchMode::Prefix`]: Symbol name must start with the query string. + /// + /// Defaults to [`SearchMode::Fuzzy`]. mode: SearchMode, + /// Controls filtering of trait-associated items (methods, constants, types). + /// - [`AssocSearchMode::Include`]: Include both associated and non-associated items. + /// - [`AssocSearchMode::Exclude`]: Exclude trait-associated items from results. + /// - [`AssocSearchMode::AssocItemsOnly`]: Only return trait-associated items. + /// + /// Defaults to [`AssocSearchMode::Include`]. assoc_mode: AssocSearchMode, + /// Whether the final symbol name comparison should be case-sensitive. + /// When `false`, matching is case-insensitive (e.g., "foo" matches "Foo"). + /// + /// Defaults to `false`. case_sensitive: bool, + /// When `true`, only return type definitions: structs, enums, unions, + /// type aliases, built-in types, and traits. Functions, constants, statics, + /// and modules are excluded. + /// + /// Defaults to `false`. only_types: bool, + /// When `true`, search library dependency roots instead of local workspace crates. + /// This enables finding symbols in external dependencies including the standard library. + /// + /// Defaults to `false` (search local workspace only). libs: bool, + /// When `true`, exclude re-exported/imported symbols from results, + /// showing only the original definitions. + /// + /// Defaults to `false`. exclude_imports: bool, } From d37d04a37c6a9cd62e4c9518ffd47352a2a2ad1e Mon Sep 17 00:00:00 2001 From: Samuel Onoja Date: Sun, 4 Jan 2026 04:51:44 +0100 Subject: [PATCH 0092/1843] recognize safety comments inside blocks and on same line in macros allow only whitespace between the comment marker and `SAFETY:` --- .../src/undocumented_unsafe_blocks.rs | 35 ++++ .../undocumented_unsafe_blocks.default.stderr | 156 +++++++++------- ...undocumented_unsafe_blocks.disabled.stderr | 168 ++++++++++-------- .../undocumented_unsafe_blocks.rs | 67 +++++++ 4 files changed, 288 insertions(+), 138 deletions(-) diff --git a/clippy_lints/src/undocumented_unsafe_blocks.rs b/clippy_lints/src/undocumented_unsafe_blocks.rs index 9d27a66a9ab8..1b26b1b32b80 100644 --- a/clippy_lints/src/undocumented_unsafe_blocks.rs +++ b/clippy_lints/src/undocumented_unsafe_blocks.rs @@ -115,6 +115,7 @@ impl<'tcx> LateLintPass<'tcx> for UndocumentedUnsafeBlocks { && !is_lint_allowed(cx, UNDOCUMENTED_UNSAFE_BLOCKS, block.hir_id) && !is_unsafe_from_proc_macro(cx, block.span) && !block_has_safety_comment(cx, block.span, self.accept_comment_above_attributes) + && !block_has_inner_safety_comment(cx, block.span) && !block_parents_have_safety_comment( self.accept_comment_above_statement, self.accept_comment_above_attributes, @@ -839,6 +840,23 @@ fn text_has_safety_comment( } } } + // Check for a comment that appears after other code on the same line (e.g., `let x = // SAFETY:`) + // This handles cases in macros where the comment is on the same line as preceding code. + // We only check the first (immediate preceding) line for this pattern. + // Only whitespace is allowed between the comment marker and `SAFETY:`. + if let Some(comment_start) = [line.find("//"), line.find("/*")].into_iter().flatten().min() + && let after_marker = &line[comment_start + 2..] // skip marker + && let trimmed = after_marker.trim_start() // skip whitespace + && trimmed.get(..7).is_some_and(|s| s.eq_ignore_ascii_case("SAFETY:")) + { + let safety_offset = 2 + (after_marker.len() - trimmed.len()); + return HasSafetyComment::Yes( + start_pos + + BytePos(u32::try_from(line_start).unwrap()) + + BytePos(u32::try_from(comment_start + safety_offset).unwrap()), + false, + ); + } // No line comments; look for the start of a block comment. // This will only find them if they are at the start of a line. let (mut line_start, mut line) = (line_start, line); @@ -894,3 +912,20 @@ fn is_const_or_static(node: &Node<'_>) -> bool { }) ) } + +fn block_has_inner_safety_comment(cx: &LateContext<'_>, span: Span) -> bool { + let source_map = cx.sess().source_map(); + if let Ok(src) = source_map.span_to_snippet(span) + && let Some(after_brace) = src + .strip_prefix("unsafe") + .and_then(|s| s.trim_start().strip_prefix('{')) + && let Some(comment) = after_brace + .trim_start() + .strip_prefix("//") + .or_else(|| after_brace.trim_start().strip_prefix("/*")) + { + comment.trim_start().to_ascii_uppercase().starts_with("SAFETY:") + } else { + false + } +} diff --git a/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.default.stderr b/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.default.stderr index 61e5af81d827..bcc46adda8a0 100644 --- a/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.default.stderr +++ b/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.default.stderr @@ -1,15 +1,39 @@ error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:270:19 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:244:22 + | +LL | let _x = unsafe { 1 }; + | ^^^^^^^^^^^^ +... +LL | t!(); + | ---- in this macro invocation + | + = help: consider adding a safety comment on the preceding line + = note: `-D clippy::undocumented-unsafe-blocks` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::undocumented_unsafe_blocks)]` + = note: this error originates in the macro `t` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: unsafe block missing a safety comment + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:256:13 + | +LL | unsafe { 1 }; + | ^^^^^^^^^^^^ +... +LL | t!(); + | ---- in this macro invocation + | + = help: consider adding a safety comment on the preceding line + = note: this error originates in the macro `t` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: unsafe block missing a safety comment + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:337:19 | LL | /* Safety: */ unsafe {} | ^^^^^^^^^ | = help: consider adding a safety comment on the preceding line - = note: `-D clippy::undocumented-unsafe-blocks` implied by `-D warnings` - = help: to override `-D warnings` add `#[allow(clippy::undocumented_unsafe_blocks)]` error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:275:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:342:5 | LL | unsafe {} | ^^^^^^^^^ @@ -17,7 +41,7 @@ LL | unsafe {} = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:280:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:347:14 | LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; | ^^^^^^^^^^^^^ @@ -25,7 +49,7 @@ LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:280:29 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:347:29 | LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; | ^^^^^^^^^^^^^ @@ -33,7 +57,7 @@ LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:280:48 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:347:48 | LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; | ^^^^^^^^^^^^^ @@ -41,7 +65,7 @@ LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:287:18 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:354:18 | LL | let _ = (42, unsafe {}, "test", unsafe {}); | ^^^^^^^^^ @@ -49,7 +73,7 @@ LL | let _ = (42, unsafe {}, "test", unsafe {}); = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:287:37 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:354:37 | LL | let _ = (42, unsafe {}, "test", unsafe {}); | ^^^^^^^^^ @@ -57,7 +81,7 @@ LL | let _ = (42, unsafe {}, "test", unsafe {}); = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:293:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:360:14 | LL | let _ = *unsafe { &42 }; | ^^^^^^^^^^^^^^ @@ -65,7 +89,7 @@ LL | let _ = *unsafe { &42 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:299:19 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:366:19 | LL | let _ = match unsafe {} { | ^^^^^^^^^ @@ -73,7 +97,7 @@ LL | let _ = match unsafe {} { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:306:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:373:14 | LL | let _ = &unsafe {}; | ^^^^^^^^^ @@ -81,7 +105,7 @@ LL | let _ = &unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:311:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:378:14 | LL | let _ = [unsafe {}; 5]; | ^^^^^^^^^ @@ -89,7 +113,7 @@ LL | let _ = [unsafe {}; 5]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:316:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:383:13 | LL | let _ = unsafe {}; | ^^^^^^^^^ @@ -97,7 +121,7 @@ LL | let _ = unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:327:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:394:8 | LL | t!(unsafe {}); | ^^^^^^^^^ @@ -105,7 +129,7 @@ LL | t!(unsafe {}); = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:334:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:401:13 | LL | unsafe {} | ^^^^^^^^^ @@ -117,7 +141,7 @@ LL | t!(); = note: this error originates in the macro `t` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:343:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:410:5 | LL | unsafe {} // SAFETY: | ^^^^^^^^^ @@ -125,7 +149,7 @@ LL | unsafe {} // SAFETY: = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:349:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:416:5 | LL | unsafe { | ^^^^^^^^ @@ -133,7 +157,7 @@ LL | unsafe { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:360:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:427:5 | LL | unsafe {}; | ^^^^^^^^^ @@ -141,7 +165,7 @@ LL | unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:365:20 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:432:20 | LL | println!("{}", unsafe { String::from_utf8_unchecked(vec![]) }); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -149,7 +173,7 @@ LL | println!("{}", unsafe { String::from_utf8_unchecked(vec![]) }); = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:373:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:440:5 | LL | unsafe impl A for () {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -157,7 +181,7 @@ LL | unsafe impl A for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:381:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:448:9 | LL | unsafe impl B for (u32) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -165,7 +189,7 @@ LL | unsafe impl B for (u32) {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:403:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:470:13 | LL | unsafe impl T for $t {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -177,7 +201,7 @@ LL | no_safety_comment!(()); = note: this error originates in the macro `no_safety_comment` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:429:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:496:13 | LL | unsafe impl T for $t {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -189,7 +213,7 @@ LL | no_safety_comment!(()); = note: this error originates in the macro `no_safety_comment` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:439:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:506:5 | LL | unsafe impl T for (i32) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -197,7 +221,7 @@ LL | unsafe impl T for (i32) {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:429:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:496:13 | LL | unsafe impl T for $t {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -209,7 +233,7 @@ LL | no_safety_comment!(u32); = note: this error originates in the macro `no_safety_comment` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:446:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:513:5 | LL | unsafe impl T for (bool) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -217,7 +241,7 @@ LL | unsafe impl T for (bool) {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:493:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:560:5 | LL | unsafe impl NoComment for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -225,7 +249,7 @@ LL | unsafe impl NoComment for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:498:19 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:565:19 | LL | /* SAFETY: */ unsafe impl InlineComment for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -233,7 +257,7 @@ LL | /* SAFETY: */ unsafe impl InlineComment for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:503:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:570:5 | LL | unsafe impl TrailingComment for () {} // SAFETY: | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -241,13 +265,13 @@ LL | unsafe impl TrailingComment for () {} // SAFETY: = help: consider adding a safety comment on the preceding line error: constant has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:508:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:575:5 | LL | const BIG_NUMBER: i32 = 1000000; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:507:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:574:8 | LL | // SAFETY: | ^^^^^^^ @@ -255,7 +279,7 @@ LL | // SAFETY: = help: to override `-D warnings` add `#[allow(clippy::unnecessary_safety_comment)]` error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:510:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:577:5 | LL | unsafe impl Interference for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -263,7 +287,7 @@ LL | unsafe impl Interference for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:518:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:585:5 | LL | unsafe impl ImplInFn for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -271,7 +295,7 @@ LL | unsafe impl ImplInFn for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:528:1 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:595:1 | LL | unsafe impl CrateRoot for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -279,7 +303,7 @@ LL | unsafe impl CrateRoot for () {} = help: consider adding a safety comment on the preceding line error: statement has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:543:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:610:5 | LL | / let _ = { LL | | @@ -289,13 +313,13 @@ LL | | }; | |______^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:542:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:609:8 | LL | // SAFETY: this is more than one level away, so it should warn | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:545:12 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:612:12 | LL | if unsafe { true } { | ^^^^^^^^^^^^^^^ @@ -303,7 +327,7 @@ LL | if unsafe { true } { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:549:23 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:616:23 | LL | let bar = unsafe {}; | ^^^^^^^^^ @@ -311,7 +335,7 @@ LL | let bar = unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:638:52 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:705:52 | LL | const NO_SAFETY_IN_TRAIT_BUT_IN_IMPL: u8 = unsafe { 0 }; | ^^^^^^^^^^^^ @@ -319,7 +343,7 @@ LL | const NO_SAFETY_IN_TRAIT_BUT_IN_IMPL: u8 = unsafe { 0 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:647:41 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:714:41 | LL | const NO_SAFETY_IN_TRAIT: i32 = unsafe { 1 }; | ^^^^^^^^^^^^ @@ -327,7 +351,7 @@ LL | const NO_SAFETY_IN_TRAIT: i32 = unsafe { 1 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:657:42 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:724:42 | LL | const HAS_SAFETY_IN_TRAIT: i32 = unsafe { 3 }; | ^^^^^^^^^^^^ @@ -335,7 +359,7 @@ LL | const HAS_SAFETY_IN_TRAIT: i32 = unsafe { 3 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:662:40 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:729:40 | LL | const NO_SAFETY_IN_IMPL: i32 = unsafe { 1 }; | ^^^^^^^^^^^^ @@ -343,136 +367,136 @@ LL | const NO_SAFETY_IN_IMPL: i32 = unsafe { 1 }; = help: consider adding a safety comment on the preceding line error: constant has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:701:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:768:5 | LL | const UNIX_EPOCH_JULIAN_DAY: i32 = | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:699:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:766:8 | LL | // SAFETY: fail ONLY if `accept-comment-above-attribute = false` | ^^^^^^^ error: statement has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:721:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:788:5 | LL | _ = bar(); | ^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:720:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:787:8 | LL | // SAFETY: unnecessary_safety_comment triggers here | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error: module has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:741:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:808:5 | LL | mod x {} | ^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:740:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:807:8 | LL | // SAFETY: ... | ^^^^^^^ error: module has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:746:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:813:5 | LL | mod y {} | ^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:744:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:811:8 | LL | // SAFETY: ... | ^^^^^^^ error: module has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:751:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:818:5 | LL | mod z {} | ^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:750:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:817:8 | LL | // SAFETY: ... | ^^^^^^^ error: module has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:759:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:826:5 | LL | mod y {} | ^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:757:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:824:8 | LL | // SAFETY: ... | ^^^^^^^ error: statement has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:774:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:841:9 | LL | let x = 34; | ^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:772:12 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:839:12 | LL | // SAFETY: ... | ^^^^^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:781:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:848:5 | LL | unsafe fn unsafe_comment() {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider changing the `safety` comment for a `# Safety` doc comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:780:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:847:8 | LL | // SAFETY: Bla | ^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:787:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:854:5 | LL | unsafe fn unsafe_block_comment() {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider changing the `safety` comment for a `# Safety` doc comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:785:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:852:8 | LL | SAFETY: Bla | ^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:791:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:858:5 | LL | fn safe_comment() {} | ^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:790:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:857:8 | LL | // SAFETY: Bla | ^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:795:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:862:5 | LL | fn safe_doc_comment() {} | ^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:794:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:861:9 | LL | /// SAFETY: Bla | ^^^^^^^ -error: aborting due to 50 previous errors +error: aborting due to 52 previous errors diff --git a/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.disabled.stderr b/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.disabled.stderr index e252cffea916..0de8ed716bed 100644 --- a/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.disabled.stderr +++ b/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.disabled.stderr @@ -1,15 +1,39 @@ error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:270:19 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:244:22 + | +LL | let _x = unsafe { 1 }; + | ^^^^^^^^^^^^ +... +LL | t!(); + | ---- in this macro invocation + | + = help: consider adding a safety comment on the preceding line + = note: `-D clippy::undocumented-unsafe-blocks` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::undocumented_unsafe_blocks)]` + = note: this error originates in the macro `t` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: unsafe block missing a safety comment + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:256:13 + | +LL | unsafe { 1 }; + | ^^^^^^^^^^^^ +... +LL | t!(); + | ---- in this macro invocation + | + = help: consider adding a safety comment on the preceding line + = note: this error originates in the macro `t` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: unsafe block missing a safety comment + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:337:19 | LL | /* Safety: */ unsafe {} | ^^^^^^^^^ | = help: consider adding a safety comment on the preceding line - = note: `-D clippy::undocumented-unsafe-blocks` implied by `-D warnings` - = help: to override `-D warnings` add `#[allow(clippy::undocumented_unsafe_blocks)]` error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:275:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:342:5 | LL | unsafe {} | ^^^^^^^^^ @@ -17,7 +41,7 @@ LL | unsafe {} = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:280:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:347:14 | LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; | ^^^^^^^^^^^^^ @@ -25,7 +49,7 @@ LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:280:29 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:347:29 | LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; | ^^^^^^^^^^^^^ @@ -33,7 +57,7 @@ LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:280:48 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:347:48 | LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; | ^^^^^^^^^^^^^ @@ -41,7 +65,7 @@ LL | let _ = [unsafe { 14 }, unsafe { 15 }, 42, unsafe { 16 }]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:287:18 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:354:18 | LL | let _ = (42, unsafe {}, "test", unsafe {}); | ^^^^^^^^^ @@ -49,7 +73,7 @@ LL | let _ = (42, unsafe {}, "test", unsafe {}); = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:287:37 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:354:37 | LL | let _ = (42, unsafe {}, "test", unsafe {}); | ^^^^^^^^^ @@ -57,7 +81,7 @@ LL | let _ = (42, unsafe {}, "test", unsafe {}); = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:293:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:360:14 | LL | let _ = *unsafe { &42 }; | ^^^^^^^^^^^^^^ @@ -65,7 +89,7 @@ LL | let _ = *unsafe { &42 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:299:19 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:366:19 | LL | let _ = match unsafe {} { | ^^^^^^^^^ @@ -73,7 +97,7 @@ LL | let _ = match unsafe {} { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:306:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:373:14 | LL | let _ = &unsafe {}; | ^^^^^^^^^ @@ -81,7 +105,7 @@ LL | let _ = &unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:311:14 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:378:14 | LL | let _ = [unsafe {}; 5]; | ^^^^^^^^^ @@ -89,7 +113,7 @@ LL | let _ = [unsafe {}; 5]; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:316:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:383:13 | LL | let _ = unsafe {}; | ^^^^^^^^^ @@ -97,7 +121,7 @@ LL | let _ = unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:327:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:394:8 | LL | t!(unsafe {}); | ^^^^^^^^^ @@ -105,7 +129,7 @@ LL | t!(unsafe {}); = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:334:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:401:13 | LL | unsafe {} | ^^^^^^^^^ @@ -117,7 +141,7 @@ LL | t!(); = note: this error originates in the macro `t` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:343:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:410:5 | LL | unsafe {} // SAFETY: | ^^^^^^^^^ @@ -125,7 +149,7 @@ LL | unsafe {} // SAFETY: = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:349:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:416:5 | LL | unsafe { | ^^^^^^^^ @@ -133,7 +157,7 @@ LL | unsafe { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:360:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:427:5 | LL | unsafe {}; | ^^^^^^^^^ @@ -141,7 +165,7 @@ LL | unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:365:20 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:432:20 | LL | println!("{}", unsafe { String::from_utf8_unchecked(vec![]) }); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -149,7 +173,7 @@ LL | println!("{}", unsafe { String::from_utf8_unchecked(vec![]) }); = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:373:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:440:5 | LL | unsafe impl A for () {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -157,7 +181,7 @@ LL | unsafe impl A for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:381:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:448:9 | LL | unsafe impl B for (u32) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -165,7 +189,7 @@ LL | unsafe impl B for (u32) {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:403:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:470:13 | LL | unsafe impl T for $t {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -177,7 +201,7 @@ LL | no_safety_comment!(()); = note: this error originates in the macro `no_safety_comment` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:429:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:496:13 | LL | unsafe impl T for $t {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -189,7 +213,7 @@ LL | no_safety_comment!(()); = note: this error originates in the macro `no_safety_comment` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:439:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:506:5 | LL | unsafe impl T for (i32) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -197,7 +221,7 @@ LL | unsafe impl T for (i32) {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:429:13 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:496:13 | LL | unsafe impl T for $t {} | ^^^^^^^^^^^^^^^^^^^^^^^ @@ -209,7 +233,7 @@ LL | no_safety_comment!(u32); = note: this error originates in the macro `no_safety_comment` (in Nightly builds, run with -Z macro-backtrace for more info) error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:446:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:513:5 | LL | unsafe impl T for (bool) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -217,7 +241,7 @@ LL | unsafe impl T for (bool) {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:493:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:560:5 | LL | unsafe impl NoComment for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -225,7 +249,7 @@ LL | unsafe impl NoComment for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:498:19 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:565:19 | LL | /* SAFETY: */ unsafe impl InlineComment for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -233,7 +257,7 @@ LL | /* SAFETY: */ unsafe impl InlineComment for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:503:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:570:5 | LL | unsafe impl TrailingComment for () {} // SAFETY: | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -241,13 +265,13 @@ LL | unsafe impl TrailingComment for () {} // SAFETY: = help: consider adding a safety comment on the preceding line error: constant has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:508:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:575:5 | LL | const BIG_NUMBER: i32 = 1000000; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:507:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:574:8 | LL | // SAFETY: | ^^^^^^^ @@ -255,7 +279,7 @@ LL | // SAFETY: = help: to override `-D warnings` add `#[allow(clippy::unnecessary_safety_comment)]` error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:510:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:577:5 | LL | unsafe impl Interference for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -263,7 +287,7 @@ LL | unsafe impl Interference for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:518:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:585:5 | LL | unsafe impl ImplInFn for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -271,7 +295,7 @@ LL | unsafe impl ImplInFn for () {} = help: consider adding a safety comment on the preceding line error: unsafe impl missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:528:1 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:595:1 | LL | unsafe impl CrateRoot for () {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -279,7 +303,7 @@ LL | unsafe impl CrateRoot for () {} = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:539:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:606:9 | LL | unsafe {}; | ^^^^^^^^^ @@ -287,7 +311,7 @@ LL | unsafe {}; = help: consider adding a safety comment on the preceding line error: statement has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:543:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:610:5 | LL | / let _ = { LL | | @@ -297,13 +321,13 @@ LL | | }; | |______^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:542:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:609:8 | LL | // SAFETY: this is more than one level away, so it should warn | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:545:12 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:612:12 | LL | if unsafe { true } { | ^^^^^^^^^^^^^^^ @@ -311,7 +335,7 @@ LL | if unsafe { true } { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:549:23 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:616:23 | LL | let bar = unsafe {}; | ^^^^^^^^^ @@ -319,7 +343,7 @@ LL | let bar = unsafe {}; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:568:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:635:9 | LL | unsafe { a_function_with_a_very_long_name_to_break_the_line() }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -327,7 +351,7 @@ LL | unsafe { a_function_with_a_very_long_name_to_break_the_line() }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:573:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:640:9 | LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -335,7 +359,7 @@ LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:578:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:645:9 | LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -343,7 +367,7 @@ LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:585:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:652:5 | LL | unsafe {} | ^^^^^^^^^ @@ -351,7 +375,7 @@ LL | unsafe {} = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:590:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:657:5 | LL | unsafe { | ^^^^^^^^ @@ -359,7 +383,7 @@ LL | unsafe { = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:598:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:665:9 | LL | unsafe { a_function_with_a_very_long_name_to_break_the_line() }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -367,7 +391,7 @@ LL | unsafe { a_function_with_a_very_long_name_to_break_the_line() }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:604:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:671:9 | LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -375,7 +399,7 @@ LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:611:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:678:9 | LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -383,7 +407,7 @@ LL | unsafe { a_const_function_with_a_very_long_name_to_break_the_line() = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:617:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:684:5 | LL | unsafe {} | ^^^^^^^^^ @@ -391,7 +415,7 @@ LL | unsafe {} = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:638:52 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:705:52 | LL | const NO_SAFETY_IN_TRAIT_BUT_IN_IMPL: u8 = unsafe { 0 }; | ^^^^^^^^^^^^ @@ -399,7 +423,7 @@ LL | const NO_SAFETY_IN_TRAIT_BUT_IN_IMPL: u8 = unsafe { 0 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:647:41 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:714:41 | LL | const NO_SAFETY_IN_TRAIT: i32 = unsafe { 1 }; | ^^^^^^^^^^^^ @@ -407,7 +431,7 @@ LL | const NO_SAFETY_IN_TRAIT: i32 = unsafe { 1 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:657:42 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:724:42 | LL | const HAS_SAFETY_IN_TRAIT: i32 = unsafe { 3 }; | ^^^^^^^^^^^^ @@ -415,7 +439,7 @@ LL | const HAS_SAFETY_IN_TRAIT: i32 = unsafe { 3 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:662:40 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:729:40 | LL | const NO_SAFETY_IN_IMPL: i32 = unsafe { 1 }; | ^^^^^^^^^^^^ @@ -423,7 +447,7 @@ LL | const NO_SAFETY_IN_IMPL: i32 = unsafe { 1 }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:673:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:740:9 | LL | unsafe { here_is_another_variable_with_long_name }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -431,7 +455,7 @@ LL | unsafe { here_is_another_variable_with_long_name }; = help: consider adding a safety comment on the preceding line error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:702:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:769:9 | LL | unsafe { Date::__from_ordinal_date_unchecked(1970, 1) }.into_julian_day_just_make_this_line_longer(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -439,19 +463,19 @@ LL | unsafe { Date::__from_ordinal_date_unchecked(1970, 1) }.into_julian = help: consider adding a safety comment on the preceding line error: statement has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:721:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:788:5 | LL | _ = bar(); | ^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:720:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:787:8 | LL | // SAFETY: unnecessary_safety_comment triggers here | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:735:12 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:802:12 | LL | return unsafe { h() }; | ^^^^^^^^^^^^^^ @@ -459,31 +483,31 @@ LL | return unsafe { h() }; = help: consider adding a safety comment on the preceding line error: module has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:741:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:808:5 | LL | mod x {} | ^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:740:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:807:8 | LL | // SAFETY: ... | ^^^^^^^ error: module has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:751:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:818:5 | LL | mod z {} | ^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:750:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:817:8 | LL | // SAFETY: ... | ^^^^^^^ error: unsafe block missing a safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:766:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:833:9 | LL | unsafe {} | ^^^^^^^^^ @@ -491,52 +515,52 @@ LL | unsafe {} = help: consider adding a safety comment on the preceding line error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:781:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:848:5 | LL | unsafe fn unsafe_comment() {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider changing the `safety` comment for a `# Safety` doc comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:780:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:847:8 | LL | // SAFETY: Bla | ^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:787:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:854:5 | LL | unsafe fn unsafe_block_comment() {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider changing the `safety` comment for a `# Safety` doc comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:785:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:852:8 | LL | SAFETY: Bla | ^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:791:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:858:5 | LL | fn safe_comment() {} | ^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:790:8 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:857:8 | LL | // SAFETY: Bla | ^^^^^^^ error: function has unnecessary safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:795:5 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:862:5 | LL | fn safe_doc_comment() {} | ^^^^^^^^^^^^^^^^^^^^^^^^ | help: consider removing the safety comment - --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:794:9 + --> tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs:861:9 | LL | /// SAFETY: Bla | ^^^^^^^ -error: aborting due to 60 previous errors +error: aborting due to 62 previous errors diff --git a/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs b/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs index db9e81cf10a1..8032c388ccfe 100644 --- a/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs +++ b/tests/ui-toml/undocumented_unsafe_blocks/undocumented_unsafe_blocks.rs @@ -201,6 +201,66 @@ fn comment_macro_def() { t!(); } +fn comment_macro_def_with_let() { + macro_rules! t { + () => { + let _x = + // SAFETY: here be exactly one dragon + unsafe { 1 }; + }; + } + + t!(); +} + +#[rustfmt::skip] +fn comment_macro_def_with_let_same_line() { + macro_rules! t { + () => { + let _x =// SAFETY: same line comment + unsafe { 1 }; + }; + } + + t!(); +} + +fn inner_comment_macro_def_with_let() { + macro_rules! t { + () => { + let _x = unsafe { + // SAFETY: inside the block + 1 + }; + }; + } + + t!(); +} + +fn no_comment_macro_def_with_let() { + macro_rules! t { + () => { + let _x = unsafe { 1 }; + //~^ undocumented_unsafe_blocks + }; + } + + t!(); +} + +fn prefixed_safety_comment_macro_def_with_let() { + macro_rules! t { + () => { + let _x =// not a SAFETY: comment, should lint + unsafe { 1 }; + //~^ undocumented_unsafe_blocks + }; + } + + t!(); +} + fn non_ascii_comment() { // ॐ᧻໒ SaFeTy: ௵∰ unsafe {}; @@ -263,6 +323,13 @@ fn in_closure(x: *const u32) { let _ = || unsafe { *x }; } +fn inner_block_comment_block_style(x: *const u32) { + let _ = unsafe { + /* SAFETY: block comment inside */ + *x + }; +} + // Invalid comments #[rustfmt::skip] From 1af7813baa02ec2f05acbaae5238b37bc0ebe5d7 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 7 Jan 2026 09:12:00 +0100 Subject: [PATCH 0093/1843] Document `WithFixture` --- .../crates/test-fixture/src/lib.rs | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs index d81f27d7c3b1..ca68edd88c05 100644 --- a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs +++ b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs @@ -37,7 +37,110 @@ use triomphe::Arc; pub const WORKSPACE: base_db::SourceRootId = base_db::SourceRootId(0); +/// A trait for setting up test databases from fixture strings. +/// +/// Fixtures are strings containing Rust source code with optional metadata that describe +/// a project setup. This is the primary way to write tests for rust-analyzer without +/// having to depend on the entire sysroot. +/// +/// # Fixture Syntax +/// +/// ## Basic Structure +/// +/// A fixture without metadata is parsed into a single source file (`/main.rs`). +/// Metadata is added after a `//-` comment prefix. +/// +/// ```text +/// //- /main.rs +/// fn main() { +/// println!("Hello"); +/// } +/// ``` +/// +/// Note that the fixture syntax is optional and can be omitted if the test only requires +/// a simple single file. +/// +/// ## File Metadata +/// +/// Each file can have the following metadata after `//-`: +/// +/// - **Path** (required): Must start with `/`, e.g., `/main.rs`, `/lib.rs`, `/foo/bar.rs` +/// - **`crate:`**: Defines a new crate with this file as its root +/// - Optional version: `crate:foo@0.1.0,https://example.com/repo.git` +/// - **`deps:,`**: Dependencies (requires `crate:`) +/// - **`extern-prelude:,`**: Limits extern prelude to specified crates +/// - **`edition:`**: Rust edition (2015, 2018, 2021, 2024). Defaults to current. +/// - **`cfg:=,`**: Configuration options, e.g., `cfg:test,feature="foo"` +/// - **`env:=`**: Environment variables +/// - **`crate-attr:`**: Crate-level attributes, e.g., `crate-attr:no_std` +/// - **`new_source_root:local|library`**: Starts a new source root +/// - **`library`**: Marks crate as external library (not workspace member) +/// +/// ## Global Meta (must appear at the top, in order) +/// +/// - **`//- toolchain: nightly|stable`**: Sets the Rust toolchain (default: stable) +/// - **`//- target_data_layout: `**: LLVM data layout string +/// - **`//- target_arch: `**: Target architecture (default: x86_64) +/// - **`//- proc_macros: ,`**: Enables predefined test proc macros +/// - **`//- minicore: , `**: Includes subset of libcore +/// +/// ## Cursor Markers +/// +/// Use `$0` to mark cursor position(s) in the fixture: +/// - Single `$0`: marks a position (use with [`with_position`](Self::with_position)) +/// - Two `$0` markers: marks a range (use with [`with_range`](Self::with_range)) +/// - Escape as `\$0` if you need a literal `$0` +/// +/// # Examples +/// +/// ## Single file with cursor position +/// ```text +/// r#" +/// fn main() { +/// let x$0 = 42; +/// } +/// "# +/// ``` +/// +/// ## Multiple crates with dependencies +/// ```text +/// r#" +/// //- /main.rs crate:main deps:helper +/// use helper::greet; +/// fn main() { greet(); } +/// +/// //- /lib.rs crate:helper +/// pub fn greet() {} +/// "# +/// ``` +/// +/// ## Using minicore for lang items +/// ```text +/// r#" +/// //- minicore: option, result, iterator +/// //- /main.rs +/// fn foo() -> Option { Some(42) } +/// "# +/// ``` +/// +/// The available minicore flags are listed at the top of crates\test-utils\src\minicore.rs. +/// +/// ## Using test proc macros +/// ```text +/// r#" +/// //- proc_macros: identity, mirror +/// //- /main.rs crate:main deps:proc_macros +/// use proc_macros::identity; +/// +/// #[identity] +/// fn foo() {} +/// "# +/// ``` +/// +/// Available proc macros: `identity` (attr), `DeriveIdentity` (derive), `input_replace` (attr), +/// `mirror` (bang), `shorten` (bang) pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_single_file( #[rust_analyzer::rust_fixture] ra_fixture: &str, @@ -50,6 +153,7 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { (db, file) } + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_many_files( #[rust_analyzer::rust_fixture] ra_fixture: &str, @@ -66,6 +170,7 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { (db, files) } + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_files(#[rust_analyzer::rust_fixture] ra_fixture: &str) -> Self { let mut db = Self::default(); @@ -75,6 +180,7 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { db } + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_files_extra_proc_macros( #[rust_analyzer::rust_fixture] ra_fixture: &str, @@ -88,6 +194,7 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { db } + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_position(#[rust_analyzer::rust_fixture] ra_fixture: &str) -> (Self, FilePosition) { let (db, file_id, range_or_offset) = Self::with_range_or_offset(ra_fixture); @@ -95,6 +202,7 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { (db, FilePosition { file_id, offset }) } + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_range(#[rust_analyzer::rust_fixture] ra_fixture: &str) -> (Self, FileRange) { let (db, file_id, range_or_offset) = Self::with_range_or_offset(ra_fixture); @@ -102,6 +210,7 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static { (db, FileRange { file_id, range }) } + /// See the trait documentation for more information on fixtures. #[track_caller] fn with_range_or_offset( #[rust_analyzer::rust_fixture] ra_fixture: &str, From ad020937855df0af1dc7416be0b12d0b8c2a89a1 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 7 Jan 2026 08:46:47 +0100 Subject: [PATCH 0094/1843] feat: Allow rust paths in symbol search --- .../rust-analyzer/crates/hir/src/symbols.rs | 72 +- .../crates/ide-db/src/symbol_index.rs | 664 +++++++++++++++++- 2 files changed, 724 insertions(+), 12 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir/src/symbols.rs b/src/tools/rust-analyzer/crates/hir/src/symbols.rs index 073142670d2a..544c759ed3a7 100644 --- a/src/tools/rust-analyzer/crates/hir/src/symbols.rs +++ b/src/tools/rust-analyzer/crates/hir/src/symbols.rs @@ -9,6 +9,7 @@ use hir_def::{ ModuleDefId, ModuleId, TraitId, db::DefDatabase, item_scope::{ImportId, ImportOrExternCrate, ImportOrGlob}, + nameres::crate_def_map, per_ns::Item, src::{HasChildSource, HasSource}, visibility::{Visibility, VisibilityExplicitness}, @@ -20,9 +21,12 @@ use hir_ty::{ }; use intern::Symbol; use rustc_hash::FxHashMap; -use syntax::{AstNode, AstPtr, SyntaxNode, SyntaxNodePtr, ToSmolStr, ast::HasName}; +use syntax::{ + AstNode, AstPtr, SyntaxNode, SyntaxNodePtr, ToSmolStr, + ast::{HasModuleItem, HasName}, +}; -use crate::{HasCrate, Module, ModuleDef, Semantics}; +use crate::{Crate, HasCrate, Module, ModuleDef, Semantics}; /// The actual data that is stored in the index. It should be as compact as /// possible. @@ -57,6 +61,70 @@ impl DeclarationLocation { } } +impl<'db> FileSymbol<'db> { + /// Create a `FileSymbol` representing a crate's root module. + /// This is used for crate search queries like `::` or `::foo`. + pub fn for_crate_root(db: &'db dyn HirDatabase, krate: Crate) -> Option> { + let display_name = krate.display_name(db)?; + let crate_name = display_name.crate_name(); + let root_module = krate.root_module(db); + let def_map = crate_def_map(db, krate.into()); + let module_data = &def_map[root_module.into()]; + + // Get the definition source (the source file for crate roots) + let definition = module_data.origin.definition_source(db); + let hir_file_id = definition.file_id; + + // For a crate root, the "declaration" is the source file itself + // We use the entire file's syntax node as the location + let syntax_node = definition.value.node(); + let ptr = SyntaxNodePtr::new(&syntax_node); + + // For the name, we need to create a synthetic name pointer. + // We'll use the first token of the file as a placeholder since crate roots + // don't have an explicit name in the source. + // We create a name_ptr pointing to the start of the file. + let name_ptr = match &definition.value { + crate::ModuleSource::SourceFile(sf) => { + // Try to find the first item with a name as a reasonable location for focus + // This is a bit of a hack but works for navigation purposes + let first_item: Option = sf.items().next(); + if let Some(item) = first_item { + if let Some(name) = item.syntax().children().find_map(syntax::ast::Name::cast) { + AstPtr::new(&name).wrap_left() + } else { + // No name found, try to use a NameRef instead + if let Some(name_ref) = + item.syntax().descendants().find_map(syntax::ast::NameRef::cast) + { + AstPtr::new(&name_ref).wrap_right() + } else { + return None; + } + } + } else { + return None; + } + } + _ => return None, + }; + + let loc = DeclarationLocation { hir_file_id, ptr, name_ptr }; + + Some(FileSymbol { + name: Symbol::intern(crate_name.as_str()), + def: ModuleDef::Module(root_module), + loc, + container_name: None, + is_alias: false, + is_assoc: false, + is_import: false, + do_not_complete: Complete::Yes, + _marker: PhantomData, + }) + } +} + /// Represents an outstanding module that the symbol collector must collect symbols from. #[derive(Debug)] struct SymbolCollectorWork { diff --git a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs index 06e1f6bb4560..ca0d5ec1e5e6 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs +++ b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs @@ -55,12 +55,17 @@ use crate::RootDatabase; /// ``` #[derive(Debug, Clone)] pub struct Query { - /// The original search query string as provided by the user. - /// Used for the final matching check via [`SearchMode::check`]. + /// The item name to search for (last segment of the path, or full query if no path). + /// When empty with a non-empty `path_filter`, returns all items in that module. query: String, /// Lowercase version of [`Self::query`], pre-computed for efficiency. /// Used to build FST automata for case-insensitive index lookups. lowercased: String, + /// Path segments to filter by (all segments except the last). + /// Empty if no `::` in the original query. + path_filter: Vec, + /// If true, the first path segment must be a crate name (query started with `::`). + anchor_to_crate: bool, /// The search strategy to use when matching symbols. /// - [`SearchMode::Exact`]: Symbol name must exactly match the query. /// - [`SearchMode::Fuzzy`]: Symbol name must contain all query characters in order (subsequence match). @@ -100,10 +105,13 @@ pub struct Query { impl Query { pub fn new(query: String) -> Query { - let lowercased = query.to_lowercase(); + let (path_filter, item_query, anchor_to_crate) = Self::parse_path_query(&query); + let lowercased = item_query.to_lowercase(); Query { - query, + query: item_query, lowercased, + path_filter, + anchor_to_crate, only_types: false, libs: false, mode: SearchMode::Fuzzy, @@ -113,6 +121,74 @@ impl Query { } } + /// Parse a query string that may contain path segments. + /// + /// Returns (path_filter, item_query, anchor_to_crate) where: + /// - `path_filter`: Path segments to match (all but the last segment) + /// - `item_query`: The item name to search for (last segment) + /// - `anchor_to_crate`: Whether the first segment must be a crate name + fn parse_path_query(query: &str) -> (Vec, String, bool) { + // Check for leading :: (absolute path / crate search) + let anchor_to_crate = query.starts_with("::"); + let query = if anchor_to_crate { &query[2..] } else { query }; + + // Handle sole "::" - return all crates + if query.is_empty() && anchor_to_crate { + return (vec![], String::new(), true); + } + + // Check for trailing :: (module browsing - returns all items in module) + let return_all_in_module = query.ends_with("::"); + let query = if return_all_in_module { query.trim_end_matches("::") } else { query }; + + if !query.contains("::") { + // No path separator - single segment + if anchor_to_crate && !return_all_in_module { + // "::foo" - fuzzy search crate names only + return (vec![], query.to_string(), true); + } + if return_all_in_module { + // "foo::" - browse all items in module "foo" + // path_filter = ["foo"], query = "", anchor_to_crate = false/true + return (vec![query.to_string()], String::new(), anchor_to_crate); + } + // Plain "foo" - normal fuzzy search + return (vec![], query.to_string(), false); + } + + // Filter out empty segments (e.g., "foo::::bar" -> "foo::bar") + let segments: Vec<&str> = query.split("::").filter(|s| !s.is_empty()).collect(); + + if segments.is_empty() { + return (vec![], String::new(), anchor_to_crate); + } + + let path: Vec = + segments[..segments.len() - 1].iter().map(|s| s.to_string()).collect(); + let item = if return_all_in_module { + // All segments go to path, item is empty + let mut path = path; + path.push(segments.last().unwrap().to_string()); + return (path, String::new(), anchor_to_crate); + } else { + segments.last().unwrap_or(&"").to_string() + }; + + (path, item, anchor_to_crate) + } + + /// Returns true if this query should return all items in a module + /// (i.e., the original query ended with `::`) + fn is_module_browsing(&self) -> bool { + self.query.is_empty() && !self.path_filter.is_empty() + } + + /// Returns true if this query is searching for crates + /// (i.e., the query was "::" alone or "::foo" for fuzzy crate search) + fn is_crate_search(&self) -> bool { + self.anchor_to_crate && self.path_filter.is_empty() + } + pub fn only_types(&mut self) { self.only_types = true; } @@ -181,7 +257,28 @@ pub fn crate_symbols(db: &dyn HirDatabase, krate: Crate) -> Box<[&SymbolIndex<'_ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { let _p = tracing::info_span!("world_symbols", query = ?query.query).entered(); - let indices: Vec<_> = if query.libs { + // Handle special case: "::" alone or "::foo" for crate search + if query.is_crate_search() { + return search_crates(db, &query); + } + + // If we have a path filter, resolve it to target modules first + let indices: Vec<_> = if !query.path_filter.is_empty() { + let target_modules = resolve_path_to_modules( + db, + &query.path_filter, + query.anchor_to_crate, + query.case_sensitive, + ); + + if target_modules.is_empty() { + return vec![]; // Path doesn't resolve to any module + } + + // Get symbol indices only for the resolved modules + target_modules.iter().map(|&module| SymbolIndex::module_symbols(db, module)).collect() + } else if query.libs { + // Original behavior for non-path queries searching libs LibraryRoots::get(db) .roots(db) .par_iter() @@ -192,6 +289,7 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { .map(|&root| SymbolIndex::library_symbols(db, root)) .collect() } else { + // Original behavior for non-path queries searching local crates let mut crates = Vec::new(); for &root in LocalRoots::get(db).roots(db).iter() { @@ -204,13 +302,131 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { }; let mut res = vec![]; - query.search::<()>(&indices, |f| { - res.push(f.clone()); - ControlFlow::Continue(()) - }); + + // For module browsing (empty query, non-empty path_filter), return all symbols + if query.is_module_browsing() { + for index in &indices { + for symbol in index.symbols.iter() { + // Apply existing filters (only_types, assoc_mode, exclude_imports, etc.) + if query.matches_symbol_filters(symbol) { + res.push(symbol.clone()); + } + } + } + } else { + // Normal search: use FST to match item name + query.search::<()>(&indices, |f| { + res.push(f.clone()); + ControlFlow::Continue(()) + }); + } + res } +/// Search for crates by name (handles "::" and "::foo" queries) +fn search_crates<'db>(db: &'db RootDatabase, query: &Query) -> Vec> { + let mut res = vec![]; + + for krate in Crate::all(db) { + let Some(display_name) = krate.display_name(db) else { continue }; + let crate_name = display_name.crate_name().as_str(); + + // If query is empty (sole "::"), return all crates + // Otherwise, fuzzy match the crate name + let matches = if query.query.is_empty() { + true + } else { + query.mode.check(&query.query, query.case_sensitive, crate_name) + }; + + if matches { + // Create a FileSymbol for the crate's root module + if let Some(symbol) = hir::symbols::FileSymbol::for_crate_root(db, krate) { + res.push(symbol); + } + } + } + + res +} + +/// Resolve a path filter to the target module(s) it points to. +/// Returns the modules whose symbol indices should be searched. +/// +/// The path_filter contains segments like ["std", "vec"] for a query like "std::vec::Vec". +/// We resolve this by: +/// 1. Finding crates matching the first segment +/// 2. Walking down the module tree following subsequent segments +fn resolve_path_to_modules( + db: &dyn HirDatabase, + path_filter: &[String], + anchor_to_crate: bool, + case_sensitive: bool, +) -> Vec { + let [first_segment, rest_segments @ ..] = path_filter else { + return vec![]; + }; + + // Helper for name comparison + let names_match = |actual: &str, expected: &str| -> bool { + if case_sensitive { actual == expected } else { actual.eq_ignore_ascii_case(expected) } + }; + + // Find crates matching the first segment + let matching_crates: Vec = Crate::all(db) + .into_iter() + .filter(|krate| { + krate + .display_name(db) + .is_some_and(|name| names_match(name.crate_name().as_str(), first_segment)) + }) + .collect(); + + // If anchor_to_crate is true, first segment MUST be a crate name + // If anchor_to_crate is false, first segment could be a crate OR a module in local crates + let mut candidate_modules: Vec = vec![]; + + // Add crate root modules for matching crates + for krate in matching_crates { + candidate_modules.push(krate.root_module(db)); + } + + // If not anchored to crate, also search for modules matching first segment in local crates + if !anchor_to_crate { + for &root in LocalRoots::get(db).roots(db).iter() { + for &krate in db.source_root_crates(root).iter() { + let root_module = Crate::from(krate).root_module(db); + for child in root_module.children(db) { + if let Some(name) = child.name(db) { + if names_match(name.as_str(), first_segment) { + candidate_modules.push(child); + } + } + } + } + } + } + + // Walk down the module tree for remaining path segments + for segment in rest_segments { + candidate_modules = candidate_modules + .into_iter() + .flat_map(|module| { + module.children(db).filter(|child| { + child.name(db).is_some_and(|name| names_match(name.as_str(), segment)) + }) + }) + .collect(); + + if candidate_modules.is_empty() { + break; + } + } + + candidate_modules +} + #[derive(Default)] pub struct SymbolIndex<'db> { symbols: Box<[FileSymbol<'db>]>, @@ -382,12 +598,14 @@ impl<'db> SymbolIndex<'db> { } impl Query { + /// Search symbols in the given indices. pub(crate) fn search<'db, T>( - self, + &self, indices: &[&'db SymbolIndex<'db>], cb: impl FnMut(&'db FileSymbol<'db>) -> ControlFlow, ) -> Option { let _p = tracing::info_span!("symbol_index::Query::search").entered(); + let mut op = fst::map::OpBuilder::new(); match self.mode { SearchMode::Exact => { @@ -466,6 +684,41 @@ impl Query { (true, AssocSearchMode::Exclude) | (false, AssocSearchMode::AssocItemsOnly) ) } + + /// Check if a symbol passes all filters except name matching. + /// Used for module browsing where we want all items in a module. + fn matches_symbol_filters(&self, symbol: &FileSymbol<'_>) -> bool { + // Check only_types filter + if self.only_types + && !matches!( + symbol.def, + hir::ModuleDef::Adt(..) + | hir::ModuleDef::TypeAlias(..) + | hir::ModuleDef::BuiltinType(..) + | hir::ModuleDef::Trait(..) + ) + { + return false; + } + + // Check assoc_mode filter + if !self.matches_assoc_mode(symbol.is_assoc) { + return false; + } + + // Check exclude_imports filter + if self.exclude_imports && symbol.is_import { + return false; + } + + // Check underscore prefix + let ignore_underscore_prefixed = !self.query.starts_with("__"); + if ignore_underscore_prefixed && symbol.name.as_str().starts_with("__") { + return false; + } + + true + } } #[cfg(test)] @@ -622,4 +875,395 @@ pub struct Foo; let symbols = world_symbols(&db, query); expect_file!["./test_data/test_symbols_exclude_imports.txt"].assert_debug_eq(&symbols); } + + #[test] + fn test_parse_path_query() { + // Plain query - no path + let (path, item, anchor) = Query::parse_path_query("Item"); + assert_eq!(path, Vec::::new()); + assert_eq!(item, "Item"); + assert!(!anchor); + + // Path with item + let (path, item, anchor) = Query::parse_path_query("foo::Item"); + assert_eq!(path, vec!["foo"]); + assert_eq!(item, "Item"); + assert!(!anchor); + + // Multi-segment path + let (path, item, anchor) = Query::parse_path_query("foo::bar::Item"); + assert_eq!(path, vec!["foo", "bar"]); + assert_eq!(item, "Item"); + assert!(!anchor); + + // Leading :: (anchor to crate) + let (path, item, anchor) = Query::parse_path_query("::std::vec::Vec"); + assert_eq!(path, vec!["std", "vec"]); + assert_eq!(item, "Vec"); + assert!(anchor); + + // Just "::" - return all crates + let (path, item, anchor) = Query::parse_path_query("::"); + assert_eq!(path, Vec::::new()); + assert_eq!(item, ""); + assert!(anchor); + + // "::foo" - fuzzy search crate names + let (path, item, anchor) = Query::parse_path_query("::foo"); + assert_eq!(path, Vec::::new()); + assert_eq!(item, "foo"); + assert!(anchor); + + // Trailing :: (module browsing) + let (path, item, anchor) = Query::parse_path_query("foo::"); + assert_eq!(path, vec!["foo"]); + assert_eq!(item, ""); + assert!(!anchor); + + // Full path with trailing :: + let (path, item, anchor) = Query::parse_path_query("foo::bar::"); + assert_eq!(path, vec!["foo", "bar"]); + assert_eq!(item, ""); + assert!(!anchor); + + // Absolute path with trailing :: + let (path, item, anchor) = Query::parse_path_query("::std::vec::"); + assert_eq!(path, vec!["std", "vec"]); + assert_eq!(item, ""); + assert!(anchor); + + // Empty segments should be filtered + let (path, item, anchor) = Query::parse_path_query("foo::::bar"); + assert_eq!(path, vec!["foo"]); + assert_eq!(item, "bar"); + assert!(!anchor); + } + + #[test] + fn test_query_modes() { + // Test is_module_browsing + let query = Query::new("foo::".to_owned()); + assert!(query.is_module_browsing()); + assert!(!query.is_crate_search()); + + // Test is_crate_search with sole :: + let query = Query::new("::".to_owned()); + assert!(!query.is_module_browsing()); + assert!(query.is_crate_search()); + + // Test is_crate_search with ::foo + let query = Query::new("::foo".to_owned()); + assert!(!query.is_module_browsing()); + assert!(query.is_crate_search()); + + // Normal query should be neither + let query = Query::new("foo".to_owned()); + assert!(!query.is_module_browsing()); + assert!(!query.is_crate_search()); + + // Path query should be neither + let query = Query::new("foo::bar".to_owned()); + assert!(!query.is_module_browsing()); + assert!(!query.is_crate_search()); + } + + #[test] + fn test_path_search() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:main +mod inner; +pub struct RootStruct; + +//- /inner.rs +pub struct InnerStruct; +pub mod nested { + pub struct NestedStruct; +} +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Search for item in specific module + let query = Query::new("inner::InnerStruct".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"InnerStruct"), "Expected InnerStruct in {:?}", names); + + // Search for item in nested module + let query = Query::new("inner::nested::NestedStruct".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"NestedStruct"), "Expected NestedStruct in {:?}", names); + + // Search with crate prefix + let query = Query::new("main::inner::InnerStruct".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"InnerStruct"), "Expected InnerStruct in {:?}", names); + + // Wrong path should return empty + let query = Query::new("wrong::InnerStruct".to_owned()); + let symbols = world_symbols(&db, query); + assert!(symbols.is_empty(), "Expected empty results for wrong path"); + } + + #[test] + fn test_module_browsing() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:main +mod mymod; + +//- /mymod.rs +pub struct MyStruct; +pub fn my_func() {} +pub const MY_CONST: u32 = 1; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Browse all items in module + let query = Query::new("main::mymod::".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + + assert!(names.contains(&"MyStruct"), "Expected MyStruct in {:?}", names); + assert!(names.contains(&"my_func"), "Expected my_func in {:?}", names); + assert!(names.contains(&"MY_CONST"), "Expected MY_CONST in {:?}", names); + } + + #[test] + fn test_fuzzy_item_with_path() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:main +mod mymod; + +//- /mymod.rs +pub struct MyLongStructName; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Fuzzy match on item name with exact path + let query = Query::new("main::mymod::MyLong".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!( + names.contains(&"MyLongStructName"), + "Expected fuzzy match for MyLongStructName in {:?}", + names + ); + } + + #[test] + fn test_case_insensitive_path() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:main +mod MyMod; + +//- /MyMod.rs +pub struct MyStruct; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Case insensitive path matching (default) + let query = Query::new("main::mymod::MyStruct".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"MyStruct"), "Expected case-insensitive match in {:?}", names); + } + + #[test] + fn test_absolute_path_search() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:mycrate +mod inner; +pub struct CrateRoot; + +//- /inner.rs +pub struct InnerItem; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Absolute path with leading :: + let query = Query::new("::mycrate::inner::InnerItem".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!( + names.contains(&"InnerItem"), + "Expected InnerItem with absolute path in {:?}", + names + ); + + // Absolute path should NOT match if crate name is wrong + let query = Query::new("::wrongcrate::inner::InnerItem".to_owned()); + let symbols = world_symbols(&db, query); + assert!(symbols.is_empty(), "Expected empty results for wrong crate name"); + } + + #[test] + fn test_wrong_path_returns_empty() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:main +mod existing; + +//- /existing.rs +pub struct MyStruct; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Non-existent module path + let query = Query::new("nonexistent::MyStruct".to_owned()); + let symbols = world_symbols(&db, query); + assert!(symbols.is_empty(), "Expected empty results for non-existent path"); + + // Correct item, wrong module + let query = Query::new("wrongmod::MyStruct".to_owned()); + let symbols = world_symbols(&db, query); + assert!(symbols.is_empty(), "Expected empty results for wrong module"); + } + + #[test] + fn test_root_module_items() { + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:mylib +pub struct RootItem; +pub fn root_fn() {} +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Items at crate root - path is just the crate name + let query = Query::new("mylib::RootItem".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"RootItem"), "Expected RootItem at crate root in {:?}", names); + + // Browse crate root + let query = Query::new("mylib::".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + assert!( + names.contains(&"RootItem"), + "Expected RootItem when browsing crate root in {:?}", + names + ); + assert!( + names.contains(&"root_fn"), + "Expected root_fn when browsing crate root in {:?}", + names + ); + } + + #[test] + fn test_crate_search_all() { + // Test that sole "::" returns all crates + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:alpha +pub struct AlphaStruct; + +//- /beta.rs crate:beta +pub struct BetaStruct; + +//- /gamma.rs crate:gamma +pub struct GammaStruct; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // Sole "::" should return all crates (as module symbols) + let query = Query::new("::".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + + assert!(names.contains(&"alpha"), "Expected alpha crate in {:?}", names); + assert!(names.contains(&"beta"), "Expected beta crate in {:?}", names); + assert!(names.contains(&"gamma"), "Expected gamma crate in {:?}", names); + assert_eq!(symbols.len(), 3, "Expected exactly 3 crates, got {:?}", names); + } + + #[test] + fn test_crate_search_fuzzy() { + // Test that "::foo" fuzzy-matches crate names + let (mut db, _) = RootDatabase::with_many_files( + r#" +//- /lib.rs crate:my_awesome_lib +pub struct AwesomeStruct; + +//- /other.rs crate:another_lib +pub struct OtherStruct; + +//- /foo.rs crate:foobar +pub struct FooStruct; +"#, + ); + + let mut local_roots = FxHashSet::default(); + local_roots.insert(WORKSPACE); + LocalRoots::get(&db).set_roots(&mut db).to(local_roots); + + // "::foo" should fuzzy-match crate names containing "foo" + let query = Query::new("::foo".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + + assert!(names.contains(&"foobar"), "Expected foobar crate in {:?}", names); + assert_eq!(symbols.len(), 1, "Expected only foobar crate, got {:?}", names); + + // "::awesome" should match my_awesome_lib + let query = Query::new("::awesome".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + + assert!(names.contains(&"my_awesome_lib"), "Expected my_awesome_lib crate in {:?}", names); + assert_eq!(symbols.len(), 1, "Expected only my_awesome_lib crate, got {:?}", names); + + // "::lib" should match multiple crates + let query = Query::new("::lib".to_owned()); + let symbols = world_symbols(&db, query); + let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect(); + + assert!(names.contains(&"my_awesome_lib"), "Expected my_awesome_lib in {:?}", names); + assert!(names.contains(&"another_lib"), "Expected another_lib in {:?}", names); + assert_eq!(symbols.len(), 2, "Expected 2 crates matching 'lib', got {:?}", names); + + // "::nonexistent" should return empty + let query = Query::new("::nonexistent".to_owned()); + let symbols = world_symbols(&db, query); + assert!(symbols.is_empty(), "Expected empty results for non-matching crate pattern"); + } } From adbdf8dbf04593bcdadf2edda98c6284eef5d295 Mon Sep 17 00:00:00 2001 From: Jamie Hill-Daniel Date: Wed, 7 Jan 2026 14:02:09 +0000 Subject: [PATCH 0095/1843] Allow invoking all help options at once --- compiler/rustc_driver_impl/src/lib.rs | 92 ++++++++++++++++++++++----- compiler/rustc_session/src/options.rs | 2 + 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 7820198f2dcf..81d25a307bdd 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -228,8 +228,10 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send)) let args = args::arg_expand_all(&default_early_dcx, at_args); - let Some(matches) = handle_options(&default_early_dcx, &args) else { - return; + let (matches, help_only) = match handle_options(&default_early_dcx, &args) { + HandledOptions::None => return, + HandledOptions::Normal(matches) => (matches, false), + HandledOptions::HelpOnly(matches) => (matches, true), }; let sopts = config::build_session_options(&mut default_early_dcx, &matches); @@ -291,6 +293,11 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send)) return early_exit(); } + // We have now handled all help options, exit + if help_only { + return early_exit(); + } + if print_crate_info(codegen_backend, sess, has_input) == Compilation::Stop { return early_exit(); } @@ -1092,7 +1099,7 @@ pub fn describe_flag_categories(early_dcx: &EarlyDiagCtxt, matches: &Matches) -> // Don't handle -W help here, because we might first load additional lints. let debug_flags = matches.opt_strs("Z"); if debug_flags.iter().any(|x| *x == "help") { - describe_debug_flags(); + describe_unstable_flags(); return true; } @@ -1131,8 +1138,8 @@ fn get_backend_from_raw_matches( get_codegen_backend(early_dcx, &sysroot, backend_name, &target) } -fn describe_debug_flags() { - safe_println!("\nAvailable options:\n"); +fn describe_unstable_flags() { + safe_println!("\nAvailable unstable options:\n"); print_flag_list("-Z", config::Z_OPTIONS); } @@ -1156,6 +1163,16 @@ fn print_flag_list(cmdline_opt: &str, flag_list: &[OptionDesc]) { } } +pub enum HandledOptions { + /// Parsing failed, or we parsed a flag causing an early exit + None, + /// Successful parsing + Normal(getopts::Matches), + /// Parsing succeeded, but we received one or more 'help' flags + /// The compiler should proceed only until a possible `-W help` flag has been processed + HelpOnly(getopts::Matches), +} + /// Process command line options. Emits messages as appropriate. If compilation /// should continue, returns a getopts::Matches object parsed from args, /// otherwise returns `None`. @@ -1183,7 +1200,7 @@ fn print_flag_list(cmdline_opt: &str, flag_list: &[OptionDesc]) { /// This does not need to be `pub` for rustc itself, but @chaosite needs it to /// be public when using rustc as a library, see /// -pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> Option { +pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> HandledOptions { // Parse with *all* options defined in the compiler, we don't worry about // option stability here we just want to parse as much as possible. let mut options = getopts::Options::new(); @@ -1229,26 +1246,69 @@ pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> Option bool { + let opt_pos = |opt| matches.opt_positions(opt).first().copied(); + let opt_help_pos = |opt| { + matches + .opt_strs_pos(opt) + .iter() + .filter_map(|(pos, oval)| if oval == "help" { Some(*pos) } else { None }) + .next() + }; + let help_pos = if args.is_empty() { Some(0) } else { opt_pos("h").or_else(|| opt_pos("help")) }; + let zhelp_pos = opt_help_pos("Z"); + let chelp_pos = opt_help_pos("C"); + let print_help = || { + // Only show unstable options in --help if we accept unstable options. + let unstable_enabled = nightly_options::is_unstable_enabled(&matches); + let nightly_build = nightly_options::match_is_nightly_build(&matches); + usage(matches.opt_present("verbose"), unstable_enabled, nightly_build); + }; + + let mut helps = [ + (help_pos, &print_help as &dyn Fn()), + (zhelp_pos, &describe_unstable_flags), + (chelp_pos, &describe_codegen_flags), + ]; + helps.sort_by_key(|(pos, _)| pos.clone()); + let mut printed_any = false; + for printer in helps.iter().filter_map(|(pos, func)| pos.is_some().then_some(func)) { + printer(); + printed_any = true; + } + printed_any } /// Warn if `-o` is used without a space between the flag name and the value diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 21fa3321a30a..0a29d45457d5 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2125,6 +2125,7 @@ options! { #[rustc_lint_opt_deny_field_access("use `Session::must_emit_unwind_tables` instead of this field")] force_unwind_tables: Option = (None, parse_opt_bool, [TRACKED], "force use of unwind tables"), + help: bool = (false, parse_no_value, [UNTRACKED], "Print codegen options"), incremental: Option = (None, parse_opt_string, [UNTRACKED], "enable incremental compilation"), #[rustc_lint_opt_deny_field_access("documented to do nothing")] @@ -2395,6 +2396,7 @@ options! { environment variable `RUSTC_GRAPHVIZ_FONT` (default: `Courier, monospace`)"), has_thread_local: Option = (None, parse_opt_bool, [TRACKED], "explicitly enable the `cfg(target_thread_local)` directive"), + help: bool = (false, parse_no_value, [UNTRACKED], "Print unstable compiler options"), higher_ranked_assumptions: bool = (false, parse_bool, [TRACKED], "allow deducing higher-ranked outlives assumptions from coroutines when proving auto traits"), hint_mostly_unused: bool = (false, parse_bool, [TRACKED], From 4909fd8c7732ffa7d9641dc200b1010a2614c30f Mon Sep 17 00:00:00 2001 From: Jamie Hill-Daniel Date: Wed, 7 Jan 2026 16:53:05 +0000 Subject: [PATCH 0096/1843] Add tests for invoking multiple help options at once --- tests/run-make/rustc-help/rmake.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/run-make/rustc-help/rmake.rs b/tests/run-make/rustc-help/rmake.rs index 85e90e6352d0..3569cba3fcce 100644 --- a/tests/run-make/rustc-help/rmake.rs +++ b/tests/run-make/rustc-help/rmake.rs @@ -18,4 +18,31 @@ fn main() { // Check the diff between `rustc --help` and `rustc --help -v`. let help_v_diff = similar::TextDiff::from_lines(&help, &help_v).unified_diff().to_string(); diff().expected_file("help-v.diff").actual_text("actual", &help_v_diff).run(); + + // Check that all help options can be invoked at once + let codegen_help = bare_rustc().arg("-Chelp").run().stdout_utf8(); + let unstable_help = bare_rustc().arg("-Zhelp").run().stdout_utf8(); + let lints_help = bare_rustc().arg("-Whelp").run().stdout_utf8(); + let expected_all = format!("{help}{codegen_help}{unstable_help}{lints_help}"); + let all_help = bare_rustc().args(["--help", "-Chelp", "-Zhelp", "-Whelp"]).run().stdout_utf8(); + diff() + .expected_text( + "(rustc --help && rustc -Chelp && rustc -Zhelp && rustc -Whelp)", + &expected_all, + ) + .actual_text("(rustc --help -Chelp -Zhelp -Whelp)", &all_help) + .run(); + + // Check that the ordering of help options is respected + // Note that this is except for `-Whelp`, which always comes last + let expected_ordered_help = format!("{unstable_help}{codegen_help}{help}{lints_help}"); + let ordered_help = + bare_rustc().args(["-Whelp", "-Zhelp", "-Chelp", "--help"]).run().stdout_utf8(); + diff() + .expected_text( + "(rustc -Whelp && rustc -Zhelp && rustc -Chelp && rustc --help)", + &expected_ordered_help, + ) + .actual_text("(rustc -Whelp -Zhelp -Chelp --help)", &ordered_help) + .run(); } From e834e60987288843436e7d44bb784cb297475dc0 Mon Sep 17 00:00:00 2001 From: Jamie Hill-Daniel Date: Wed, 7 Jan 2026 16:55:37 +0000 Subject: [PATCH 0097/1843] Test behavior of `--help --invalid-flag` --- tests/run-make/rustc-help/rmake.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/run-make/rustc-help/rmake.rs b/tests/run-make/rustc-help/rmake.rs index 3569cba3fcce..17811ef18449 100644 --- a/tests/run-make/rustc-help/rmake.rs +++ b/tests/run-make/rustc-help/rmake.rs @@ -45,4 +45,7 @@ fn main() { ) .actual_text("(rustc -Whelp -Zhelp -Chelp --help)", &ordered_help) .run(); + + // Test that `rustc --help` does not suppress invalid flag errors + let help = bare_rustc().arg("--help --invalid-flag").run_fail().stdout_utf8(); } From 7630884c55c77baf3cb399b67146f1f5bd5b18df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= Date: Tue, 6 Jan 2026 20:01:25 +0100 Subject: [PATCH 0098/1843] Avoid `unsafe fn` in remaining x86 tests --- .../stdarch/crates/core_arch/src/x86/avx.rs | 231 +++-- .../stdarch/crates/core_arch/src/x86/avx2.rs | 374 ++++--- .../crates/core_arch/src/x86/avx512bf16.rs | 279 ++--- .../crates/core_arch/src/x86/avx512bw.rs | 314 +++--- .../crates/core_arch/src/x86/avx512dq.rs | 52 +- .../crates/core_arch/src/x86/avx512f.rs | 966 +++++++++++------- .../crates/core_arch/src/x86/avx512fp16.rs | 99 +- .../crates/core_arch/src/x86/avx512vbmi2.rs | 124 ++- .../crates/core_arch/src/x86/avxneconvert.rs | 56 +- .../stdarch/crates/core_arch/src/x86/f16c.rs | 2 +- .../stdarch/crates/core_arch/src/x86/fxsr.rs | 10 +- .../stdarch/crates/core_arch/src/x86/gfni.rs | 210 ++-- .../stdarch/crates/core_arch/src/x86/kl.rs | 88 +- .../stdarch/crates/core_arch/src/x86/rtm.rs | 34 +- .../stdarch/crates/core_arch/src/x86/sse.rs | 60 +- .../stdarch/crates/core_arch/src/x86/sse2.rs | 178 ++-- .../stdarch/crates/core_arch/src/x86/sse3.rs | 8 +- .../stdarch/crates/core_arch/src/x86/sse41.rs | 12 +- .../stdarch/crates/core_arch/src/x86/sse42.rs | 3 +- .../stdarch/crates/core_arch/src/x86/sse4a.rs | 12 +- .../stdarch/crates/core_arch/src/x86/xsave.rs | 36 +- .../crates/core_arch/src/x86_64/amx.rs | 854 ++++++++-------- .../crates/core_arch/src/x86_64/avx512f.rs | 659 +++++++----- .../crates/core_arch/src/x86_64/fxsr.rs | 10 +- .../crates/core_arch/src/x86_64/sse2.rs | 6 +- .../crates/core_arch/src/x86_64/xsave.rs | 30 +- 26 files changed, 2691 insertions(+), 2016 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx.rs b/library/stdarch/crates/core_arch/src/x86/avx.rs index e0e01ae6d034..c406d537bc1c 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx.rs @@ -3369,6 +3369,7 @@ unsafe extern "C" { #[cfg(test)] mod tests { use crate::core_arch::assert_eq_const as assert_eq; + use crate::core_arch::simd::*; use crate::hint::black_box; use crate::ptr; use stdarch_test::simd_test; @@ -3464,7 +3465,7 @@ mod tests { } #[simd_test(enable = "avx")] - unsafe fn test_mm256_max_pd() { + fn test_mm256_max_pd() { let a = _mm256_setr_pd(1., 4., 5., 8.); let b = _mm256_setr_pd(2., 3., 6., 7.); let r = _mm256_max_pd(a, b); @@ -3474,23 +3475,22 @@ mod tests { // > value in the second operand (source operand) is returned. let w = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); let x = _mm256_max_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); - let wu: [u64; 4] = transmute(w); - let xu: [u64; 4] = transmute(x); - assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); - assert_eq!(xu, [0u64; 4]); + let wu = _mm256_castpd_si256(w).as_u64x4(); + let xu = _mm256_castpd_si256(x).as_u64x4(); + assert_eq!(wu, u64x4::splat(0x8000_0000_0000_0000u64)); + assert_eq!(xu, u64x4::splat(0u64)); // > If only one value is a NaN (SNaN or QNaN) for this instruction, the // > second operand (source operand), either a NaN or a valid // > floating-point value, is written to the result. let y = _mm256_max_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); let z = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); - let yf: [f64; 4] = transmute(y); - let zf: [f64; 4] = transmute(z); - assert_eq!(yf, [0.0; 4]); + assert_eq_m256d(y, _mm256_set1_pd(0.0)); + let zf = *z.as_f64x4().as_array(); assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] - unsafe fn test_mm256_max_ps() { + fn test_mm256_max_ps() { let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); let r = _mm256_max_ps(a, b); @@ -3500,23 +3500,22 @@ mod tests { // > value in the second operand (source operand) is returned. let w = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); let x = _mm256_max_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); - let wu: [u32; 8] = transmute(w); - let xu: [u32; 8] = transmute(x); - assert_eq!(wu, [0x8000_0000u32; 8]); - assert_eq!(xu, [0u32; 8]); + let wu = _mm256_castps_si256(w).as_u32x8(); + let xu = _mm256_castps_si256(x).as_u32x8(); + assert_eq!(wu, u32x8::splat(0x8000_0000u32)); + assert_eq!(xu, u32x8::splat(0u32)); // > If only one value is a NaN (SNaN or QNaN) for this instruction, the // > second operand (source operand), either a NaN or a valid // > floating-point value, is written to the result. let y = _mm256_max_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); let z = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); - let yf: [f32; 8] = transmute(y); - let zf: [f32; 8] = transmute(z); - assert_eq!(yf, [0.0; 8]); + assert_eq_m256(y, _mm256_set1_ps(0.0)); + let zf = *z.as_f32x8().as_array(); assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] - unsafe fn test_mm256_min_pd() { + fn test_mm256_min_pd() { let a = _mm256_setr_pd(1., 4., 5., 8.); let b = _mm256_setr_pd(2., 3., 6., 7.); let r = _mm256_min_pd(a, b); @@ -3526,23 +3525,22 @@ mod tests { // > value in the second operand (source operand) is returned. let w = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); let x = _mm256_min_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); - let wu: [u64; 4] = transmute(w); - let xu: [u64; 4] = transmute(x); - assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); - assert_eq!(xu, [0u64; 4]); + let wu = _mm256_castpd_si256(w).as_u64x4(); + let xu = _mm256_castpd_si256(x).as_u64x4(); + assert_eq!(wu, u64x4::splat(0x8000_0000_0000_0000u64)); + assert_eq!(xu, u64x4::splat(0u64)); // > If only one value is a NaN (SNaN or QNaN) for this instruction, the // > second operand (source operand), either a NaN or a valid // > floating-point value, is written to the result. let y = _mm256_min_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); let z = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); - let yf: [f64; 4] = transmute(y); - let zf: [f64; 4] = transmute(z); - assert_eq!(yf, [0.0; 4]); + assert_eq_m256d(y, _mm256_set1_pd(0.0)); + let zf = *z.as_f64x4().as_array(); assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] - unsafe fn test_mm256_min_ps() { + fn test_mm256_min_ps() { let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); let r = _mm256_min_ps(a, b); @@ -3552,18 +3550,17 @@ mod tests { // > value in the second operand (source operand) is returned. let w = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); let x = _mm256_min_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); - let wu: [u32; 8] = transmute(w); - let xu: [u32; 8] = transmute(x); - assert_eq!(wu, [0x8000_0000u32; 8]); - assert_eq!(xu, [0u32; 8]); + let wu = _mm256_castps_si256(w).as_u32x8(); + let xu = _mm256_castps_si256(x).as_u32x8(); + assert_eq!(wu, u32x8::splat(0x8000_0000u32)); + assert_eq!(xu, u32x8::splat(0u32)); // > If only one value is a NaN (SNaN or QNaN) for this instruction, the // > second operand (source operand), either a NaN or a valid // > floating-point value, is written to the result. let y = _mm256_min_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); let z = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); - let yf: [f32; 8] = transmute(y); - let zf: [f32; 8] = transmute(z); - assert_eq!(yf, [0.0; 8]); + assert_eq_m256(y, _mm256_set1_ps(0.0)); + let zf = *z.as_f32x8().as_array(); assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } @@ -4247,183 +4244,203 @@ mod tests { } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_load_pd() { + const fn test_mm256_load_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let p = ptr::addr_of!(a) as *const f64; - let r = _mm256_load_pd(p); + let r = unsafe { _mm256_load_pd(p) }; let e = _mm256_setr_pd(1., 2., 3., 4.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_store_pd() { + const fn test_mm256_store_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let mut r = _mm256_undefined_pd(); - _mm256_store_pd(ptr::addr_of_mut!(r) as *mut f64, a); + unsafe { + _mm256_store_pd(ptr::addr_of_mut!(r) as *mut f64, a); + } assert_eq_m256d(r, a); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_load_ps() { + const fn test_mm256_load_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); let p = ptr::addr_of!(a) as *const f32; - let r = _mm256_load_ps(p); + let r = unsafe { _mm256_load_ps(p) }; let e = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); assert_eq_m256(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_store_ps() { + const fn test_mm256_store_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); let mut r = _mm256_undefined_ps(); - _mm256_store_ps(ptr::addr_of_mut!(r) as *mut f32, a); + unsafe { + _mm256_store_ps(ptr::addr_of_mut!(r) as *mut f32, a); + } assert_eq_m256(r, a); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_loadu_pd() { + const fn test_mm256_loadu_pd() { let a = &[1.0f64, 2., 3., 4.]; let p = a.as_ptr(); - let r = _mm256_loadu_pd(black_box(p)); + let r = unsafe { _mm256_loadu_pd(black_box(p)) }; let e = _mm256_setr_pd(1., 2., 3., 4.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_storeu_pd() { + const fn test_mm256_storeu_pd() { let a = _mm256_set1_pd(9.); let mut r = _mm256_undefined_pd(); - _mm256_storeu_pd(ptr::addr_of_mut!(r) as *mut f64, a); + unsafe { + _mm256_storeu_pd(ptr::addr_of_mut!(r) as *mut f64, a); + } assert_eq_m256d(r, a); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_loadu_ps() { + const fn test_mm256_loadu_ps() { let a = &[4., 3., 2., 5., 8., 9., 64., 50.]; let p = a.as_ptr(); - let r = _mm256_loadu_ps(black_box(p)); + let r = unsafe { _mm256_loadu_ps(black_box(p)) }; let e = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); assert_eq_m256(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_storeu_ps() { + const fn test_mm256_storeu_ps() { let a = _mm256_set1_ps(9.); let mut r = _mm256_undefined_ps(); - _mm256_storeu_ps(ptr::addr_of_mut!(r) as *mut f32, a); + unsafe { + _mm256_storeu_ps(ptr::addr_of_mut!(r) as *mut f32, a); + } assert_eq_m256(r, a); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_load_si256() { + const fn test_mm256_load_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let p = ptr::addr_of!(a); - let r = _mm256_load_si256(p); + let r = unsafe { _mm256_load_si256(p) }; let e = _mm256_setr_epi64x(1, 2, 3, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_store_si256() { + const fn test_mm256_store_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let mut r = _mm256_undefined_si256(); - _mm256_store_si256(ptr::addr_of_mut!(r), a); + unsafe { + _mm256_store_si256(ptr::addr_of_mut!(r), a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_loadu_si256() { + const fn test_mm256_loadu_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let p = ptr::addr_of!(a); - let r = _mm256_loadu_si256(black_box(p)); + let r = unsafe { _mm256_loadu_si256(black_box(p)) }; let e = _mm256_setr_epi64x(1, 2, 3, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_storeu_si256() { + const fn test_mm256_storeu_si256() { let a = _mm256_set1_epi8(9); let mut r = _mm256_undefined_si256(); - _mm256_storeu_si256(ptr::addr_of_mut!(r), a); + unsafe { + _mm256_storeu_si256(ptr::addr_of_mut!(r), a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_maskload_pd() { + const fn test_mm256_maskload_pd() { let a = &[1.0f64, 2., 3., 4.]; let p = a.as_ptr(); let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let r = _mm256_maskload_pd(black_box(p), mask); + let r = unsafe { _mm256_maskload_pd(black_box(p), mask) }; let e = _mm256_setr_pd(0., 2., 0., 4.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_maskstore_pd() { + const fn test_mm256_maskstore_pd() { let mut r = _mm256_set1_pd(0.); let mask = _mm256_setr_epi64x(0, !0, 0, !0); let a = _mm256_setr_pd(1., 2., 3., 4.); - _mm256_maskstore_pd(ptr::addr_of_mut!(r) as *mut f64, mask, a); + unsafe { + _mm256_maskstore_pd(ptr::addr_of_mut!(r) as *mut f64, mask, a); + } let e = _mm256_setr_pd(0., 2., 0., 4.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm_maskload_pd() { + const fn test_mm_maskload_pd() { let a = &[1.0f64, 2.]; let p = a.as_ptr(); let mask = _mm_setr_epi64x(0, !0); - let r = _mm_maskload_pd(black_box(p), mask); + let r = unsafe { _mm_maskload_pd(black_box(p), mask) }; let e = _mm_setr_pd(0., 2.); assert_eq_m128d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm_maskstore_pd() { + const fn test_mm_maskstore_pd() { let mut r = _mm_set1_pd(0.); let mask = _mm_setr_epi64x(0, !0); let a = _mm_setr_pd(1., 2.); - _mm_maskstore_pd(ptr::addr_of_mut!(r) as *mut f64, mask, a); + unsafe { + _mm_maskstore_pd(ptr::addr_of_mut!(r) as *mut f64, mask, a); + } let e = _mm_setr_pd(0., 2.); assert_eq_m128d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_maskload_ps() { + const fn test_mm256_maskload_ps() { let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; let p = a.as_ptr(); let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let r = _mm256_maskload_ps(black_box(p), mask); + let r = unsafe { _mm256_maskload_ps(black_box(p), mask) }; let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); assert_eq_m256(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_maskstore_ps() { + const fn test_mm256_maskstore_ps() { let mut r = _mm256_set1_ps(0.); let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - _mm256_maskstore_ps(ptr::addr_of_mut!(r) as *mut f32, mask, a); + unsafe { + _mm256_maskstore_ps(ptr::addr_of_mut!(r) as *mut f32, mask, a); + } let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); assert_eq_m256(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm_maskload_ps() { + const fn test_mm_maskload_ps() { let a = &[1.0f32, 2., 3., 4.]; let p = a.as_ptr(); let mask = _mm_setr_epi32(0, !0, 0, !0); - let r = _mm_maskload_ps(black_box(p), mask); + let r = unsafe { _mm_maskload_ps(black_box(p), mask) }; let e = _mm_setr_ps(0., 2., 0., 4.); assert_eq_m128(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm_maskstore_ps() { + const fn test_mm_maskstore_ps() { let mut r = _mm_set1_ps(0.); let mask = _mm_setr_epi32(0, !0, 0, !0); let a = _mm_setr_ps(1., 2., 3., 4.); - _mm_maskstore_ps(ptr::addr_of_mut!(r) as *mut f32, mask, a); + unsafe { + _mm_maskstore_ps(ptr::addr_of_mut!(r) as *mut f32, mask, a); + } let e = _mm_setr_ps(0., 2., 0., 4.); assert_eq_m128(r, e); } @@ -4453,7 +4470,7 @@ mod tests { } #[simd_test(enable = "avx")] - unsafe fn test_mm256_lddqu_si256() { + fn test_mm256_lddqu_si256() { #[rustfmt::skip] let a = _mm256_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, @@ -4462,7 +4479,7 @@ mod tests { 25, 26, 27, 28, 29, 30, 31, 32, ); let p = ptr::addr_of!(a); - let r = _mm256_lddqu_si256(black_box(p)); + let r = unsafe { _mm256_lddqu_si256(black_box(p)) }; #[rustfmt::skip] let e = _mm256_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, @@ -4475,17 +4492,19 @@ mod tests { #[simd_test(enable = "avx")] #[cfg_attr(miri, ignore)] // Non-temporal store, which is not supported by Miri - unsafe fn test_mm256_stream_si256() { + fn test_mm256_stream_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let mut r = _mm256_undefined_si256(); - _mm256_stream_si256(ptr::addr_of_mut!(r), a); + unsafe { + _mm256_stream_si256(ptr::addr_of_mut!(r), a); + } _mm_sfence(); assert_eq_m256i(r, a); } #[simd_test(enable = "avx")] #[cfg_attr(miri, ignore)] // Non-temporal store, which is not supported by Miri - unsafe fn test_mm256_stream_pd() { + fn test_mm256_stream_pd() { #[repr(align(32))] struct Memory { pub data: [f64; 4], @@ -4493,7 +4512,9 @@ mod tests { let a = _mm256_set1_pd(7.0); let mut mem = Memory { data: [-1.0; 4] }; - _mm256_stream_pd(ptr::addr_of_mut!(mem.data[0]), a); + unsafe { + _mm256_stream_pd(ptr::addr_of_mut!(mem.data[0]), a); + } _mm_sfence(); for i in 0..4 { assert_eq!(mem.data[i], get_m256d(a, i)); @@ -4502,7 +4523,7 @@ mod tests { #[simd_test(enable = "avx")] #[cfg_attr(miri, ignore)] // Non-temporal store, which is not supported by Miri - unsafe fn test_mm256_stream_ps() { + fn test_mm256_stream_ps() { #[repr(align(32))] struct Memory { pub data: [f32; 8], @@ -4510,7 +4531,9 @@ mod tests { let a = _mm256_set1_ps(7.0); let mut mem = Memory { data: [-1.0; 8] }; - _mm256_stream_ps(ptr::addr_of_mut!(mem.data[0]), a); + unsafe { + _mm256_stream_ps(ptr::addr_of_mut!(mem.data[0]), a); + } _mm_sfence(); for i in 0..8 { assert_eq!(mem.data[i], get_m256(a, i)); @@ -5147,29 +5170,29 @@ mod tests { } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_loadu2_m128() { + const fn test_mm256_loadu2_m128() { let hi = &[5., 6., 7., 8.]; let hiaddr = hi.as_ptr(); let lo = &[1., 2., 3., 4.]; let loaddr = lo.as_ptr(); - let r = _mm256_loadu2_m128(hiaddr, loaddr); + let r = unsafe { _mm256_loadu2_m128(hiaddr, loaddr) }; let e = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); assert_eq_m256(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_loadu2_m128d() { + const fn test_mm256_loadu2_m128d() { let hi = &[3., 4.]; let hiaddr = hi.as_ptr(); let lo = &[1., 2.]; let loaddr = lo.as_ptr(); - let r = _mm256_loadu2_m128d(hiaddr, loaddr); + let r = unsafe { _mm256_loadu2_m128d(hiaddr, loaddr) }; let e = _mm256_setr_pd(1., 2., 3., 4.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_loadu2_m128i() { + const fn test_mm256_loadu2_m128i() { #[rustfmt::skip] let hi = _mm_setr_epi8( 17, 18, 19, 20, 21, 22, 23, 24, @@ -5180,7 +5203,9 @@ mod tests { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm256_loadu2_m128i(ptr::addr_of!(hi) as *const _, ptr::addr_of!(lo) as *const _); + let r = unsafe { + _mm256_loadu2_m128i(ptr::addr_of!(hi) as *const _, ptr::addr_of!(lo) as *const _) + }; #[rustfmt::skip] let e = _mm256_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, @@ -5192,35 +5217,39 @@ mod tests { } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_storeu2_m128() { + const fn test_mm256_storeu2_m128() { let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); let mut hi = _mm_undefined_ps(); let mut lo = _mm_undefined_ps(); - _mm256_storeu2_m128( - ptr::addr_of_mut!(hi) as *mut f32, - ptr::addr_of_mut!(lo) as *mut f32, - a, - ); + unsafe { + _mm256_storeu2_m128( + ptr::addr_of_mut!(hi) as *mut f32, + ptr::addr_of_mut!(lo) as *mut f32, + a, + ); + } assert_eq_m128(hi, _mm_setr_ps(5., 6., 7., 8.)); assert_eq_m128(lo, _mm_setr_ps(1., 2., 3., 4.)); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_storeu2_m128d() { + const fn test_mm256_storeu2_m128d() { let a = _mm256_setr_pd(1., 2., 3., 4.); let mut hi = _mm_undefined_pd(); let mut lo = _mm_undefined_pd(); - _mm256_storeu2_m128d( - ptr::addr_of_mut!(hi) as *mut f64, - ptr::addr_of_mut!(lo) as *mut f64, - a, - ); + unsafe { + _mm256_storeu2_m128d( + ptr::addr_of_mut!(hi) as *mut f64, + ptr::addr_of_mut!(lo) as *mut f64, + a, + ); + } assert_eq_m128d(hi, _mm_setr_pd(3., 4.)); assert_eq_m128d(lo, _mm_setr_pd(1., 2.)); } #[simd_test(enable = "avx")] - const unsafe fn test_mm256_storeu2_m128i() { + const fn test_mm256_storeu2_m128i() { #[rustfmt::skip] let a = _mm256_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, @@ -5230,7 +5259,9 @@ mod tests { ); let mut hi = _mm_undefined_si128(); let mut lo = _mm_undefined_si128(); - _mm256_storeu2_m128i(ptr::addr_of_mut!(hi), ptr::addr_of_mut!(lo), a); + unsafe { + _mm256_storeu2_m128i(ptr::addr_of_mut!(hi), ptr::addr_of_mut!(lo), a); + } #[rustfmt::skip] let e_hi = _mm_setr_epi8( 17, 18, 19, 20, 21, 22, 23, 24, diff --git a/library/stdarch/crates/core_arch/src/x86/avx2.rs b/library/stdarch/crates/core_arch/src/x86/avx2.rs index 6f56e937c6db..6a39a0aaf8fe 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx2.rs @@ -4672,81 +4672,89 @@ mod tests { } #[simd_test(enable = "avx2")] - const unsafe fn test_mm_maskload_epi32() { + const fn test_mm_maskload_epi32() { let nums = [1, 2, 3, 4]; let a = &nums as *const i32; let mask = _mm_setr_epi32(-1, 0, 0, -1); - let r = _mm_maskload_epi32(a, mask); + let r = unsafe { _mm_maskload_epi32(a, mask) }; let e = _mm_setr_epi32(1, 0, 0, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm256_maskload_epi32() { + const fn test_mm256_maskload_epi32() { let nums = [1, 2, 3, 4, 5, 6, 7, 8]; let a = &nums as *const i32; let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0); - let r = _mm256_maskload_epi32(a, mask); + let r = unsafe { _mm256_maskload_epi32(a, mask) }; let e = _mm256_setr_epi32(1, 0, 0, 4, 0, 6, 7, 0); assert_eq_m256i(r, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm_maskload_epi64() { + const fn test_mm_maskload_epi64() { let nums = [1_i64, 2_i64]; let a = &nums as *const i64; let mask = _mm_setr_epi64x(0, -1); - let r = _mm_maskload_epi64(a, mask); + let r = unsafe { _mm_maskload_epi64(a, mask) }; let e = _mm_setr_epi64x(0, 2); assert_eq_m128i(r, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm256_maskload_epi64() { + const fn test_mm256_maskload_epi64() { let nums = [1_i64, 2_i64, 3_i64, 4_i64]; let a = &nums as *const i64; let mask = _mm256_setr_epi64x(0, -1, -1, 0); - let r = _mm256_maskload_epi64(a, mask); + let r = unsafe { _mm256_maskload_epi64(a, mask) }; let e = _mm256_setr_epi64x(0, 2, 3, 0); assert_eq_m256i(r, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm_maskstore_epi32() { + const fn test_mm_maskstore_epi32() { let a = _mm_setr_epi32(1, 2, 3, 4); let mut arr = [-1, -1, -1, -1]; let mask = _mm_setr_epi32(-1, 0, 0, -1); - _mm_maskstore_epi32(arr.as_mut_ptr(), mask, a); + unsafe { + _mm_maskstore_epi32(arr.as_mut_ptr(), mask, a); + } let e = [1, -1, -1, 4]; assert_eq!(arr, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm256_maskstore_epi32() { + const fn test_mm256_maskstore_epi32() { let a = _mm256_setr_epi32(1, 0x6d726f, 3, 42, 0x777161, 6, 7, 8); let mut arr = [-1, -1, -1, 0x776173, -1, 0x68657265, -1, -1]; let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0); - _mm256_maskstore_epi32(arr.as_mut_ptr(), mask, a); + unsafe { + _mm256_maskstore_epi32(arr.as_mut_ptr(), mask, a); + } let e = [1, -1, -1, 42, -1, 6, 7, -1]; assert_eq!(arr, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm_maskstore_epi64() { + const fn test_mm_maskstore_epi64() { let a = _mm_setr_epi64x(1_i64, 2_i64); let mut arr = [-1_i64, -1_i64]; let mask = _mm_setr_epi64x(0, -1); - _mm_maskstore_epi64(arr.as_mut_ptr(), mask, a); + unsafe { + _mm_maskstore_epi64(arr.as_mut_ptr(), mask, a); + } let e = [-1, 2]; assert_eq!(arr, e); } #[simd_test(enable = "avx2")] - const unsafe fn test_mm256_maskstore_epi64() { + const fn test_mm256_maskstore_epi64() { let a = _mm256_setr_epi64x(1_i64, 2_i64, 3_i64, 4_i64); let mut arr = [-1_i64, -1_i64, -1_i64, -1_i64]; let mask = _mm256_setr_epi64x(0, -1, -1, 0); - _mm256_maskstore_epi64(arr.as_mut_ptr(), mask, a); + unsafe { + _mm256_maskstore_epi64(arr.as_mut_ptr(), mask, a); + } let e = [-1, 2, 3, -1]; assert_eq!(arr, e); } @@ -5301,9 +5309,9 @@ mod tests { } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_stream_load_si256() { + fn test_mm256_stream_load_si256() { let a = _mm256_set_epi64x(5, 6, 7, 8); - let r = _mm256_stream_load_si256(core::ptr::addr_of!(a) as *const _); + let r = unsafe { _mm256_stream_load_si256(core::ptr::addr_of!(a) as *const _) }; assert_eq_m256i(a, r); } @@ -5506,88 +5514,98 @@ mod tests { } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i32gather_epi32() { + fn test_mm_i32gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + let r = unsafe { _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) }; assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_epi32() { + fn test_mm_mask_i32gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm_mask_i32gather_epi32::<4>( - _mm_set1_epi32(256), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm_setr_epi32(-1, -1, -1, 0), - ); + let r = unsafe { + _mm_mask_i32gather_epi32::<4>( + _mm_set1_epi32(256), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm_setr_epi32(-1, -1, -1, 0), + ) + }; assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i32gather_epi32() { + fn test_mm256_i32gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = - _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); + let r = unsafe { + _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)) + }; assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_epi32() { + fn test_mm256_mask_i32gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm256_mask_i32gather_epi32::<4>( - _mm256_set1_epi32(256), - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), - _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0), - ); + let r = unsafe { + _mm256_mask_i32gather_epi32::<4>( + _mm256_set1_epi32(256), + arr.as_ptr(), + _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), + _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0), + ) + }; assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 64, 256, 256, 256, 256, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i32gather_ps() { + fn test_mm_i32gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + let r = unsafe { _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) }; assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_ps() { + fn test_mm_mask_i32gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i32gather_ps::<4>( - _mm_set1_ps(256.0), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - ); + let r = unsafe { + _mm_mask_i32gather_ps::<4>( + _mm_set1_ps(256.0), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), + ) + }; assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i32gather_ps() { + fn test_mm256_i32gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = - _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); + let r = unsafe { + _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)) + }; assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 32.0, 48.0, 1.0, 2.0, 3.0, 4.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_ps() { + fn test_mm256_mask_i32gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i32gather_ps::<4>( - _mm256_set1_ps(256.0), - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), - _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0), - ); + let r = unsafe { + _mm256_mask_i32gather_ps::<4>( + _mm256_set1_ps(256.0), + arr.as_ptr(), + _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), + _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0), + ) + }; assert_eq_m256( r, _mm256_setr_ps(0.0, 16.0, 64.0, 256.0, 256.0, 256.0, 256.0, 256.0), @@ -5595,254 +5613,282 @@ mod tests { } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i32gather_epi64() { + fn test_mm_i32gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); + let r = unsafe { _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)) }; assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_epi64() { + fn test_mm_mask_i32gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i32gather_epi64::<8>( - _mm_set1_epi64x(256), - arr.as_ptr(), - _mm_setr_epi32(16, 16, 16, 16), - _mm_setr_epi64x(-1, 0), - ); + let r = unsafe { + _mm_mask_i32gather_epi64::<8>( + _mm_set1_epi64x(256), + arr.as_ptr(), + _mm_setr_epi32(16, 16, 16, 16), + _mm_setr_epi64x(-1, 0), + ) + }; assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i32gather_epi64() { + fn test_mm256_i32gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + let r = unsafe { _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) }; assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_epi64() { + fn test_mm256_mask_i32gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i32gather_epi64::<8>( - _mm256_set1_epi64x(256), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm256_setr_epi64x(-1, -1, -1, 0), - ); + let r = unsafe { + _mm256_mask_i32gather_epi64::<8>( + _mm256_set1_epi64x(256), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm256_setr_epi64x(-1, -1, -1, 0), + ) + }; assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i32gather_pd() { + fn test_mm_i32gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); + let r = unsafe { _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)) }; assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_pd() { + fn test_mm_mask_i32gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i32gather_pd::<8>( - _mm_set1_pd(256.0), - arr.as_ptr(), - _mm_setr_epi32(16, 16, 16, 16), - _mm_setr_pd(-1.0, 0.0), - ); + let r = unsafe { + _mm_mask_i32gather_pd::<8>( + _mm_set1_pd(256.0), + arr.as_ptr(), + _mm_setr_epi32(16, 16, 16, 16), + _mm_setr_pd(-1.0, 0.0), + ) + }; assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i32gather_pd() { + fn test_mm256_i32gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + let r = unsafe { _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)) }; assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_pd() { + fn test_mm256_mask_i32gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i32gather_pd::<8>( - _mm256_set1_pd(256.0), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - ); + let r = unsafe { + _mm256_mask_i32gather_pd::<8>( + _mm256_set1_pd(256.0), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), + ) + }; assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i64gather_epi32() { + fn test_mm_i64gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + let r = unsafe { _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) }; assert_eq_m128i(r, _mm_setr_epi32(0, 16, 0, 0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_epi32() { + fn test_mm_mask_i64gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm_mask_i64gather_epi32::<4>( - _mm_set1_epi32(256), - arr.as_ptr(), - _mm_setr_epi64x(0, 16), - _mm_setr_epi32(-1, 0, -1, 0), - ); + let r = unsafe { + _mm_mask_i64gather_epi32::<4>( + _mm_set1_epi32(256), + arr.as_ptr(), + _mm_setr_epi64x(0, 16), + _mm_setr_epi32(-1, 0, -1, 0), + ) + }; assert_eq_m128i(r, _mm_setr_epi32(0, 256, 0, 0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i64gather_epi32() { + fn test_mm256_i64gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + let r = + unsafe { _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) }; assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_epi32() { + fn test_mm256_mask_i64gather_epi32() { let arr: [i32; 128] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing - let r = _mm256_mask_i64gather_epi32::<4>( - _mm_set1_epi32(256), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm_setr_epi32(-1, -1, -1, 0), - ); + let r = unsafe { + _mm256_mask_i64gather_epi32::<4>( + _mm_set1_epi32(256), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm_setr_epi32(-1, -1, -1, 0), + ) + }; assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i64gather_ps() { + fn test_mm_i64gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + let r = unsafe { _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) }; assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 0.0, 0.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_ps() { + fn test_mm_mask_i64gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i64gather_ps::<4>( - _mm_set1_ps(256.0), - arr.as_ptr(), - _mm_setr_epi64x(0, 16), - _mm_setr_ps(-1.0, 0.0, -1.0, 0.0), - ); + let r = unsafe { + _mm_mask_i64gather_ps::<4>( + _mm_set1_ps(256.0), + arr.as_ptr(), + _mm_setr_epi64x(0, 16), + _mm_setr_ps(-1.0, 0.0, -1.0, 0.0), + ) + }; assert_eq_m128(r, _mm_setr_ps(0.0, 256.0, 0.0, 0.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i64gather_ps() { + fn test_mm256_i64gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + let r = + unsafe { _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) }; assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_ps() { + fn test_mm256_mask_i64gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i64gather_ps::<4>( - _mm_set1_ps(256.0), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - ); + let r = unsafe { + _mm256_mask_i64gather_ps::<4>( + _mm_set1_ps(256.0), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), + ) + }; assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i64gather_epi64() { + fn test_mm_i64gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + let r = unsafe { _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) }; assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_epi64() { + fn test_mm_mask_i64gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i64gather_epi64::<8>( - _mm_set1_epi64x(256), - arr.as_ptr(), - _mm_setr_epi64x(16, 16), - _mm_setr_epi64x(-1, 0), - ); + let r = unsafe { + _mm_mask_i64gather_epi64::<8>( + _mm_set1_epi64x(256), + arr.as_ptr(), + _mm_setr_epi64x(16, 16), + _mm_setr_epi64x(-1, 0), + ) + }; assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i64gather_epi64() { + fn test_mm256_i64gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + let r = + unsafe { _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) }; assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_epi64() { + fn test_mm256_mask_i64gather_epi64() { let arr: [i64; 128] = core::array::from_fn(|i| i as i64); // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i64gather_epi64::<8>( - _mm256_set1_epi64x(256), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm256_setr_epi64x(-1, -1, -1, 0), - ); + let r = unsafe { + _mm256_mask_i64gather_epi64::<8>( + _mm256_set1_epi64x(256), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm256_setr_epi64x(-1, -1, -1, 0), + ) + }; assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_i64gather_pd() { + fn test_mm_i64gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + let r = unsafe { _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)) }; assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_pd() { + fn test_mm_mask_i64gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i64gather_pd::<8>( - _mm_set1_pd(256.0), - arr.as_ptr(), - _mm_setr_epi64x(16, 16), - _mm_setr_pd(-1.0, 0.0), - ); + let r = unsafe { + _mm_mask_i64gather_pd::<8>( + _mm_set1_pd(256.0), + arr.as_ptr(), + _mm_setr_epi64x(16, 16), + _mm_setr_pd(-1.0, 0.0), + ) + }; assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_i64gather_pd() { + fn test_mm256_i64gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + let r = + unsafe { _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)) }; assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); } #[simd_test(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_pd() { + fn test_mm256_mask_i64gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i64gather_pd::<8>( - _mm256_set1_pd(256.0), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - ); + let r = unsafe { + _mm256_mask_i64gather_pd::<8>( + _mm256_set1_pd(256.0), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), + ) + }; assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bf16.rs b/library/stdarch/crates/core_arch/src/x86/avx512bf16.rs index 83dbc540a0c8..66eef063eed8 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bf16.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bf16.rs @@ -593,7 +593,7 @@ pub fn _mm_cvtness_sbh(a: f32) -> bf16 { #[cfg(test)] mod tests { - use crate::core_arch::simd::u16x4; + use crate::core_arch::simd::{f32x4, f32x8, f32x16, u16x4, u16x8, u16x16, u16x32}; use crate::{ core_arch::x86::*, mem::{transmute, transmute_copy}, @@ -601,13 +601,13 @@ mod tests { use stdarch_test::simd_test; #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_cvtne2ps_pbh() { + fn test_mm_cvtne2ps_pbh() { let a_array = [178.125_f32, 10.5_f32, 3.75_f32, 50.25_f32]; let b_array = [-178.125_f32, -10.5_f32, -3.75_f32, -50.25_f32]; - let a: __m128 = transmute(a_array); - let b: __m128 = transmute(b_array); + let a = f32x4::from_array(a_array).as_m128(); + let b = f32x4::from_array(b_array).as_m128(); let c: __m128bh = _mm_cvtne2ps_pbh(a, b); - let result: [u16; 8] = transmute(c.as_u16x8()); + let result = *c.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0b1_10000110_0110010, @@ -623,7 +623,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_mask_cvtne2ps_pbh() { + fn test_mm_mask_cvtne2ps_pbh() { let a_array = [178.125_f32, 10.5_f32, 3.75_f32, 50.25_f32]; let b_array = [-178.125_f32, -10.5_f32, -3.75_f32, -50.25_f32]; #[rustfmt::skip] @@ -637,12 +637,12 @@ mod tests { 0b0_10000000_1110000, 0b0_10000100_1001001, ]; - let src: __m128bh = transmute(src_array); - let a: __m128 = transmute(a_array); - let b: __m128 = transmute(b_array); + let src = u16x8::from_array(src_array).as_m128bh(); + let a = f32x4::from_array(a_array).as_m128(); + let b = f32x4::from_array(b_array).as_m128(); let k: __mmask8 = 0b1111_1111; let c: __m128bh = _mm_mask_cvtne2ps_pbh(src, k, a, b); - let result: [u16; 8] = transmute(c.as_u16x8()); + let result = *c.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0b1_10000110_0110010, @@ -657,20 +657,20 @@ mod tests { assert_eq!(result, expected_result); let k = 0b0000_0000; let c = _mm_mask_cvtne2ps_pbh(src, k, a, b); - let result: [u16; 8] = transmute(c.as_u16x8()); + let result = *c.as_u16x8().as_array(); let expected_result = src_array; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_maskz_cvtne2ps_pbh() { + fn test_mm_maskz_cvtne2ps_pbh() { let a_array = [178.125_f32, 10.5_f32, 3.75_f32, 50.25_f32]; let b_array = [-178.125_f32, -10.5_f32, -3.75_f32, -50.25_f32]; - let a: __m128 = transmute(a_array); - let b: __m128 = transmute(b_array); + let a = f32x4::from_array(a_array).as_m128(); + let b = f32x4::from_array(b_array).as_m128(); let k: __mmask8 = 0b1111_1111; let c: __m128bh = _mm_maskz_cvtne2ps_pbh(k, a, b); - let result: [u16; 8] = transmute(c.as_u16x8()); + let result = *c.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0b1_10000110_0110010, @@ -685,7 +685,7 @@ mod tests { assert_eq!(result, expected_result); let k = 0b0011_1100; let c = _mm_maskz_cvtne2ps_pbh(k, a, b); - let result: [u16; 8] = transmute(c.as_u16x8()); + let result = *c.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0, @@ -701,7 +701,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_cvtne2ps_pbh() { + fn test_mm256_cvtne2ps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -723,10 +723,10 @@ mod tests { -1000.158_f32, -575.575_f32, ]; - let a: __m256 = transmute(a_array); - let b: __m256 = transmute(b_array); + let a = f32x8::from_array(a_array).as_m256(); + let b = f32x8::from_array(b_array).as_m256(); let c: __m256bh = _mm256_cvtne2ps_pbh(a, b); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0b1_10000110_0110010, @@ -750,7 +750,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_mask_cvtne2ps_pbh() { + fn test_mm256_mask_cvtne2ps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -790,12 +790,12 @@ mod tests { 0b0_10000000_1110000, 0b0_10000100_1001001, ]; - let src: __m256bh = transmute(src_array); - let a: __m256 = transmute(a_array); - let b: __m256 = transmute(b_array); + let src = u16x16::from_array(src_array).as_m256bh(); + let a = f32x8::from_array(a_array).as_m256(); + let b = f32x8::from_array(b_array).as_m256(); let k: __mmask16 = 0xffff; let c: __m256bh = _mm256_mask_cvtne2ps_pbh(src, k, a, b); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0b1_10000110_0110010, @@ -818,13 +818,13 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0; let c: __m256bh = _mm256_mask_cvtne2ps_pbh(src, k, a, b); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); let expected_result = src_array; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_maskz_cvtne2ps_pbh() { + fn test_mm256_maskz_cvtne2ps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -846,11 +846,11 @@ mod tests { -1000.158_f32, -575.575_f32, ]; - let a: __m256 = transmute(a_array); - let b: __m256 = transmute(b_array); + let a = f32x8::from_array(a_array).as_m256(); + let b = f32x8::from_array(b_array).as_m256(); let k: __mmask16 = 0xffff; let c: __m256bh = _mm256_maskz_cvtne2ps_pbh(k, a, b); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0b1_10000110_0110010, @@ -873,7 +873,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0b0110_1100_0011_0110; let c: __m256bh = _mm256_maskz_cvtne2ps_pbh(k, a, b); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0, @@ -897,7 +897,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_cvtne2ps_pbh() { + fn test_mm512_cvtne2ps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -935,10 +935,10 @@ mod tests { -1000.158_f32, -575.575_f32, ]; - let a: __m512 = transmute(a_array); - let b: __m512 = transmute(b_array); + let a = f32x16::from_array(a_array).as_m512(); + let b = f32x16::from_array(b_array).as_m512(); let c: __m512bh = _mm512_cvtne2ps_pbh(a, b); - let result: [u16; 32] = transmute(c.as_u16x32()); + let result = *c.as_u16x32().as_array(); #[rustfmt::skip] let expected_result: [u16; 32] = [ 0b1_10000110_0110010, @@ -978,7 +978,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_mask_cvtne2ps_pbh() { + fn test_mm512_mask_cvtne2ps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1050,12 +1050,12 @@ mod tests { 0b0_10000000_1110000, 0b0_10000100_1001001, ]; - let src: __m512bh = transmute(src_array); - let a: __m512 = transmute(a_array); - let b: __m512 = transmute(b_array); + let src = u16x32::from_array(src_array).as_m512bh(); + let a = f32x16::from_array(a_array).as_m512(); + let b = f32x16::from_array(b_array).as_m512(); let k: __mmask32 = 0xffffffff; let c: __m512bh = _mm512_mask_cvtne2ps_pbh(src, k, a, b); - let result: [u16; 32] = transmute(c.as_u16x32()); + let result = *c.as_u16x32().as_array(); #[rustfmt::skip] let expected_result: [u16; 32] = [ 0b1_10000110_0110010, @@ -1094,13 +1094,13 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask32 = 0; let c: __m512bh = _mm512_mask_cvtne2ps_pbh(src, k, a, b); - let result: [u16; 32] = transmute(c.as_u16x32()); + let result = *c.as_u16x32().as_array(); let expected_result = src_array; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_maskz_cvtne2ps_pbh() { + fn test_mm512_maskz_cvtne2ps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1138,11 +1138,11 @@ mod tests { -1000.158_f32, -575.575_f32, ]; - let a: __m512 = transmute(a_array); - let b: __m512 = transmute(b_array); + let a = f32x16::from_array(a_array).as_m512(); + let b = f32x16::from_array(b_array).as_m512(); let k: __mmask32 = 0xffffffff; let c: __m512bh = _mm512_maskz_cvtne2ps_pbh(k, a, b); - let result: [u16; 32] = transmute(c.as_u16x32()); + let result = *c.as_u16x32().as_array(); #[rustfmt::skip] let expected_result: [u16; 32] = [ 0b1_10000110_0110010, @@ -1181,7 +1181,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask32 = 0b1100_1010_1001_0110_1010_0011_0101_0110; let c: __m512bh = _mm512_maskz_cvtne2ps_pbh(k, a, b); - let result: [u16; 32] = transmute(c.as_u16x32()); + let result = *c.as_u16x32().as_array(); #[rustfmt::skip] let expected_result: [u16; 32] = [ 0, @@ -1221,7 +1221,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_cvtneps_pbh() { + fn test_mm256_cvtneps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1233,9 +1233,9 @@ mod tests { 1000.158_f32, 575.575_f32, ]; - let a: __m256 = transmute(a_array); + let a = f32x8::from_array(a_array).as_m256(); let c: __m128bh = _mm256_cvtneps_pbh(a); - let result: [u16; 8] = transmute(c.as_u16x8()); + let result = *c.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0b0_10000110_0110010, @@ -1251,7 +1251,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_mask_cvtneps_pbh() { + fn test_mm256_mask_cvtneps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1273,11 +1273,11 @@ mod tests { 0b1_10001000_1111010, 0b1_10001000_0010000, ]; - let src: __m128bh = transmute(src_array); - let a: __m256 = transmute(a_array); + let src = u16x8::from_array(src_array).as_m128bh(); + let a = f32x8::from_array(a_array).as_m256(); let k: __mmask8 = 0xff; let b = _mm256_mask_cvtneps_pbh(src, k, a); - let result: [u16; 8] = transmute(b.as_u16x8()); + let result = *b.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0b0_10000110_0110010, @@ -1292,13 +1292,13 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask8 = 0x0; let b: __m128bh = _mm256_mask_cvtneps_pbh(src, k, a); - let result: [u16; 8] = transmute(b.as_u16x8()); + let result = *b.as_u16x8().as_array(); let expected_result: [u16; 8] = src_array; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_maskz_cvtneps_pbh() { + fn test_mm256_maskz_cvtneps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1310,10 +1310,10 @@ mod tests { 1000.158_f32, 575.575_f32, ]; - let a: __m256 = transmute(a_array); + let a = f32x8::from_array(a_array).as_m256(); let k: __mmask8 = 0xff; let b = _mm256_maskz_cvtneps_pbh(k, a); - let result: [u16; 8] = transmute(b.as_u16x8()); + let result = *b.as_u16x8().as_array(); #[rustfmt::skip] let expected_result: [u16; 8] = [ 0b0_10000110_0110010, @@ -1328,14 +1328,14 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask8 = 0x6; let b: __m128bh = _mm256_maskz_cvtneps_pbh(k, a); - let result: [u16; 8] = transmute(b.as_u16x8()); + let result = *b.as_u16x8().as_array(); let expected_result: [u16; 8] = [0, 0b0_10000010_0101000, 0b0_10000000_1110000, 0, 0, 0, 0, 0]; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_cvtneps_pbh() { + fn test_mm512_cvtneps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1355,9 +1355,9 @@ mod tests { 1000.158_f32, 575.575_f32, ]; - let a: __m512 = transmute(a_array); + let a = f32x16::from_array(a_array).as_m512(); let c: __m256bh = _mm512_cvtneps_pbh(a); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0b0_10000110_0110010, @@ -1381,7 +1381,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_mask_cvtneps_pbh() { + fn test_mm512_mask_cvtneps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1419,11 +1419,11 @@ mod tests { 0b1_10001000_1111010, 0b1_10001000_0010000, ]; - let src: __m256bh = transmute(src_array); - let a: __m512 = transmute(a_array); + let src = u16x16::from_array(src_array).as_m256bh(); + let a = f32x16::from_array(a_array).as_m512(); let k: __mmask16 = 0xffff; let c: __m256bh = _mm512_mask_cvtneps_pbh(src, k, a); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0b0_10000110_0110010, @@ -1446,13 +1446,13 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0; let c: __m256bh = _mm512_mask_cvtneps_pbh(src, k, a); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); let expected_result = src_array; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_maskz_cvtneps_pbh() { + fn test_mm512_maskz_cvtneps_pbh() { #[rustfmt::skip] let a_array = [ 178.125_f32, @@ -1472,10 +1472,10 @@ mod tests { 1000.158_f32, 575.575_f32, ]; - let a: __m512 = transmute(a_array); + let a = f32x16::from_array(a_array).as_m512(); let k: __mmask16 = 0xffff; let c: __m256bh = _mm512_maskz_cvtneps_pbh(k, a); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0b0_10000110_0110010, @@ -1498,7 +1498,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0x653a; let c: __m256bh = _mm512_maskz_cvtneps_pbh(k, a); - let result: [u16; 16] = transmute(c.as_u16x16()); + let result = *c.as_u16x16().as_array(); #[rustfmt::skip] let expected_result: [u16; 16] = [ 0, @@ -1522,74 +1522,74 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_dpbf16_ps() { + fn test_mm_dpbf16_ps() { let a_array = [8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32]; let b_array = [-1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32]; - let a1: __m128 = transmute(a_array); - let b1: __m128 = transmute(b_array); - let src: __m128 = transmute([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]); + let a1 = f32x4::from_array(a_array).as_m128(); + let b1 = f32x4::from_array(b_array).as_m128(); + let src = f32x4::from_array([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]).as_m128(); let a: __m128bh = _mm_cvtne2ps_pbh(a1, a1); let b: __m128bh = _mm_cvtne2ps_pbh(b1, b1); let c: __m128 = _mm_dpbf16_ps(src, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32]; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_mask_dpbf16_ps() { + fn test_mm_mask_dpbf16_ps() { let a_array = [8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32]; let b_array = [-1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32]; - let a1: __m128 = transmute(a_array); - let b1: __m128 = transmute(b_array); + let a1 = f32x4::from_array(a_array).as_m128(); + let b1 = f32x4::from_array(b_array).as_m128(); let k: __mmask8 = 0xf3; - let src: __m128 = transmute([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]); + let src = f32x4::from_array([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]).as_m128(); let a: __m128bh = _mm_cvtne2ps_pbh(a1, a1); let b: __m128bh = _mm_cvtne2ps_pbh(b1, b1); let c: __m128 = _mm_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32]; assert_eq!(result, expected_result); let k: __mmask8 = 0xff; let c: __m128 = _mm_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32]; assert_eq!(result, expected_result); let k: __mmask8 = 0; let c: __m128 = _mm_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_maskz_dpbf16_ps() { + fn test_mm_maskz_dpbf16_ps() { let a_array = [8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32]; let b_array = [-1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32]; - let a1: __m128 = transmute(a_array); - let b1: __m128 = transmute(b_array); + let a1 = f32x4::from_array(a_array).as_m128(); + let b1 = f32x4::from_array(b_array).as_m128(); let k: __mmask8 = 0xf3; - let src: __m128 = transmute([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]); + let src = f32x4::from_array([1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32]).as_m128(); let a: __m128bh = _mm_cvtne2ps_pbh(a1, a1); let b: __m128bh = _mm_cvtne2ps_pbh(b1, b1); let c: __m128 = _mm_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, 0.0, 0.0]; assert_eq!(result, expected_result); let k: __mmask8 = 0xff; let c: __m128 = _mm_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [-18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32]; assert_eq!(result, expected_result); let k: __mmask8 = 0; let c: __m128 = _mm_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 4] = transmute(c.as_f32x4()); + let result = *c.as_f32x4().as_array(); let expected_result: [f32; 4] = [0.0, 0.0, 0.0, 0.0]; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_dpbf16_ps() { + fn test_mm256_dpbf16_ps() { #[rustfmt::skip] let a_array = [ 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, @@ -1597,16 +1597,16 @@ mod tests { let b_array = [ -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, ]; - let a1: __m256 = transmute(a_array); - let b1: __m256 = transmute(b_array); + let a1 = f32x8::from_array(a_array).as_m256(); + let b1 = f32x8::from_array(b_array).as_m256(); #[rustfmt::skip] - let src: __m256 = transmute([ + let src = f32x8::from_array([ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, - ]); + ]).as_m256(); let a: __m256bh = _mm256_cvtne2ps_pbh(a1, a1); let b: __m256bh = _mm256_cvtne2ps_pbh(b1, b1); let c: __m256 = _mm256_dpbf16_ps(src, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); #[rustfmt::skip] let expected_result: [f32; 8] = [ -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, @@ -1615,7 +1615,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_mask_dpbf16_ps() { + fn test_mm256_mask_dpbf16_ps() { #[rustfmt::skip] let a_array = [ 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, @@ -1623,17 +1623,17 @@ mod tests { let b_array = [ -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, ]; - let a1: __m256 = transmute(a_array); - let b1: __m256 = transmute(b_array); + let a1 = f32x8::from_array(a_array).as_m256(); + let b1 = f32x8::from_array(b_array).as_m256(); let k: __mmask8 = 0x33; #[rustfmt::skip] - let src: __m256 = transmute([ + let src = f32x8::from_array([ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, - ]); + ]).as_m256(); let a: __m256bh = _mm256_cvtne2ps_pbh(a1, a1); let b: __m256bh = _mm256_cvtne2ps_pbh(b1, b1); let c: __m256 = _mm256_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); #[rustfmt::skip] let expected_result: [f32; 8] = [ -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, @@ -1641,7 +1641,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask8 = 0xff; let c: __m256 = _mm256_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); #[rustfmt::skip] let expected_result: [f32; 8] = [ -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, @@ -1649,7 +1649,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask8 = 0; let c: __m256 = _mm256_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); #[rustfmt::skip] let expected_result: [f32; 8] = [ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, @@ -1658,7 +1658,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm256_maskz_dpbf16_ps() { + fn test_mm256_maskz_dpbf16_ps() { #[rustfmt::skip] let a_array = [ 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, @@ -1666,17 +1666,17 @@ mod tests { let b_array = [ -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, ]; - let a1: __m256 = transmute(a_array); - let b1: __m256 = transmute(b_array); + let a1 = f32x8::from_array(a_array).as_m256(); + let b1 = f32x8::from_array(b_array).as_m256(); let k: __mmask8 = 0x33; #[rustfmt::skip] - let src: __m256 = transmute([ + let src = f32x8::from_array([ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, - ]); + ]).as_m256(); let a: __m256bh = _mm256_cvtne2ps_pbh(a1, a1); let b: __m256bh = _mm256_cvtne2ps_pbh(b1, b1); let c: __m256 = _mm256_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); #[rustfmt::skip] let expected_result: [f32; 8] = [ -18.0_f32, -52.0_f32, 0.0, 0.0, -18.0_f32, -52.0_f32, 0.0, 0.0, @@ -1684,7 +1684,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask8 = 0xff; let c: __m256 = _mm256_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); #[rustfmt::skip] let expected_result: [f32; 8] = [ -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, @@ -1692,13 +1692,13 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask8 = 0; let c: __m256 = _mm256_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 8] = transmute(c.as_f32x8()); + let result = *c.as_f32x8().as_array(); let expected_result: [f32; 8] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; assert_eq!(result, expected_result); } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_dpbf16_ps() { + fn test_mm512_dpbf16_ps() { #[rustfmt::skip] let a_array = [ 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, @@ -1708,16 +1708,17 @@ mod tests { -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, ]; - let a1: __m512 = transmute(a_array); - let b1: __m512 = transmute(b_array); - let src: __m512 = transmute([ + let a1 = f32x16::from_array(a_array).as_m512(); + let b1 = f32x16::from_array(b_array).as_m512(); + let src = f32x16::from_array([ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, - ]); + ]) + .as_m512(); let a: __m512bh = _mm512_cvtne2ps_pbh(a1, a1); let b: __m512bh = _mm512_cvtne2ps_pbh(b1, b1); let c: __m512 = _mm512_dpbf16_ps(src, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, @@ -1727,7 +1728,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_mask_dpbf16_ps() { + fn test_mm512_mask_dpbf16_ps() { #[rustfmt::skip] let a_array = [ 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, @@ -1737,18 +1738,18 @@ mod tests { -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, ]; - let a1: __m512 = transmute(a_array); - let b1: __m512 = transmute(b_array); + let a1 = f32x16::from_array(a_array).as_m512(); + let b1 = f32x16::from_array(b_array).as_m512(); let k: __mmask16 = 0x3333; #[rustfmt::skip] - let src: __m512 = transmute([ + let src = f32x16::from_array([ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, - ]); + ]).as_m512(); let a: __m512bh = _mm512_cvtne2ps_pbh(a1, a1); let b: __m512bh = _mm512_cvtne2ps_pbh(b1, b1); let c: __m512 = _mm512_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, -18.0_f32, -52.0_f32, 3.0_f32, 4.0_f32, @@ -1757,7 +1758,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0xffff; let c: __m512 = _mm512_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, @@ -1766,7 +1767,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0; let c: __m512 = _mm512_mask_dpbf16_ps(src, k, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, @@ -1776,7 +1777,7 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512f")] - unsafe fn test_mm512_maskz_dpbf16_ps() { + fn test_mm512_maskz_dpbf16_ps() { #[rustfmt::skip] let a_array = [ 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, 8.5_f32, 10.5_f32, 3.75_f32, 50.25_f32, @@ -1786,18 +1787,18 @@ mod tests { -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, -1.0_f32, ]; - let a1: __m512 = transmute(a_array); - let b1: __m512 = transmute(b_array); + let a1 = f32x16::from_array(a_array).as_m512(); + let b1 = f32x16::from_array(b_array).as_m512(); let k: __mmask16 = 0x3333; #[rustfmt::skip] - let src: __m512 = transmute([ + let src = f32x16::from_array([ 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, 1.0_f32, 2.0_f32, 3.0_f32, 4.0_f32, - ]); + ]).as_m512(); let a: __m512bh = _mm512_cvtne2ps_pbh(a1, a1); let b: __m512bh = _mm512_cvtne2ps_pbh(b1, b1); let c: __m512 = _mm512_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ -18.0_f32, -52.0_f32, 0.0, 0.0, -18.0_f32, -52.0_f32, 0.0, 0.0, -18.0_f32, -52.0_f32, @@ -1806,7 +1807,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0xffff; let c: __m512 = _mm512_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, -18.0_f32, -52.0_f32, -16.0_f32, -50.0_f32, @@ -1815,7 +1816,7 @@ mod tests { assert_eq!(result, expected_result); let k: __mmask16 = 0; let c: __m512 = _mm512_maskz_dpbf16_ps(k, src, a, b); - let result: [f32; 16] = transmute(c.as_f32x16()); + let result = *c.as_f32x16().as_array(); #[rustfmt::skip] let expected_result: [f32; 16] = [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, @@ -1943,28 +1944,28 @@ mod tests { } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_cvtneps_pbh() { + fn test_mm_cvtneps_pbh() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r: u16x4 = transmute_copy(&_mm_cvtneps_pbh(a)); + let r: u16x4 = unsafe { transmute_copy(&_mm_cvtneps_pbh(a)) }; let e = u16x4::new(BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR); assert_eq!(r, e); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_mask_cvtneps_pbh() { + fn test_mm_mask_cvtneps_pbh() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let src = __m128bh([5, 6, 7, 8, !0, !0, !0, !0]); let k = 0b1010; - let r: u16x4 = transmute_copy(&_mm_mask_cvtneps_pbh(src, k, a)); + let r: u16x4 = unsafe { transmute_copy(&_mm_mask_cvtneps_pbh(src, k, a)) }; let e = u16x4::new(5, BF16_TWO, 7, BF16_FOUR); assert_eq!(r, e); } #[simd_test(enable = "avx512bf16,avx512vl")] - unsafe fn test_mm_maskz_cvtneps_pbh() { + fn test_mm_maskz_cvtneps_pbh() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let k = 0b1010; - let r: u16x4 = transmute_copy(&_mm_maskz_cvtneps_pbh(k, a)); + let r: u16x4 = unsafe { transmute_copy(&_mm_maskz_cvtneps_pbh(k, a)) }; let e = u16x4::new(0, BF16_TWO, 0, BF16_FOUR); assert_eq!(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs index 6b4b88621efd..8e074fdcfa48 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -17098,37 +17098,37 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_loadu_epi16() { + const fn test_mm512_loadu_epi16() { #[rustfmt::skip] let a: [i16; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; - let r = _mm512_loadu_epi16(&a[0]); + let r = unsafe { _mm512_loadu_epi16(&a[0]) }; #[rustfmt::skip] let e = _mm512_set_epi16(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_loadu_epi16() { + const fn test_mm256_loadu_epi16() { let a: [i16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; - let r = _mm256_loadu_epi16(&a[0]); + let r = unsafe { _mm256_loadu_epi16(&a[0]) }; let e = _mm256_set_epi16(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_loadu_epi16() { + const fn test_mm_loadu_epi16() { let a: [i16; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; - let r = _mm_loadu_epi16(&a[0]); + let r = unsafe { _mm_loadu_epi16(&a[0]) }; let e = _mm_set_epi16(8, 7, 6, 5, 4, 3, 2, 1); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_loadu_epi8() { + const fn test_mm512_loadu_epi8() { #[rustfmt::skip] let a: [i8; 64] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; - let r = _mm512_loadu_epi8(&a[0]); + let r = unsafe { _mm512_loadu_epi8(&a[0]) }; #[rustfmt::skip] let e = _mm512_set_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); @@ -17136,73 +17136,85 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_loadu_epi8() { + const fn test_mm256_loadu_epi8() { #[rustfmt::skip] let a: [i8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; - let r = _mm256_loadu_epi8(&a[0]); + let r = unsafe { _mm256_loadu_epi8(&a[0]) }; #[rustfmt::skip] let e = _mm256_set_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_loadu_epi8() { + const fn test_mm_loadu_epi8() { let a: [i8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; - let r = _mm_loadu_epi8(&a[0]); + let r = unsafe { _mm_loadu_epi8(&a[0]) }; let e = _mm_set_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_storeu_epi16() { + const fn test_mm512_storeu_epi16() { let a = _mm512_set1_epi16(9); let mut r = _mm512_undefined_epi32(); - _mm512_storeu_epi16(&mut r as *mut _ as *mut i16, a); + unsafe { + _mm512_storeu_epi16(&mut r as *mut _ as *mut i16, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_storeu_epi16() { + const fn test_mm256_storeu_epi16() { let a = _mm256_set1_epi16(9); let mut r = _mm256_set1_epi32(0); - _mm256_storeu_epi16(&mut r as *mut _ as *mut i16, a); + unsafe { + _mm256_storeu_epi16(&mut r as *mut _ as *mut i16, a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_storeu_epi16() { + const fn test_mm_storeu_epi16() { let a = _mm_set1_epi16(9); let mut r = _mm_set1_epi32(0); - _mm_storeu_epi16(&mut r as *mut _ as *mut i16, a); + unsafe { + _mm_storeu_epi16(&mut r as *mut _ as *mut i16, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_storeu_epi8() { + const fn test_mm512_storeu_epi8() { let a = _mm512_set1_epi8(9); let mut r = _mm512_undefined_epi32(); - _mm512_storeu_epi8(&mut r as *mut _ as *mut i8, a); + unsafe { + _mm512_storeu_epi8(&mut r as *mut _ as *mut i8, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_storeu_epi8() { + const fn test_mm256_storeu_epi8() { let a = _mm256_set1_epi8(9); let mut r = _mm256_set1_epi32(0); - _mm256_storeu_epi8(&mut r as *mut _ as *mut i8, a); + unsafe { + _mm256_storeu_epi8(&mut r as *mut _ as *mut i8, a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_storeu_epi8() { + const fn test_mm_storeu_epi8() { let a = _mm_set1_epi8(9); let mut r = _mm_set1_epi32(0); - _mm_storeu_epi8(&mut r as *mut _ as *mut i8, a); + unsafe { + _mm_storeu_epi8(&mut r as *mut _ as *mut i8, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_mask_loadu_epi16() { + const fn test_mm512_mask_loadu_epi16() { let src = _mm512_set1_epi16(42); let a = &[ 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -17210,52 +17222,54 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b10101010_11001100_11101000_11001010; - let r = _mm512_mask_loadu_epi16(src, m, black_box(p)); + let r = unsafe { _mm512_mask_loadu_epi16(src, m, black_box(p)) }; let e = &[ 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, ]; - let e = _mm512_loadu_epi16(e.as_ptr()); + let e = unsafe { _mm512_loadu_epi16(e.as_ptr()) }; assert_eq_m512i(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_maskz_loadu_epi16() { + const fn test_mm512_maskz_loadu_epi16() { let a = &[ 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ]; let p = a.as_ptr(); let m = 0b10101010_11001100_11101000_11001010; - let r = _mm512_maskz_loadu_epi16(m, black_box(p)); + let r = unsafe { _mm512_maskz_loadu_epi16(m, black_box(p)) }; let e = &[ 0_i16, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16, 0, 0, 19, 20, 0, 0, 23, 24, 0, 26, 0, 28, 0, 30, 0, 32, ]; - let e = _mm512_loadu_epi16(e.as_ptr()); + let e = unsafe { _mm512_loadu_epi16(e.as_ptr()) }; assert_eq_m512i(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_mask_storeu_epi16() { + const fn test_mm512_mask_storeu_epi16() { let mut r = [42_i16; 32]; let a = &[ 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ]; - let a = _mm512_loadu_epi16(a.as_ptr()); + let a = unsafe { _mm512_loadu_epi16(a.as_ptr()) }; let m = 0b10101010_11001100_11101000_11001010; - _mm512_mask_storeu_epi16(r.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_storeu_epi16(r.as_mut_ptr(), m, a); + } let e = &[ 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, ]; - let e = _mm512_loadu_epi16(e.as_ptr()); - assert_eq_m512i(_mm512_loadu_epi16(r.as_ptr()), e); + let e = unsafe { _mm512_loadu_epi16(e.as_ptr()) }; + assert_eq_m512i(unsafe { _mm512_loadu_epi16(r.as_ptr()) }, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_mask_loadu_epi8() { + const fn test_mm512_mask_loadu_epi8() { let src = _mm512_set1_epi8(42); let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -17264,18 +17278,18 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b00000000_11111111_11111111_00000000_10101010_11001100_11101000_11001010; - let r = _mm512_mask_loadu_epi8(src, m, black_box(p)); + let r = unsafe { _mm512_mask_loadu_epi8(src, m, black_box(p)) }; let e = &[ 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, 42, 42, 42, 42, 42, 42, 42, 42, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 42, 42, 42, 42, 42, 42, 42, 42, ]; - let e = _mm512_loadu_epi8(e.as_ptr()); + let e = unsafe { _mm512_loadu_epi8(e.as_ptr()) }; assert_eq_m512i(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_maskz_loadu_epi8() { + const fn test_mm512_maskz_loadu_epi8() { let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, @@ -17283,77 +17297,81 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b00000000_11111111_11111111_00000000_10101010_11001100_11101000_11001010; - let r = _mm512_maskz_loadu_epi8(m, black_box(p)); + let r = unsafe { _mm512_maskz_loadu_epi8(m, black_box(p)) }; let e = &[ 0_i8, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16, 0, 0, 19, 20, 0, 0, 23, 24, 0, 26, 0, 28, 0, 30, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 0, 0, 0, 0, 0, 0, 0, 0, ]; - let e = _mm512_loadu_epi8(e.as_ptr()); + let e = unsafe { _mm512_loadu_epi8(e.as_ptr()) }; assert_eq_m512i(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_mm512_mask_storeu_epi8() { + const fn test_mm512_mask_storeu_epi8() { let mut r = [42_i8; 64]; let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, ]; - let a = _mm512_loadu_epi8(a.as_ptr()); + let a = unsafe { _mm512_loadu_epi8(a.as_ptr()) }; let m = 0b00000000_11111111_11111111_00000000_10101010_11001100_11101000_11001010; - _mm512_mask_storeu_epi8(r.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_storeu_epi8(r.as_mut_ptr(), m, a); + } let e = &[ 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, 42, 42, 42, 42, 42, 42, 42, 42, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 42, 42, 42, 42, 42, 42, 42, 42, ]; - let e = _mm512_loadu_epi8(e.as_ptr()); - assert_eq_m512i(_mm512_loadu_epi8(r.as_ptr()), e); + let e = unsafe { _mm512_loadu_epi8(e.as_ptr()) }; + assert_eq_m512i(unsafe { _mm512_loadu_epi8(r.as_ptr()) }, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_mask_loadu_epi16() { + const fn test_mm256_mask_loadu_epi16() { let src = _mm256_set1_epi16(42); let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm256_mask_loadu_epi16(src, m, black_box(p)); + let r = unsafe { _mm256_mask_loadu_epi16(src, m, black_box(p)) }; let e = &[ 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, ]; - let e = _mm256_loadu_epi16(e.as_ptr()); + let e = unsafe { _mm256_loadu_epi16(e.as_ptr()) }; assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_maskz_loadu_epi16() { + const fn test_mm256_maskz_loadu_epi16() { let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm256_maskz_loadu_epi16(m, black_box(p)); + let r = unsafe { _mm256_maskz_loadu_epi16(m, black_box(p)) }; let e = &[0_i16, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16]; - let e = _mm256_loadu_epi16(e.as_ptr()); + let e = unsafe { _mm256_loadu_epi16(e.as_ptr()) }; assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_mask_storeu_epi16() { + const fn test_mm256_mask_storeu_epi16() { let mut r = [42_i16; 16]; let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; - let a = _mm256_loadu_epi16(a.as_ptr()); + let a = unsafe { _mm256_loadu_epi16(a.as_ptr()) }; let m = 0b11101000_11001010; - _mm256_mask_storeu_epi16(r.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_storeu_epi16(r.as_mut_ptr(), m, a); + } let e = &[ 42_i16, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, ]; - let e = _mm256_loadu_epi16(e.as_ptr()); - assert_eq_m256i(_mm256_loadu_epi16(r.as_ptr()), e); + let e = unsafe { _mm256_loadu_epi16(e.as_ptr()) }; + assert_eq_m256i(unsafe { _mm256_loadu_epi16(r.as_ptr()) }, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_mask_loadu_epi8() { + const fn test_mm256_mask_loadu_epi8() { let src = _mm256_set1_epi8(42); let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -17361,122 +17379,124 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b10101010_11001100_11101000_11001010; - let r = _mm256_mask_loadu_epi8(src, m, black_box(p)); + let r = unsafe { _mm256_mask_loadu_epi8(src, m, black_box(p)) }; let e = &[ 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, ]; - let e = _mm256_loadu_epi8(e.as_ptr()); + let e = unsafe { _mm256_loadu_epi8(e.as_ptr()) }; assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_maskz_loadu_epi8() { + const fn test_mm256_maskz_loadu_epi8() { let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ]; let p = a.as_ptr(); let m = 0b10101010_11001100_11101000_11001010; - let r = _mm256_maskz_loadu_epi8(m, black_box(p)); + let r = unsafe { _mm256_maskz_loadu_epi8(m, black_box(p)) }; let e = &[ 0_i8, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16, 0, 0, 19, 20, 0, 0, 23, 24, 0, 26, 0, 28, 0, 30, 0, 32, ]; - let e = _mm256_loadu_epi8(e.as_ptr()); + let e = unsafe { _mm256_loadu_epi8(e.as_ptr()) }; assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm256_mask_storeu_epi8() { + const fn test_mm256_mask_storeu_epi8() { let mut r = [42_i8; 32]; let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ]; - let a = _mm256_loadu_epi8(a.as_ptr()); + let a = unsafe { _mm256_loadu_epi8(a.as_ptr()) }; let m = 0b10101010_11001100_11101000_11001010; - _mm256_mask_storeu_epi8(r.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_storeu_epi8(r.as_mut_ptr(), m, a); + } let e = &[ 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, 42, 42, 19, 20, 42, 42, 23, 24, 42, 26, 42, 28, 42, 30, 42, 32, ]; - let e = _mm256_loadu_epi8(e.as_ptr()); - assert_eq_m256i(_mm256_loadu_epi8(r.as_ptr()), e); + let e = unsafe { _mm256_loadu_epi8(e.as_ptr()) }; + assert_eq_m256i(unsafe { _mm256_loadu_epi8(r.as_ptr()) }, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_mask_loadu_epi16() { + const fn test_mm_mask_loadu_epi16() { let src = _mm_set1_epi16(42); let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm_mask_loadu_epi16(src, m, black_box(p)); + let r = unsafe { _mm_mask_loadu_epi16(src, m, black_box(p)) }; let e = &[42_i16, 2, 42, 4, 42, 42, 7, 8]; - let e = _mm_loadu_epi16(e.as_ptr()); + let e = unsafe { _mm_loadu_epi16(e.as_ptr()) }; assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_maskz_loadu_epi16() { + const fn test_mm_maskz_loadu_epi16() { let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm_maskz_loadu_epi16(m, black_box(p)); + let r = unsafe { _mm_maskz_loadu_epi16(m, black_box(p)) }; let e = &[0_i16, 2, 0, 4, 0, 0, 7, 8]; - let e = _mm_loadu_epi16(e.as_ptr()); + let e = unsafe { _mm_loadu_epi16(e.as_ptr()) }; assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_mask_storeu_epi16() { + const fn test_mm_mask_storeu_epi16() { let mut r = [42_i16; 8]; let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; - let a = _mm_loadu_epi16(a.as_ptr()); + let a = unsafe { _mm_loadu_epi16(a.as_ptr()) }; let m = 0b11001010; - _mm_mask_storeu_epi16(r.as_mut_ptr(), m, a); + unsafe { _mm_mask_storeu_epi16(r.as_mut_ptr(), m, a) }; let e = &[42_i16, 2, 42, 4, 42, 42, 7, 8]; - let e = _mm_loadu_epi16(e.as_ptr()); - assert_eq_m128i(_mm_loadu_epi16(r.as_ptr()), e); + let e = unsafe { _mm_loadu_epi16(e.as_ptr()) }; + assert_eq_m128i(unsafe { _mm_loadu_epi16(r.as_ptr()) }, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_mask_loadu_epi8() { + const fn test_mm_mask_loadu_epi8() { let src = _mm_set1_epi8(42); let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm_mask_loadu_epi8(src, m, black_box(p)); + let r = unsafe { _mm_mask_loadu_epi8(src, m, black_box(p)) }; let e = &[ 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, ]; - let e = _mm_loadu_epi8(e.as_ptr()); + let e = unsafe { _mm_loadu_epi8(e.as_ptr()) }; assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_maskz_loadu_epi8() { + const fn test_mm_maskz_loadu_epi8() { let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm_maskz_loadu_epi8(m, black_box(p)); + let r = unsafe { _mm_maskz_loadu_epi8(m, black_box(p)) }; let e = &[0_i8, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16]; - let e = _mm_loadu_epi8(e.as_ptr()); + let e = unsafe { _mm_loadu_epi8(e.as_ptr()) }; assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - const unsafe fn test_mm_mask_storeu_epi8() { + const fn test_mm_mask_storeu_epi8() { let mut r = [42_i8; 16]; let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; - let a = _mm_loadu_epi8(a.as_ptr()); + let a = unsafe { _mm_loadu_epi8(a.as_ptr()) }; let m = 0b11101000_11001010; - _mm_mask_storeu_epi8(r.as_mut_ptr(), m, a); + unsafe { _mm_mask_storeu_epi8(r.as_mut_ptr(), m, a) }; let e = &[ 42_i8, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16, ]; - let e = _mm_loadu_epi8(e.as_ptr()); - assert_eq_m128i(_mm_loadu_epi8(r.as_ptr()), e); + let e = unsafe { _mm_loadu_epi8(e.as_ptr()) }; + assert_eq_m128i(unsafe { _mm_loadu_epi8(r.as_ptr()) }, e); } #[simd_test(enable = "avx512bw")] @@ -20714,36 +20734,40 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const unsafe fn test_store_mask64() { + const fn test_store_mask64() { let a: __mmask64 = 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000; let mut r = 0; - _store_mask64(&mut r, a); + unsafe { + _store_mask64(&mut r, a); + } assert_eq!(r, a); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_store_mask32() { + const fn test_store_mask32() { let a: __mmask32 = 0b11111111_00000000_11111111_00000000; let mut r = 0; - _store_mask32(&mut r, a); + unsafe { + _store_mask32(&mut r, a); + } assert_eq!(r, a); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_load_mask64() { + const fn test_load_mask64() { let p: __mmask64 = 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000; - let r = _load_mask64(&p); + let r = unsafe { _load_mask64(&p) }; let e: __mmask64 = 0b11111111_00000000_11111111_00000000_11111111_00000000_11111111_00000000; assert_eq!(r, e); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_load_mask32() { + const fn test_load_mask32() { let p: __mmask32 = 0b11111111_00000000_11111111_00000000; - let r = _load_mask32(&p); + let r = unsafe { _load_mask32(&p) }; let e: __mmask32 = 0b11111111_00000000_11111111_00000000; assert_eq!(r, e); } @@ -21163,21 +21187,21 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const unsafe fn test_kortest_mask32_u8() { + const fn test_kortest_mask32_u8() { let a: __mmask32 = 0b0110100101101001_0110100101101001; let b: __mmask32 = 0b1011011010110110_1011011010110110; let mut all_ones: u8 = 0; - let r = _kortest_mask32_u8(a, b, &mut all_ones); + let r = unsafe { _kortest_mask32_u8(a, b, &mut all_ones) }; assert_eq!(r, 0); assert_eq!(all_ones, 1); } #[simd_test(enable = "avx512bw")] - const unsafe fn test_kortest_mask64_u8() { + const fn test_kortest_mask64_u8() { let a: __mmask64 = 0b0110100101101001_0110100101101001; let b: __mmask64 = 0b1011011010110110_1011011010110110; let mut all_ones: u8 = 0; - let r = _kortest_mask64_u8(a, b, &mut all_ones); + let r = unsafe { _kortest_mask64_u8(a, b, &mut all_ones) }; assert_eq!(r, 0); assert_eq!(all_ones, 0); } @@ -21299,11 +21323,11 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const unsafe fn test_ktest_mask32_u8() { + const fn test_ktest_mask32_u8() { let a: __mmask32 = 0b0110100100111100_0110100100111100; let b: __mmask32 = 0b1001011011000011_1001011011000011; let mut and_not: u8 = 0; - let r = _ktest_mask32_u8(a, b, &mut and_not); + let r = unsafe { _ktest_mask32_u8(a, b, &mut and_not) }; assert_eq!(r, 1); assert_eq!(and_not, 0); } @@ -21325,11 +21349,11 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const unsafe fn test_ktest_mask64_u8() { + const fn test_ktest_mask64_u8() { let a: __mmask64 = 0b0110100100111100_0110100100111100; let b: __mmask64 = 0b1001011011000011_1001011011000011; let mut and_not: u8 = 0; - let r = _ktest_mask64_u8(a, b, &mut and_not); + let r = unsafe { _ktest_mask64_u8(a, b, &mut and_not) }; assert_eq!(r, 1); assert_eq!(and_not, 0); } @@ -21951,32 +21975,38 @@ mod tests { } #[simd_test(enable = "avx512bw")] - unsafe fn test_mm512_mask_cvtsepi16_storeu_epi8() { + fn test_mm512_mask_cvtsepi16_storeu_epi8() { let a = _mm512_set1_epi16(i16::MAX); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtsepi16_storeu_epi8( - &mut r as *mut _ as *mut i8, - 0b11111111_11111111_11111111_11111111, - a, - ); + unsafe { + _mm512_mask_cvtsepi16_storeu_epi8( + &mut r as *mut _ as *mut i8, + 0b11111111_11111111_11111111_11111111, + a, + ); + } let e = _mm256_set1_epi8(i8::MAX); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - unsafe fn test_mm256_mask_cvtsepi16_storeu_epi8() { + fn test_mm256_mask_cvtsepi16_storeu_epi8() { let a = _mm256_set1_epi16(i16::MAX); let mut r = _mm_undefined_si128(); - _mm256_mask_cvtsepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + unsafe { + _mm256_mask_cvtsepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + } let e = _mm_set1_epi8(i8::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - unsafe fn test_mm_mask_cvtsepi16_storeu_epi8() { + fn test_mm_mask_cvtsepi16_storeu_epi8() { let a = _mm_set1_epi16(i16::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtsepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtsepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, 0, 0, 0, 0, @@ -21986,63 +22016,75 @@ mod tests { } #[simd_test(enable = "avx512bw")] - unsafe fn test_mm512_mask_cvtepi16_storeu_epi8() { + fn test_mm512_mask_cvtepi16_storeu_epi8() { let a = _mm512_set1_epi16(8); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtepi16_storeu_epi8( - &mut r as *mut _ as *mut i8, - 0b11111111_11111111_11111111_11111111, - a, - ); + unsafe { + _mm512_mask_cvtepi16_storeu_epi8( + &mut r as *mut _ as *mut i8, + 0b11111111_11111111_11111111_11111111, + a, + ); + } let e = _mm256_set1_epi8(8); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - unsafe fn test_mm256_mask_cvtepi16_storeu_epi8() { + fn test_mm256_mask_cvtepi16_storeu_epi8() { let a = _mm256_set1_epi16(8); let mut r = _mm_undefined_si128(); - _mm256_mask_cvtepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + unsafe { + _mm256_mask_cvtepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + } let e = _mm_set1_epi8(8); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - unsafe fn test_mm_mask_cvtepi16_storeu_epi8() { + fn test_mm_mask_cvtepi16_storeu_epi8() { let a = _mm_set1_epi16(8); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw")] - unsafe fn test_mm512_mask_cvtusepi16_storeu_epi8() { + fn test_mm512_mask_cvtusepi16_storeu_epi8() { let a = _mm512_set1_epi16(i16::MAX); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtusepi16_storeu_epi8( - &mut r as *mut _ as *mut i8, - 0b11111111_11111111_11111111_11111111, - a, - ); + unsafe { + _mm512_mask_cvtusepi16_storeu_epi8( + &mut r as *mut _ as *mut i8, + 0b11111111_11111111_11111111_11111111, + a, + ); + } let e = _mm256_set1_epi8(u8::MAX as i8); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - unsafe fn test_mm256_mask_cvtusepi16_storeu_epi8() { + fn test_mm256_mask_cvtusepi16_storeu_epi8() { let a = _mm256_set1_epi16(i16::MAX); let mut r = _mm_undefined_si128(); - _mm256_mask_cvtusepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + unsafe { + _mm256_mask_cvtusepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + } let e = _mm_set1_epi8(u8::MAX as i8); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512bw,avx512vl")] - unsafe fn test_mm_mask_cvtusepi16_storeu_epi8() { + fn test_mm_mask_cvtusepi16_storeu_epi8() { let a = _mm_set1_epi16(i16::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtusepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtusepi16_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, diff --git a/library/stdarch/crates/core_arch/src/x86/avx512dq.rs b/library/stdarch/crates/core_arch/src/x86/avx512dq.rs index ebe75cd22d8e..9e1a4c0b2955 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512dq.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512dq.rs @@ -7401,27 +7401,25 @@ unsafe extern "C" { mod tests { use super::*; use crate::core_arch::assert_eq_const as assert_eq; + use crate::core_arch::x86::*; use stdarch_test::simd_test; - use crate::core_arch::x86::*; - use crate::mem::transmute; + const OPRND1_64: f64 = f64::from_bits(0x3333333333333333); + const OPRND2_64: f64 = f64::from_bits(0x5555555555555555); - const OPRND1_64: f64 = unsafe { transmute(0x3333333333333333_u64) }; - const OPRND2_64: f64 = unsafe { transmute(0x5555555555555555_u64) }; + const AND_64: f64 = f64::from_bits(0x1111111111111111); + const ANDN_64: f64 = f64::from_bits(0x4444444444444444); + const OR_64: f64 = f64::from_bits(0x7777777777777777); + const XOR_64: f64 = f64::from_bits(0x6666666666666666); - const AND_64: f64 = unsafe { transmute(0x1111111111111111_u64) }; - const ANDN_64: f64 = unsafe { transmute(0x4444444444444444_u64) }; - const OR_64: f64 = unsafe { transmute(0x7777777777777777_u64) }; - const XOR_64: f64 = unsafe { transmute(0x6666666666666666_u64) }; + const OPRND1_32: f32 = f32::from_bits(0x33333333); + const OPRND2_32: f32 = f32::from_bits(0x55555555); - const OPRND1_32: f32 = unsafe { transmute(0x33333333_u32) }; - const OPRND2_32: f32 = unsafe { transmute(0x55555555_u32) }; - - const AND_32: f32 = unsafe { transmute(0x11111111_u32) }; - const ANDN_32: f32 = unsafe { transmute(0x44444444_u32) }; - const OR_32: f32 = unsafe { transmute(0x77777777_u32) }; - const XOR_32: f32 = unsafe { transmute(0x66666666_u32) }; + const AND_32: f32 = f32::from_bits(0x11111111); + const ANDN_32: f32 = f32::from_bits(0x44444444); + const OR_32: f32 = f32::from_bits(0x77777777); + const XOR_32: f32 = f32::from_bits(0x66666666); #[simd_test(enable = "avx512dq,avx512vl")] const fn test_mm_mask_and_pd() { @@ -10023,11 +10021,11 @@ mod tests { } #[simd_test(enable = "avx512dq")] - const unsafe fn test_kortest_mask8_u8() { + const fn test_kortest_mask8_u8() { let a: __mmask8 = 0b01101001; let b: __mmask8 = 0b10110110; let mut all_ones: u8 = 0; - let r = _kortest_mask8_u8(a, b, &mut all_ones); + let r = unsafe { _kortest_mask8_u8(a, b, &mut all_ones) }; assert_eq!(r, 0); assert_eq!(all_ones, 1); } @@ -10049,7 +10047,7 @@ mod tests { } #[simd_test(enable = "avx512dq")] - const unsafe fn test_kshiftli_mask8() { + const fn test_kshiftli_mask8() { let a: __mmask8 = 0b01101001; let r = _kshiftli_mask8::<3>(a); let e: __mmask8 = 0b01001000; @@ -10089,11 +10087,11 @@ mod tests { } #[simd_test(enable = "avx512dq")] - const unsafe fn test_ktest_mask8_u8() { + const fn test_ktest_mask8_u8() { let a: __mmask8 = 0b01101001; let b: __mmask8 = 0b10010110; let mut and_not: u8 = 0; - let r = _ktest_mask8_u8(a, b, &mut and_not); + let r = unsafe { _ktest_mask8_u8(a, b, &mut and_not) }; assert_eq!(r, 1); assert_eq!(and_not, 0); } @@ -10115,11 +10113,11 @@ mod tests { } #[simd_test(enable = "avx512dq")] - const unsafe fn test_ktest_mask16_u8() { + const fn test_ktest_mask16_u8() { let a: __mmask16 = 0b0110100100111100; let b: __mmask16 = 0b1001011011000011; let mut and_not: u8 = 0; - let r = _ktest_mask16_u8(a, b, &mut and_not); + let r = unsafe { _ktest_mask16_u8(a, b, &mut and_not) }; assert_eq!(r, 1); assert_eq!(and_not, 0); } @@ -10141,18 +10139,20 @@ mod tests { } #[simd_test(enable = "avx512dq")] - const unsafe fn test_load_mask8() { + const fn test_load_mask8() { let a: __mmask8 = 0b01101001; - let r = _load_mask8(&a); + let r = unsafe { _load_mask8(&a) }; let e: __mmask8 = 0b01101001; assert_eq!(r, e); } #[simd_test(enable = "avx512dq")] - const unsafe fn test_store_mask8() { + const fn test_store_mask8() { let a: __mmask8 = 0b01101001; let mut r = 0; - _store_mask8(&mut r, a); + unsafe { + _store_mask8(&mut r, a); + } let e: __mmask8 = 0b01101001; assert_eq!(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index 76b753938383..3730496e1ec3 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -48062,9 +48062,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_ternarylogic_epi32() { - use core::intrinsics::simd::simd_xor; - + fn test_mm512_ternarylogic_epi32() { let a = _mm512_set4_epi32(0b100, 0b110, 0b001, 0b101); let b = _mm512_set4_epi32(0b010, 0b011, 0b001, 0b110); let c = _mm512_set4_epi32(0b001, 0b000, 0b001, 0b111); @@ -48077,7 +48075,7 @@ mod tests { let r = _mm512_ternarylogic_epi32::<0b10010110>(a, b, c); let e = _mm512_set4_epi32(0b111, 0b101, 0b001, 0b100); assert_eq_m512i(r, e); - assert_eq_m512i(r, simd_xor(simd_xor(a, b), c)); + assert_eq_m512i(r, _mm512_xor_si512(_mm512_xor_si512(a, b), c)); // Majority (2 or more bits set). let r = _mm512_ternarylogic_epi32::<0b1110_1000>(a, b, c); @@ -48110,9 +48108,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_ternarylogic_epi32() { - use core::intrinsics::simd::simd_xor; - + fn test_mm256_ternarylogic_epi32() { let _mm256_set4_epi32 = |a, b, c, d| _mm256_setr_epi32(a, b, c, d, a, b, c, d); let a = _mm256_set4_epi32(0b100, 0b110, 0b001, 0b101); @@ -48127,7 +48123,7 @@ mod tests { let r = _mm256_ternarylogic_epi32::<0b10010110>(a, b, c); let e = _mm256_set4_epi32(0b111, 0b101, 0b001, 0b100); assert_eq_m256i(r, e); - assert_eq_m256i(r, simd_xor(simd_xor(a, b), c)); + assert_eq_m256i(r, _mm256_xor_si256(_mm256_xor_si256(a, b), c)); // Majority (2 or more bits set). let r = _mm256_ternarylogic_epi32::<0b1110_1000>(a, b, c); @@ -48160,9 +48156,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_ternarylogic_epi32() { - use core::intrinsics::simd::simd_xor; - + fn test_mm_ternarylogic_epi32() { let a = _mm_setr_epi32(0b100, 0b110, 0b001, 0b101); let b = _mm_setr_epi32(0b010, 0b011, 0b001, 0b110); let c = _mm_setr_epi32(0b001, 0b000, 0b001, 0b111); @@ -48175,7 +48169,7 @@ mod tests { let r = _mm_ternarylogic_epi32::<0b10010110>(a, b, c); let e = _mm_setr_epi32(0b111, 0b101, 0b001, 0b100); assert_eq_m128i(r, e); - assert_eq_m128i(r, simd_xor(simd_xor(a, b), c)); + assert_eq_m128i(r, _mm_xor_si128(_mm_xor_si128(a, b), c)); // Majority (2 or more bits set). let r = _mm_ternarylogic_epi32::<0b1110_1000>(a, b, c); @@ -51447,20 +51441,20 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32gather_ps() { + fn test_mm512_i32gather_ps() { let arr: [f32; 256] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing #[rustfmt::skip] let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176); - let r = _mm512_i32gather_ps::<4>(index, arr.as_ptr()); + let r = unsafe { _mm512_i32gather_ps::<4>(index, arr.as_ptr()) }; #[rustfmt::skip] assert_eq_m512(r, _mm512_setr_ps(0., 16., 32., 48., 64., 80., 96., 112., 120., 128., 136., 144., 152., 160., 168., 176.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32gather_ps() { + fn test_mm512_mask_i32gather_ps() { let arr: [f32; 256] = core::array::from_fn(|i| i as f32); let src = _mm512_set1_ps(2.); let mask = 0b10101010_10101010; @@ -51468,27 +51462,27 @@ mod tests { let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176); // A multiplier of 4 is word-addressing - let r = _mm512_mask_i32gather_ps::<4>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i32gather_ps::<4>(src, mask, index, arr.as_ptr()) }; #[rustfmt::skip] assert_eq_m512(r, _mm512_setr_ps(2., 16., 2., 48., 2., 80., 2., 112., 2., 128., 2., 144., 2., 160., 2., 176.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32gather_epi32() { + fn test_mm512_i32gather_epi32() { let arr: [i32; 256] = core::array::from_fn(|i| i as i32); // A multiplier of 4 is word-addressing #[rustfmt::skip] let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176); - let r = _mm512_i32gather_epi32::<4>(index, arr.as_ptr()); + let r = unsafe { _mm512_i32gather_epi32::<4>(index, arr.as_ptr()) }; #[rustfmt::skip] assert_eq_m512i(r, _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32gather_epi32() { + fn test_mm512_mask_i32gather_epi32() { let arr: [i32; 256] = core::array::from_fn(|i| i as i32); let src = _mm512_set1_epi32(2); let mask = 0b10101010_10101010; @@ -51496,7 +51490,7 @@ mod tests { 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, ); // A multiplier of 4 is word-addressing - let r = _mm512_mask_i32gather_epi32::<4>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i32gather_epi32::<4>(src, mask, index, arr.as_ptr()) }; assert_eq_m512i( r, _mm512_setr_epi32(2, 16, 2, 48, 2, 80, 2, 112, 2, 144, 2, 176, 2, 208, 2, 240), @@ -51504,7 +51498,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32scatter_ps() { + fn test_mm512_i32scatter_ps() { let mut arr = [0f32; 256]; #[rustfmt::skip] let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, @@ -51513,7 +51507,9 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); // A multiplier of 4 is word-addressing - _mm512_i32scatter_ps::<4>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i32scatter_ps::<4>(arr.as_mut_ptr(), index, src); + } let mut expected = [0f32; 256]; for i in 0..16 { expected[i * 16] = (i + 1) as f32; @@ -51522,7 +51518,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32scatter_ps() { + fn test_mm512_mask_i32scatter_ps() { let mut arr = [0f32; 256]; let mask = 0b10101010_10101010; #[rustfmt::skip] @@ -51532,7 +51528,9 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); // A multiplier of 4 is word-addressing - _mm512_mask_i32scatter_ps::<4>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i32scatter_ps::<4>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0f32; 256]; for i in 0..8 { expected[i * 32 + 16] = 2. * (i + 1) as f32; @@ -51541,7 +51539,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32scatter_epi32() { + fn test_mm512_i32scatter_epi32() { let mut arr = [0i32; 256]; #[rustfmt::skip] @@ -51549,7 +51547,9 @@ mod tests { 128, 144, 160, 176, 192, 208, 224, 240); let src = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // A multiplier of 4 is word-addressing - _mm512_i32scatter_epi32::<4>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i32scatter_epi32::<4>(arr.as_mut_ptr(), index, src); + } let mut expected = [0i32; 256]; for i in 0..16 { expected[i * 16] = (i + 1) as i32; @@ -51558,7 +51558,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32scatter_epi32() { + fn test_mm512_mask_i32scatter_epi32() { let mut arr = [0i32; 256]; let mask = 0b10101010_10101010; #[rustfmt::skip] @@ -51566,7 +51566,9 @@ mod tests { 128, 144, 160, 176, 192, 208, 224, 240); let src = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // A multiplier of 4 is word-addressing - _mm512_mask_i32scatter_epi32::<4>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i32scatter_epi32::<4>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0i32; 256]; for i in 0..8 { expected[i * 32 + 16] = 2 * (i + 1) as i32; @@ -52850,29 +52852,31 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_loadu_pd() { + const fn test_mm512_loadu_pd() { let a = &[4., 3., 2., 5., 8., 9., 64., 50.]; let p = a.as_ptr(); - let r = _mm512_loadu_pd(black_box(p)); + let r = unsafe { _mm512_loadu_pd(black_box(p)) }; let e = _mm512_setr_pd(4., 3., 2., 5., 8., 9., 64., 50.); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_storeu_pd() { + const fn test_mm512_storeu_pd() { let a = _mm512_set1_pd(9.); let mut r = _mm512_undefined_pd(); - _mm512_storeu_pd(&mut r as *mut _ as *mut f64, a); + unsafe { + _mm512_storeu_pd(&mut r as *mut _ as *mut f64, a); + } assert_eq_m512d(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_loadu_ps() { + const fn test_mm512_loadu_ps() { let a = &[ 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., ]; let p = a.as_ptr(); - let r = _mm512_loadu_ps(black_box(p)); + let r = unsafe { _mm512_loadu_ps(black_box(p)) }; let e = _mm512_setr_ps( 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., ); @@ -52880,36 +52884,38 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_storeu_ps() { + const fn test_mm512_storeu_ps() { let a = _mm512_set1_ps(9.); let mut r = _mm512_undefined_ps(); - _mm512_storeu_ps(&mut r as *mut _ as *mut f32, a); + unsafe { + _mm512_storeu_ps(&mut r as *mut _ as *mut f32, a); + } assert_eq_m512(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_loadu_epi32() { + const fn test_mm512_mask_loadu_epi32() { let src = _mm512_set1_epi32(42); let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_mask_loadu_epi32(src, m, black_box(p)); + let r = unsafe { _mm512_mask_loadu_epi32(src, m, black_box(p)) }; let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_loadu_epi32() { + const fn test_mm512_maskz_loadu_epi32() { let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_maskz_loadu_epi32(m, black_box(p)); + let r = unsafe { _mm512_maskz_loadu_epi32(m, black_box(p)) }; let e = _mm512_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_load_epi32() { + const fn test_mm512_mask_load_epi32() { #[repr(align(64))] struct Align { data: [i32; 16], // 64 bytes @@ -52920,13 +52926,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_mask_load_epi32(src, m, black_box(p)); + let r = unsafe { _mm512_mask_load_epi32(src, m, black_box(p)) }; let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_load_epi32() { + const fn test_mm512_maskz_load_epi32() { #[repr(align(64))] struct Align { data: [i32; 16], // 64 bytes @@ -52936,23 +52942,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_maskz_load_epi32(m, black_box(p)); + let r = unsafe { _mm512_maskz_load_epi32(m, black_box(p)) }; let e = _mm512_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8, 0, 0, 0, 12, 0, 14, 15, 16); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_storeu_epi32() { + const fn test_mm512_mask_storeu_epi32() { let mut r = [42_i32; 16]; let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let m = 0b11101000_11001010; - _mm512_mask_storeu_epi32(r.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_storeu_epi32(r.as_mut_ptr(), m, a); + } let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); - assert_eq_m512i(_mm512_loadu_epi32(r.as_ptr()), e); + assert_eq_m512i(unsafe { _mm512_loadu_epi32(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_store_epi32() { + const fn test_mm512_mask_store_epi32() { #[repr(align(64))] struct Align { data: [i32; 16], @@ -52960,34 +52968,36 @@ mod tests { let mut r = Align { data: [42; 16] }; let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let m = 0b11101000_11001010; - _mm512_mask_store_epi32(r.data.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_store_epi32(r.data.as_mut_ptr(), m, a); + } let e = _mm512_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8, 42, 42, 42, 12, 42, 14, 15, 16); - assert_eq_m512i(_mm512_load_epi32(r.data.as_ptr()), e); + assert_eq_m512i(unsafe { _mm512_load_epi32(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_loadu_epi64() { + const fn test_mm512_mask_loadu_epi64() { let src = _mm512_set1_epi64(42); let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm512_mask_loadu_epi64(src, m, black_box(p)); + let r = unsafe { _mm512_mask_loadu_epi64(src, m, black_box(p)) }; let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_loadu_epi64() { + const fn test_mm512_maskz_loadu_epi64() { let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm512_maskz_loadu_epi64(m, black_box(p)); + let r = unsafe { _mm512_maskz_loadu_epi64(m, black_box(p)) }; let e = _mm512_setr_epi64(0, 2, 0, 4, 0, 0, 7, 8); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_load_epi64() { + const fn test_mm512_mask_load_epi64() { #[repr(align(64))] struct Align { data: [i64; 8], // 64 bytes @@ -52998,13 +53008,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm512_mask_load_epi64(src, m, black_box(p)); + let r = unsafe { _mm512_mask_load_epi64(src, m, black_box(p)) }; let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_load_epi64() { + const fn test_mm512_maskz_load_epi64() { #[repr(align(64))] struct Align { data: [i64; 8], // 64 bytes @@ -53014,23 +53024,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm512_maskz_load_epi64(m, black_box(p)); + let r = unsafe { _mm512_maskz_load_epi64(m, black_box(p)) }; let e = _mm512_setr_epi64(0, 2, 0, 4, 0, 0, 7, 8); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_storeu_epi64() { + const fn test_mm512_mask_storeu_epi64() { let mut r = [42_i64; 8]; let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let m = 0b11001010; - _mm512_mask_storeu_epi64(r.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_storeu_epi64(r.as_mut_ptr(), m, a); + } let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); - assert_eq_m512i(_mm512_loadu_epi64(r.as_ptr()), e); + assert_eq_m512i(unsafe { _mm512_loadu_epi64(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_store_epi64() { + const fn test_mm512_mask_store_epi64() { #[repr(align(64))] struct Align { data: [i64; 8], @@ -53039,13 +53051,15 @@ mod tests { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let m = 0b11001010; let p = r.data.as_mut_ptr(); - _mm512_mask_store_epi64(p, m, a); + unsafe { + _mm512_mask_store_epi64(p, m, a); + } let e = _mm512_setr_epi64(42, 2, 42, 4, 42, 42, 7, 8); - assert_eq_m512i(_mm512_load_epi64(r.data.as_ptr()), e); + assert_eq_m512i(unsafe { _mm512_load_epi64(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_loadu_ps() { + const fn test_mm512_mask_loadu_ps() { let src = _mm512_set1_ps(42.0); let a = &[ 1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, @@ -53053,7 +53067,7 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_mask_loadu_ps(src, m, black_box(p)); + let r = unsafe { _mm512_mask_loadu_ps(src, m, black_box(p)) }; let e = _mm512_setr_ps( 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, 16.0, @@ -53062,14 +53076,14 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_loadu_ps() { + const fn test_mm512_maskz_loadu_ps() { let a = &[ 1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_maskz_loadu_ps(m, black_box(p)); + let r = unsafe { _mm512_maskz_loadu_ps(m, black_box(p)) }; let e = _mm512_setr_ps( 0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0, 0.0, 0.0, 0.0, 12.0, 0.0, 14.0, 15.0, 16.0, ); @@ -53077,7 +53091,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_load_ps() { + const fn test_mm512_mask_load_ps() { #[repr(align(64))] struct Align { data: [f32; 16], // 64 bytes @@ -53091,7 +53105,7 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_mask_load_ps(src, m, black_box(p)); + let r = unsafe { _mm512_mask_load_ps(src, m, black_box(p)) }; let e = _mm512_setr_ps( 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, 16.0, @@ -53100,7 +53114,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_load_ps() { + const fn test_mm512_maskz_load_ps() { #[repr(align(64))] struct Align { data: [f32; 16], // 64 bytes @@ -53113,7 +53127,7 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_maskz_load_ps(m, black_box(p)); + let r = unsafe { _mm512_maskz_load_ps(m, black_box(p)) }; let e = _mm512_setr_ps( 0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0, 0.0, 0.0, 0.0, 12.0, 0.0, 14.0, 15.0, 16.0, ); @@ -53121,22 +53135,24 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_storeu_ps() { + const fn test_mm512_mask_storeu_ps() { let mut r = [42_f32; 16]; let a = _mm512_setr_ps( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ); let m = 0b11101000_11001010; - _mm512_mask_storeu_ps(r.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_storeu_ps(r.as_mut_ptr(), m, a); + } let e = _mm512_setr_ps( 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, 16.0, ); - assert_eq_m512(_mm512_loadu_ps(r.as_ptr()), e); + assert_eq_m512(unsafe { _mm512_loadu_ps(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_store_ps() { + const fn test_mm512_mask_store_ps() { #[repr(align(64))] struct Align { data: [f32; 16], @@ -53146,37 +53162,39 @@ mod tests { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ); let m = 0b11101000_11001010; - _mm512_mask_store_ps(r.data.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_store_ps(r.data.as_mut_ptr(), m, a); + } let e = _mm512_setr_ps( 42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0, 42.0, 42.0, 42.0, 12.0, 42.0, 14.0, 15.0, 16.0, ); - assert_eq_m512(_mm512_load_ps(r.data.as_ptr()), e); + assert_eq_m512(unsafe { _mm512_load_ps(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_loadu_pd() { + const fn test_mm512_mask_loadu_pd() { let src = _mm512_set1_pd(42.0); let a = &[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm512_mask_loadu_pd(src, m, black_box(p)); + let r = unsafe { _mm512_mask_loadu_pd(src, m, black_box(p)) }; let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_loadu_pd() { + const fn test_mm512_maskz_loadu_pd() { let a = &[1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm512_maskz_loadu_pd(m, black_box(p)); + let r = unsafe { _mm512_maskz_loadu_pd(m, black_box(p)) }; let e = _mm512_setr_pd(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_load_pd() { + const fn test_mm512_mask_load_pd() { #[repr(align(64))] struct Align { data: [f64; 8], // 64 bytes @@ -53187,13 +53205,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm512_mask_load_pd(src, m, black_box(p)); + let r = unsafe { _mm512_mask_load_pd(src, m, black_box(p)) }; let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_maskz_load_pd() { + const fn test_mm512_maskz_load_pd() { #[repr(align(64))] struct Align { data: [f64; 8], // 64 bytes @@ -53203,23 +53221,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm512_maskz_load_pd(m, black_box(p)); + let r = unsafe { _mm512_maskz_load_pd(m, black_box(p)) }; let e = _mm512_setr_pd(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_storeu_pd() { + const fn test_mm512_mask_storeu_pd() { let mut r = [42_f64; 8]; let a = _mm512_setr_pd(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); let m = 0b11001010; - _mm512_mask_storeu_pd(r.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_storeu_pd(r.as_mut_ptr(), m, a); + } let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); - assert_eq_m512d(_mm512_loadu_pd(r.as_ptr()), e); + assert_eq_m512d(unsafe { _mm512_loadu_pd(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_mask_store_pd() { + const fn test_mm512_mask_store_pd() { #[repr(align(64))] struct Align { data: [f64; 8], @@ -53227,34 +53247,36 @@ mod tests { let mut r = Align { data: [42.0; 8] }; let a = _mm512_setr_pd(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); let m = 0b11001010; - _mm512_mask_store_pd(r.data.as_mut_ptr(), m, a); + unsafe { + _mm512_mask_store_pd(r.data.as_mut_ptr(), m, a); + } let e = _mm512_setr_pd(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); - assert_eq_m512d(_mm512_load_pd(r.data.as_ptr()), e); + assert_eq_m512d(unsafe { _mm512_load_pd(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_loadu_epi32() { + const fn test_mm256_mask_loadu_epi32() { let src = _mm256_set1_epi32(42); let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm256_mask_loadu_epi32(src, m, black_box(p)); + let r = unsafe { _mm256_mask_loadu_epi32(src, m, black_box(p)) }; let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_loadu_epi32() { + const fn test_mm256_maskz_loadu_epi32() { let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm256_maskz_loadu_epi32(m, black_box(p)); + let r = unsafe { _mm256_maskz_loadu_epi32(m, black_box(p)) }; let e = _mm256_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_load_epi32() { + const fn test_mm256_mask_load_epi32() { #[repr(align(32))] struct Align { data: [i32; 8], // 32 bytes @@ -53265,13 +53287,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm256_mask_load_epi32(src, m, black_box(p)); + let r = unsafe { _mm256_mask_load_epi32(src, m, black_box(p)) }; let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_load_epi32() { + const fn test_mm256_maskz_load_epi32() { #[repr(align(32))] struct Align { data: [i32; 8], // 32 bytes @@ -53281,23 +53303,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm256_maskz_load_epi32(m, black_box(p)); + let r = unsafe { _mm256_maskz_load_epi32(m, black_box(p)) }; let e = _mm256_setr_epi32(0, 2, 0, 4, 0, 0, 7, 8); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_storeu_epi32() { + const fn test_mm256_mask_storeu_epi32() { let mut r = [42_i32; 8]; let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); let m = 0b11001010; - _mm256_mask_storeu_epi32(r.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_storeu_epi32(r.as_mut_ptr(), m, a); + } let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); - assert_eq_m256i(_mm256_loadu_epi32(r.as_ptr()), e); + assert_eq_m256i(unsafe { _mm256_loadu_epi32(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_store_epi32() { + const fn test_mm256_mask_store_epi32() { #[repr(align(64))] struct Align { data: [i32; 8], @@ -53305,34 +53329,36 @@ mod tests { let mut r = Align { data: [42; 8] }; let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); let m = 0b11001010; - _mm256_mask_store_epi32(r.data.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_store_epi32(r.data.as_mut_ptr(), m, a); + } let e = _mm256_setr_epi32(42, 2, 42, 4, 42, 42, 7, 8); - assert_eq_m256i(_mm256_load_epi32(r.data.as_ptr()), e); + assert_eq_m256i(unsafe { _mm256_load_epi32(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_loadu_epi64() { + const fn test_mm256_mask_loadu_epi64() { let src = _mm256_set1_epi64x(42); let a = &[1_i64, 2, 3, 4]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm256_mask_loadu_epi64(src, m, black_box(p)); + let r = unsafe { _mm256_mask_loadu_epi64(src, m, black_box(p)) }; let e = _mm256_setr_epi64x(42, 2, 42, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_loadu_epi64() { + const fn test_mm256_maskz_loadu_epi64() { let a = &[1_i64, 2, 3, 4]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm256_maskz_loadu_epi64(m, black_box(p)); + let r = unsafe { _mm256_maskz_loadu_epi64(m, black_box(p)) }; let e = _mm256_setr_epi64x(0, 2, 0, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_load_epi64() { + const fn test_mm256_mask_load_epi64() { #[repr(align(32))] struct Align { data: [i64; 4], // 32 bytes @@ -53343,13 +53369,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm256_mask_load_epi64(src, m, black_box(p)); + let r = unsafe { _mm256_mask_load_epi64(src, m, black_box(p)) }; let e = _mm256_setr_epi64x(42, 2, 42, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_load_epi64() { + const fn test_mm256_maskz_load_epi64() { #[repr(align(32))] struct Align { data: [i64; 4], // 32 bytes @@ -53359,23 +53385,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm256_maskz_load_epi64(m, black_box(p)); + let r = unsafe { _mm256_maskz_load_epi64(m, black_box(p)) }; let e = _mm256_setr_epi64x(0, 2, 0, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_storeu_epi64() { + const fn test_mm256_mask_storeu_epi64() { let mut r = [42_i64; 4]; let a = _mm256_setr_epi64x(1, 2, 3, 4); let m = 0b1010; - _mm256_mask_storeu_epi64(r.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_storeu_epi64(r.as_mut_ptr(), m, a); + } let e = _mm256_setr_epi64x(42, 2, 42, 4); - assert_eq_m256i(_mm256_loadu_epi64(r.as_ptr()), e); + assert_eq_m256i(unsafe { _mm256_loadu_epi64(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_store_epi64() { + const fn test_mm256_mask_store_epi64() { #[repr(align(32))] struct Align { data: [i64; 4], @@ -53383,34 +53411,36 @@ mod tests { let mut r = Align { data: [42; 4] }; let a = _mm256_setr_epi64x(1, 2, 3, 4); let m = 0b1010; - _mm256_mask_store_epi64(r.data.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_store_epi64(r.data.as_mut_ptr(), m, a); + } let e = _mm256_setr_epi64x(42, 2, 42, 4); - assert_eq_m256i(_mm256_load_epi64(r.data.as_ptr()), e); + assert_eq_m256i(unsafe { _mm256_load_epi64(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_loadu_ps() { + const fn test_mm256_mask_loadu_ps() { let src = _mm256_set1_ps(42.0); let a = &[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm256_mask_loadu_ps(src, m, black_box(p)); + let r = unsafe { _mm256_mask_loadu_ps(src, m, black_box(p)) }; let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); assert_eq_m256(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_loadu_ps() { + const fn test_mm256_maskz_loadu_ps() { let a = &[1.0_f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let p = a.as_ptr(); let m = 0b11001010; - let r = _mm256_maskz_loadu_ps(m, black_box(p)); + let r = unsafe { _mm256_maskz_loadu_ps(m, black_box(p)) }; let e = _mm256_setr_ps(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); assert_eq_m256(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_load_ps() { + const fn test_mm256_mask_load_ps() { #[repr(align(32))] struct Align { data: [f32; 8], // 32 bytes @@ -53421,13 +53451,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm256_mask_load_ps(src, m, black_box(p)); + let r = unsafe { _mm256_mask_load_ps(src, m, black_box(p)) }; let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); assert_eq_m256(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_load_ps() { + const fn test_mm256_maskz_load_ps() { #[repr(align(32))] struct Align { data: [f32; 8], // 32 bytes @@ -53437,23 +53467,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b11001010; - let r = _mm256_maskz_load_ps(m, black_box(p)); + let r = unsafe { _mm256_maskz_load_ps(m, black_box(p)) }; let e = _mm256_setr_ps(0.0, 2.0, 0.0, 4.0, 0.0, 0.0, 7.0, 8.0); assert_eq_m256(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_storeu_ps() { + const fn test_mm256_mask_storeu_ps() { let mut r = [42_f32; 8]; let a = _mm256_setr_ps(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); let m = 0b11001010; - _mm256_mask_storeu_ps(r.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_storeu_ps(r.as_mut_ptr(), m, a); + } let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); - assert_eq_m256(_mm256_loadu_ps(r.as_ptr()), e); + assert_eq_m256(unsafe { _mm256_loadu_ps(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_store_ps() { + const fn test_mm256_mask_store_ps() { #[repr(align(32))] struct Align { data: [f32; 8], @@ -53461,34 +53493,36 @@ mod tests { let mut r = Align { data: [42.0; 8] }; let a = _mm256_setr_ps(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); let m = 0b11001010; - _mm256_mask_store_ps(r.data.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_store_ps(r.data.as_mut_ptr(), m, a); + } let e = _mm256_setr_ps(42.0, 2.0, 42.0, 4.0, 42.0, 42.0, 7.0, 8.0); - assert_eq_m256(_mm256_load_ps(r.data.as_ptr()), e); + assert_eq_m256(unsafe { _mm256_load_ps(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_loadu_pd() { + const fn test_mm256_mask_loadu_pd() { let src = _mm256_set1_pd(42.0); let a = &[1.0_f64, 2.0, 3.0, 4.0]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm256_mask_loadu_pd(src, m, black_box(p)); + let r = unsafe { _mm256_mask_loadu_pd(src, m, black_box(p)) }; let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); assert_eq_m256d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_loadu_pd() { + const fn test_mm256_maskz_loadu_pd() { let a = &[1.0_f64, 2.0, 3.0, 4.0]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm256_maskz_loadu_pd(m, black_box(p)); + let r = unsafe { _mm256_maskz_loadu_pd(m, black_box(p)) }; let e = _mm256_setr_pd(0.0, 2.0, 0.0, 4.0); assert_eq_m256d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_load_pd() { + const fn test_mm256_mask_load_pd() { #[repr(align(32))] struct Align { data: [f64; 4], // 32 bytes @@ -53499,13 +53533,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm256_mask_load_pd(src, m, black_box(p)); + let r = unsafe { _mm256_mask_load_pd(src, m, black_box(p)) }; let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); assert_eq_m256d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_maskz_load_pd() { + const fn test_mm256_maskz_load_pd() { #[repr(align(32))] struct Align { data: [f64; 4], // 32 bytes @@ -53515,23 +53549,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm256_maskz_load_pd(m, black_box(p)); + let r = unsafe { _mm256_maskz_load_pd(m, black_box(p)) }; let e = _mm256_setr_pd(0.0, 2.0, 0.0, 4.0); assert_eq_m256d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_storeu_pd() { + const fn test_mm256_mask_storeu_pd() { let mut r = [42_f64; 4]; let a = _mm256_setr_pd(1.0, 2.0, 3.0, 4.0); let m = 0b1010; - _mm256_mask_storeu_pd(r.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_storeu_pd(r.as_mut_ptr(), m, a); + } let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); - assert_eq_m256d(_mm256_loadu_pd(r.as_ptr()), e); + assert_eq_m256d(unsafe { _mm256_loadu_pd(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_mask_store_pd() { + const fn test_mm256_mask_store_pd() { #[repr(align(32))] struct Align { data: [f64; 4], @@ -53539,34 +53575,36 @@ mod tests { let mut r = Align { data: [42.0; 4] }; let a = _mm256_setr_pd(1.0, 2.0, 3.0, 4.0); let m = 0b1010; - _mm256_mask_store_pd(r.data.as_mut_ptr(), m, a); + unsafe { + _mm256_mask_store_pd(r.data.as_mut_ptr(), m, a); + } let e = _mm256_setr_pd(42.0, 2.0, 42.0, 4.0); - assert_eq_m256d(_mm256_load_pd(r.data.as_ptr()), e); + assert_eq_m256d(unsafe { _mm256_load_pd(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_loadu_epi32() { + const fn test_mm_mask_loadu_epi32() { let src = _mm_set1_epi32(42); let a = &[1_i32, 2, 3, 4]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm_mask_loadu_epi32(src, m, black_box(p)); + let r = unsafe { _mm_mask_loadu_epi32(src, m, black_box(p)) }; let e = _mm_setr_epi32(42, 2, 42, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_loadu_epi32() { + const fn test_mm_maskz_loadu_epi32() { let a = &[1_i32, 2, 3, 4]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm_maskz_loadu_epi32(m, black_box(p)); + let r = unsafe { _mm_maskz_loadu_epi32(m, black_box(p)) }; let e = _mm_setr_epi32(0, 2, 0, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_load_epi32() { + const fn test_mm_mask_load_epi32() { #[repr(align(16))] struct Align { data: [i32; 4], // 32 bytes @@ -53577,13 +53615,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm_mask_load_epi32(src, m, black_box(p)); + let r = unsafe { _mm_mask_load_epi32(src, m, black_box(p)) }; let e = _mm_setr_epi32(42, 2, 42, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_load_epi32() { + const fn test_mm_maskz_load_epi32() { #[repr(align(16))] struct Align { data: [i32; 4], // 16 bytes @@ -53593,23 +53631,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm_maskz_load_epi32(m, black_box(p)); + let r = unsafe { _mm_maskz_load_epi32(m, black_box(p)) }; let e = _mm_setr_epi32(0, 2, 0, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_storeu_epi32() { + const fn test_mm_mask_storeu_epi32() { let mut r = [42_i32; 4]; let a = _mm_setr_epi32(1, 2, 3, 4); let m = 0b1010; - _mm_mask_storeu_epi32(r.as_mut_ptr(), m, a); + unsafe { + _mm_mask_storeu_epi32(r.as_mut_ptr(), m, a); + } let e = _mm_setr_epi32(42, 2, 42, 4); - assert_eq_m128i(_mm_loadu_epi32(r.as_ptr()), e); + assert_eq_m128i(unsafe { _mm_loadu_epi32(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_store_epi32() { + const fn test_mm_mask_store_epi32() { #[repr(align(16))] struct Align { data: [i32; 4], // 16 bytes @@ -53617,34 +53657,36 @@ mod tests { let mut r = Align { data: [42; 4] }; let a = _mm_setr_epi32(1, 2, 3, 4); let m = 0b1010; - _mm_mask_store_epi32(r.data.as_mut_ptr(), m, a); + unsafe { + _mm_mask_store_epi32(r.data.as_mut_ptr(), m, a); + } let e = _mm_setr_epi32(42, 2, 42, 4); - assert_eq_m128i(_mm_load_epi32(r.data.as_ptr()), e); + assert_eq_m128i(unsafe { _mm_load_epi32(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_loadu_epi64() { + const fn test_mm_mask_loadu_epi64() { let src = _mm_set1_epi64x(42); let a = &[1_i64, 2]; let p = a.as_ptr(); let m = 0b10; - let r = _mm_mask_loadu_epi64(src, m, black_box(p)); + let r = unsafe { _mm_mask_loadu_epi64(src, m, black_box(p)) }; let e = _mm_setr_epi64x(42, 2); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_loadu_epi64() { + const fn test_mm_maskz_loadu_epi64() { let a = &[1_i64, 2]; let p = a.as_ptr(); let m = 0b10; - let r = _mm_maskz_loadu_epi64(m, black_box(p)); + let r = unsafe { _mm_maskz_loadu_epi64(m, black_box(p)) }; let e = _mm_setr_epi64x(0, 2); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_load_epi64() { + const fn test_mm_mask_load_epi64() { #[repr(align(16))] struct Align { data: [i64; 2], // 16 bytes @@ -53653,13 +53695,13 @@ mod tests { let a = Align { data: [1_i64, 2] }; let p = a.data.as_ptr(); let m = 0b10; - let r = _mm_mask_load_epi64(src, m, black_box(p)); + let r = unsafe { _mm_mask_load_epi64(src, m, black_box(p)) }; let e = _mm_setr_epi64x(42, 2); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_load_epi64() { + const fn test_mm_maskz_load_epi64() { #[repr(align(16))] struct Align { data: [i64; 2], // 16 bytes @@ -53667,23 +53709,25 @@ mod tests { let a = Align { data: [1_i64, 2] }; let p = a.data.as_ptr(); let m = 0b10; - let r = _mm_maskz_load_epi64(m, black_box(p)); + let r = unsafe { _mm_maskz_load_epi64(m, black_box(p)) }; let e = _mm_setr_epi64x(0, 2); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_storeu_epi64() { + const fn test_mm_mask_storeu_epi64() { let mut r = [42_i64; 2]; let a = _mm_setr_epi64x(1, 2); let m = 0b10; - _mm_mask_storeu_epi64(r.as_mut_ptr(), m, a); + unsafe { + _mm_mask_storeu_epi64(r.as_mut_ptr(), m, a); + } let e = _mm_setr_epi64x(42, 2); - assert_eq_m128i(_mm_loadu_epi64(r.as_ptr()), e); + assert_eq_m128i(unsafe { _mm_loadu_epi64(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_store_epi64() { + const fn test_mm_mask_store_epi64() { #[repr(align(16))] struct Align { data: [i64; 2], // 16 bytes @@ -53691,34 +53735,36 @@ mod tests { let mut r = Align { data: [42; 2] }; let a = _mm_setr_epi64x(1, 2); let m = 0b10; - _mm_mask_store_epi64(r.data.as_mut_ptr(), m, a); + unsafe { + _mm_mask_store_epi64(r.data.as_mut_ptr(), m, a); + } let e = _mm_setr_epi64x(42, 2); - assert_eq_m128i(_mm_load_epi64(r.data.as_ptr()), e); + assert_eq_m128i(unsafe { _mm_load_epi64(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_loadu_ps() { + const fn test_mm_mask_loadu_ps() { let src = _mm_set1_ps(42.0); let a = &[1.0_f32, 2.0, 3.0, 4.0]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm_mask_loadu_ps(src, m, black_box(p)); + let r = unsafe { _mm_mask_loadu_ps(src, m, black_box(p)) }; let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); assert_eq_m128(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_loadu_ps() { + const fn test_mm_maskz_loadu_ps() { let a = &[1.0_f32, 2.0, 3.0, 4.0]; let p = a.as_ptr(); let m = 0b1010; - let r = _mm_maskz_loadu_ps(m, black_box(p)); + let r = unsafe { _mm_maskz_loadu_ps(m, black_box(p)) }; let e = _mm_setr_ps(0.0, 2.0, 0.0, 4.0); assert_eq_m128(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_load_ps() { + const fn test_mm_mask_load_ps() { #[repr(align(16))] struct Align { data: [f32; 4], // 16 bytes @@ -53729,13 +53775,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm_mask_load_ps(src, m, black_box(p)); + let r = unsafe { _mm_mask_load_ps(src, m, black_box(p)) }; let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); assert_eq_m128(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_load_ps() { + const fn test_mm_maskz_load_ps() { #[repr(align(16))] struct Align { data: [f32; 4], // 16 bytes @@ -53745,23 +53791,25 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b1010; - let r = _mm_maskz_load_ps(m, black_box(p)); + let r = unsafe { _mm_maskz_load_ps(m, black_box(p)) }; let e = _mm_setr_ps(0.0, 2.0, 0.0, 4.0); assert_eq_m128(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_storeu_ps() { + const fn test_mm_mask_storeu_ps() { let mut r = [42_f32; 4]; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let m = 0b1010; - _mm_mask_storeu_ps(r.as_mut_ptr(), m, a); + unsafe { + _mm_mask_storeu_ps(r.as_mut_ptr(), m, a); + } let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); - assert_eq_m128(_mm_loadu_ps(r.as_ptr()), e); + assert_eq_m128(unsafe { _mm_loadu_ps(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_store_ps() { + const fn test_mm_mask_store_ps() { #[repr(align(16))] struct Align { data: [f32; 4], // 16 bytes @@ -53769,34 +53817,36 @@ mod tests { let mut r = Align { data: [42.0; 4] }; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let m = 0b1010; - _mm_mask_store_ps(r.data.as_mut_ptr(), m, a); + unsafe { + _mm_mask_store_ps(r.data.as_mut_ptr(), m, a); + } let e = _mm_setr_ps(42.0, 2.0, 42.0, 4.0); - assert_eq_m128(_mm_load_ps(r.data.as_ptr()), e); + assert_eq_m128(unsafe { _mm_load_ps(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_loadu_pd() { + const fn test_mm_mask_loadu_pd() { let src = _mm_set1_pd(42.0); let a = &[1.0_f64, 2.0]; let p = a.as_ptr(); let m = 0b10; - let r = _mm_mask_loadu_pd(src, m, black_box(p)); + let r = unsafe { _mm_mask_loadu_pd(src, m, black_box(p)) }; let e = _mm_setr_pd(42.0, 2.0); assert_eq_m128d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_loadu_pd() { + const fn test_mm_maskz_loadu_pd() { let a = &[1.0_f64, 2.0]; let p = a.as_ptr(); let m = 0b10; - let r = _mm_maskz_loadu_pd(m, black_box(p)); + let r = unsafe { _mm_maskz_loadu_pd(m, black_box(p)) }; let e = _mm_setr_pd(0.0, 2.0); assert_eq_m128d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_load_pd() { + const fn test_mm_mask_load_pd() { #[repr(align(16))] struct Align { data: [f64; 2], // 16 bytes @@ -53807,13 +53857,13 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b10; - let r = _mm_mask_load_pd(src, m, black_box(p)); + let r = unsafe { _mm_mask_load_pd(src, m, black_box(p)) }; let e = _mm_setr_pd(42.0, 2.0); assert_eq_m128d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_maskz_load_pd() { + const fn test_mm_maskz_load_pd() { #[repr(align(16))] struct Align { data: [f64; 2], // 16 bytes @@ -53823,77 +53873,79 @@ mod tests { }; let p = a.data.as_ptr(); let m = 0b10; - let r = _mm_maskz_load_pd(m, black_box(p)); + let r = unsafe { _mm_maskz_load_pd(m, black_box(p)) }; let e = _mm_setr_pd(0.0, 2.0); assert_eq_m128d(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm_mask_load_ss() { + fn test_mm_mask_load_ss() { #[repr(align(16))] struct Align { data: f32, } let src = _mm_set_ss(2.0); let mem = Align { data: 1.0 }; - let r = _mm_mask_load_ss(src, 0b1, &mem.data); + let r = unsafe { _mm_mask_load_ss(src, 0b1, &mem.data) }; assert_eq_m128(r, _mm_set_ss(1.0)); - let r = _mm_mask_load_ss(src, 0b0, &mem.data); + let r = unsafe { _mm_mask_load_ss(src, 0b0, &mem.data) }; assert_eq_m128(r, _mm_set_ss(2.0)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm_maskz_load_ss() { + fn test_mm_maskz_load_ss() { #[repr(align(16))] struct Align { data: f32, } let mem = Align { data: 1.0 }; - let r = _mm_maskz_load_ss(0b1, &mem.data); + let r = unsafe { _mm_maskz_load_ss(0b1, &mem.data) }; assert_eq_m128(r, _mm_set_ss(1.0)); - let r = _mm_maskz_load_ss(0b0, &mem.data); + let r = unsafe { _mm_maskz_load_ss(0b0, &mem.data) }; assert_eq_m128(r, _mm_set_ss(0.0)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm_mask_load_sd() { + fn test_mm_mask_load_sd() { #[repr(align(16))] struct Align { data: f64, } let src = _mm_set_sd(2.0); let mem = Align { data: 1.0 }; - let r = _mm_mask_load_sd(src, 0b1, &mem.data); + let r = unsafe { _mm_mask_load_sd(src, 0b1, &mem.data) }; assert_eq_m128d(r, _mm_set_sd(1.0)); - let r = _mm_mask_load_sd(src, 0b0, &mem.data); + let r = unsafe { _mm_mask_load_sd(src, 0b0, &mem.data) }; assert_eq_m128d(r, _mm_set_sd(2.0)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm_maskz_load_sd() { + fn test_mm_maskz_load_sd() { #[repr(align(16))] struct Align { data: f64, } let mem = Align { data: 1.0 }; - let r = _mm_maskz_load_sd(0b1, &mem.data); + let r = unsafe { _mm_maskz_load_sd(0b1, &mem.data) }; assert_eq_m128d(r, _mm_set_sd(1.0)); - let r = _mm_maskz_load_sd(0b0, &mem.data); + let r = unsafe { _mm_maskz_load_sd(0b0, &mem.data) }; assert_eq_m128d(r, _mm_set_sd(0.0)); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_storeu_pd() { + const fn test_mm_mask_storeu_pd() { let mut r = [42_f64; 2]; let a = _mm_setr_pd(1.0, 2.0); let m = 0b10; - _mm_mask_storeu_pd(r.as_mut_ptr(), m, a); + unsafe { + _mm_mask_storeu_pd(r.as_mut_ptr(), m, a); + } let e = _mm_setr_pd(42.0, 2.0); - assert_eq_m128d(_mm_loadu_pd(r.as_ptr()), e); + assert_eq_m128d(unsafe { _mm_loadu_pd(r.as_ptr()) }, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_mask_store_pd() { + const fn test_mm_mask_store_pd() { #[repr(align(16))] struct Align { data: [f64; 2], // 16 bytes @@ -53901,36 +53953,46 @@ mod tests { let mut r = Align { data: [42.0; 2] }; let a = _mm_setr_pd(1.0, 2.0); let m = 0b10; - _mm_mask_store_pd(r.data.as_mut_ptr(), m, a); + unsafe { + _mm_mask_store_pd(r.data.as_mut_ptr(), m, a); + } let e = _mm_setr_pd(42.0, 2.0); - assert_eq_m128d(_mm_load_pd(r.data.as_ptr()), e); + assert_eq_m128d(unsafe { _mm_load_pd(r.data.as_ptr()) }, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm_mask_store_ss() { + fn test_mm_mask_store_ss() { #[repr(align(16))] struct Align { data: f32, } let a = _mm_set_ss(2.0); let mut mem = Align { data: 1.0 }; - _mm_mask_store_ss(&mut mem.data, 0b1, a); + unsafe { + _mm_mask_store_ss(&mut mem.data, 0b1, a); + } assert_eq!(mem.data, 2.0); - _mm_mask_store_ss(&mut mem.data, 0b0, a); + unsafe { + _mm_mask_store_ss(&mut mem.data, 0b0, a); + } assert_eq!(mem.data, 2.0); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm_mask_store_sd() { + fn test_mm_mask_store_sd() { #[repr(align(16))] struct Align { data: f64, } let a = _mm_set_sd(2.0); let mut mem = Align { data: 1.0 }; - _mm_mask_store_sd(&mut mem.data, 0b1, a); + unsafe { + _mm_mask_store_sd(&mut mem.data, 0b1, a); + } assert_eq!(mem.data, 2.0); - _mm_mask_store_sd(&mut mem.data, 0b0, a); + unsafe { + _mm_mask_store_sd(&mut mem.data, 0b0, a); + } assert_eq!(mem.data, 2.0); } @@ -57931,11 +57993,11 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_kortest_mask16_u8() { + const fn test_kortest_mask16_u8() { let a: __mmask16 = 0b0110100101101001; let b: __mmask16 = 0b1011011010110110; let mut all_ones: u8 = 0; - let r = _kortest_mask16_u8(a, b, &mut all_ones); + let r = unsafe { _kortest_mask16_u8(a, b, &mut all_ones) }; assert_eq!(r, 0); assert_eq!(all_ones, 1); } @@ -57997,18 +58059,20 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_load_mask16() { + const fn test_load_mask16() { let a: __mmask16 = 0b1001011011000011; - let r = _load_mask16(&a); + let r = unsafe { _load_mask16(&a) }; let e: __mmask16 = 0b1001011011000011; assert_eq!(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_store_mask16() { + const fn test_store_mask16() { let a: __mmask16 = 0b0110100100111100; let mut r = 0; - _store_mask16(&mut r, a); + unsafe { + _store_mask16(&mut r, a); + } let e: __mmask16 = 0b0110100100111100; assert_eq!(r, e); } @@ -58189,7 +58253,7 @@ mod tests { #[simd_test(enable = "avx512f")] #[cfg_attr(miri, ignore)] - unsafe fn test_mm512_stream_ps() { + fn test_mm512_stream_ps() { #[repr(align(64))] struct Memory { pub data: [f32; 16], // 64 bytes @@ -58197,7 +58261,9 @@ mod tests { let a = _mm512_set1_ps(7.0); let mut mem = Memory { data: [-1.0; 16] }; - _mm512_stream_ps(&mut mem.data[0] as *mut f32, a); + unsafe { + _mm512_stream_ps(&mut mem.data[0] as *mut f32, a); + } _mm_sfence(); for i in 0..16 { assert_eq!(mem.data[i], get_m512(a, i)); @@ -58206,7 +58272,7 @@ mod tests { #[simd_test(enable = "avx512f")] #[cfg_attr(miri, ignore)] - unsafe fn test_mm512_stream_pd() { + fn test_mm512_stream_pd() { #[repr(align(64))] struct Memory { pub data: [f64; 8], @@ -58214,7 +58280,9 @@ mod tests { let a = _mm512_set1_pd(7.0); let mut mem = Memory { data: [-1.0; 8] }; - _mm512_stream_pd(&mut mem.data[0] as *mut f64, a); + unsafe { + _mm512_stream_pd(&mut mem.data[0] as *mut f64, a); + } _mm_sfence(); for i in 0..8 { assert_eq!(mem.data[i], get_m512d(a, i)); @@ -58223,7 +58291,7 @@ mod tests { #[simd_test(enable = "avx512f")] #[cfg_attr(miri, ignore)] - unsafe fn test_mm512_stream_si512() { + fn test_mm512_stream_si512() { #[repr(align(64))] struct Memory { pub data: [i64; 8], @@ -58231,7 +58299,9 @@ mod tests { let a = _mm512_set1_epi32(7); let mut mem = Memory { data: [-1; 8] }; - _mm512_stream_si512(mem.data.as_mut_ptr().cast(), a); + unsafe { + _mm512_stream_si512(mem.data.as_mut_ptr().cast(), a); + } _mm_sfence(); for i in 0..8 { assert_eq!(mem.data[i], get_m512i(a, i)); @@ -58239,9 +58309,9 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_stream_load_si512() { + fn test_mm512_stream_load_si512() { let a = _mm512_set_epi64(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm512_stream_load_si512(core::ptr::addr_of!(a) as *const _); + let r = unsafe { _mm512_stream_load_si512(core::ptr::addr_of!(a) as *const _) }; assert_eq_m512i(a, r); } @@ -58558,75 +58628,103 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_compressstoreu_epi32() { + fn test_mm512_mask_compressstoreu_epi32() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let mut r = [0_i32; 16]; - _mm512_mask_compressstoreu_epi32(r.as_mut_ptr(), 0, a); + unsafe { + _mm512_mask_compressstoreu_epi32(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i32; 16]); - _mm512_mask_compressstoreu_epi32(r.as_mut_ptr(), 0b1111000011001010, a); + unsafe { + _mm512_mask_compressstoreu_epi32(r.as_mut_ptr(), 0b1111000011001010, a); + } assert_eq!(&r, &[2, 4, 7, 8, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0]); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_compressstoreu_epi32() { + fn test_mm256_mask_compressstoreu_epi32() { let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); let mut r = [0_i32; 8]; - _mm256_mask_compressstoreu_epi32(r.as_mut_ptr(), 0, a); + unsafe { + _mm256_mask_compressstoreu_epi32(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i32; 8]); - _mm256_mask_compressstoreu_epi32(r.as_mut_ptr(), 0b11001010, a); + unsafe { + _mm256_mask_compressstoreu_epi32(r.as_mut_ptr(), 0b11001010, a); + } assert_eq!(&r, &[2, 4, 7, 8, 0, 0, 0, 0]); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_compressstoreu_epi32() { + fn test_mm_mask_compressstoreu_epi32() { let a = _mm_setr_epi32(1, 2, 3, 4); let mut r = [0_i32; 4]; - _mm_mask_compressstoreu_epi32(r.as_mut_ptr(), 0, a); + unsafe { + _mm_mask_compressstoreu_epi32(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i32; 4]); - _mm_mask_compressstoreu_epi32(r.as_mut_ptr(), 0b1011, a); + unsafe { + _mm_mask_compressstoreu_epi32(r.as_mut_ptr(), 0b1011, a); + } assert_eq!(&r, &[1, 2, 4, 0]); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_compressstoreu_epi64() { + fn test_mm512_mask_compressstoreu_epi64() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let mut r = [0_i64; 8]; - _mm512_mask_compressstoreu_epi64(r.as_mut_ptr(), 0, a); + unsafe { + _mm512_mask_compressstoreu_epi64(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i64; 8]); - _mm512_mask_compressstoreu_epi64(r.as_mut_ptr(), 0b11001010, a); + unsafe { + _mm512_mask_compressstoreu_epi64(r.as_mut_ptr(), 0b11001010, a); + } assert_eq!(&r, &[2, 4, 7, 8, 0, 0, 0, 0]); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_compressstoreu_epi64() { + fn test_mm256_mask_compressstoreu_epi64() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let mut r = [0_i64; 4]; - _mm256_mask_compressstoreu_epi64(r.as_mut_ptr(), 0, a); + unsafe { + _mm256_mask_compressstoreu_epi64(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i64; 4]); - _mm256_mask_compressstoreu_epi64(r.as_mut_ptr(), 0b1011, a); + unsafe { + _mm256_mask_compressstoreu_epi64(r.as_mut_ptr(), 0b1011, a); + } assert_eq!(&r, &[1, 2, 4, 0]); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_compressstoreu_epi64() { + fn test_mm_mask_compressstoreu_epi64() { let a = _mm_setr_epi64x(1, 2); let mut r = [0_i64; 2]; - _mm_mask_compressstoreu_epi64(r.as_mut_ptr(), 0, a); + unsafe { + _mm_mask_compressstoreu_epi64(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i64; 2]); - _mm_mask_compressstoreu_epi64(r.as_mut_ptr(), 0b10, a); + unsafe { + _mm_mask_compressstoreu_epi64(r.as_mut_ptr(), 0b10, a); + } assert_eq!(&r, &[2, 0]); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_compressstoreu_ps() { + fn test_mm512_mask_compressstoreu_ps() { let a = _mm512_setr_ps( 1_f32, 2_f32, 3_f32, 4_f32, 5_f32, 6_f32, 7_f32, 8_f32, 9_f32, 10_f32, 11_f32, 12_f32, 13_f32, 14_f32, 15_f32, 16_f32, ); let mut r = [0_f32; 16]; - _mm512_mask_compressstoreu_ps(r.as_mut_ptr(), 0, a); + unsafe { + _mm512_mask_compressstoreu_ps(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_f32; 16]); - _mm512_mask_compressstoreu_ps(r.as_mut_ptr(), 0b1111000011001010, a); + unsafe { + _mm512_mask_compressstoreu_ps(r.as_mut_ptr(), 0b1111000011001010, a); + } assert_eq!( &r, &[ @@ -58637,12 +58735,16 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_compressstoreu_ps() { + fn test_mm256_mask_compressstoreu_ps() { let a = _mm256_setr_ps(1_f32, 2_f32, 3_f32, 4_f32, 5_f32, 6_f32, 7_f32, 8_f32); let mut r = [0_f32; 8]; - _mm256_mask_compressstoreu_ps(r.as_mut_ptr(), 0, a); + unsafe { + _mm256_mask_compressstoreu_ps(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_f32; 8]); - _mm256_mask_compressstoreu_ps(r.as_mut_ptr(), 0b11001010, a); + unsafe { + _mm256_mask_compressstoreu_ps(r.as_mut_ptr(), 0b11001010, a); + } assert_eq!( &r, &[2_f32, 4_f32, 7_f32, 8_f32, 0_f32, 0_f32, 0_f32, 0_f32] @@ -58650,47 +58752,63 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_compressstoreu_ps() { + fn test_mm_mask_compressstoreu_ps() { let a = _mm_setr_ps(1_f32, 2_f32, 3_f32, 4_f32); let mut r = [0.; 4]; - _mm_mask_compressstoreu_ps(r.as_mut_ptr(), 0, a); + unsafe { + _mm_mask_compressstoreu_ps(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0.; 4]); - _mm_mask_compressstoreu_ps(r.as_mut_ptr(), 0b1011, a); + unsafe { + _mm_mask_compressstoreu_ps(r.as_mut_ptr(), 0b1011, a); + } assert_eq!(&r, &[1_f32, 2_f32, 4_f32, 0_f32]); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_compressstoreu_pd() { + fn test_mm512_mask_compressstoreu_pd() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); let mut r = [0.; 8]; - _mm512_mask_compressstoreu_pd(r.as_mut_ptr(), 0, a); + unsafe { + _mm512_mask_compressstoreu_pd(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0.; 8]); - _mm512_mask_compressstoreu_pd(r.as_mut_ptr(), 0b11001010, a); + unsafe { + _mm512_mask_compressstoreu_pd(r.as_mut_ptr(), 0b11001010, a); + } assert_eq!(&r, &[2., 4., 7., 8., 0., 0., 0., 0.]); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_compressstoreu_pd() { + fn test_mm256_mask_compressstoreu_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let mut r = [0.; 4]; - _mm256_mask_compressstoreu_pd(r.as_mut_ptr(), 0, a); + unsafe { + _mm256_mask_compressstoreu_pd(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0.; 4]); - _mm256_mask_compressstoreu_pd(r.as_mut_ptr(), 0b1011, a); + unsafe { + _mm256_mask_compressstoreu_pd(r.as_mut_ptr(), 0b1011, a); + } assert_eq!(&r, &[1., 2., 4., 0.]); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_compressstoreu_pd() { + fn test_mm_mask_compressstoreu_pd() { let a = _mm_setr_pd(1., 2.); let mut r = [0.; 2]; - _mm_mask_compressstoreu_pd(r.as_mut_ptr(), 0, a); + unsafe { + _mm_mask_compressstoreu_pd(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0.; 2]); - _mm_mask_compressstoreu_pd(r.as_mut_ptr(), 0b10, a); + unsafe { + _mm_mask_compressstoreu_pd(r.as_mut_ptr(), 0b10, a); + } assert_eq!(&r, &[2., 0.]); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_expand_epi32() { + fn test_mm512_mask_expand_epi32() { let src = _mm512_set1_epi32(200); let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); let r = _mm512_mask_expand_epi32(src, 0, a); @@ -58826,109 +58944,135 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_loadu_epi32() { + const fn test_mm512_loadu_epi32() { let a = &[4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50]; let p = a.as_ptr(); - let r = _mm512_loadu_epi32(black_box(p)); + let r = unsafe { _mm512_loadu_epi32(black_box(p)) }; let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_loadu_epi32() { + const fn test_mm256_loadu_epi32() { let a = &[4, 3, 2, 5, 8, 9, 64, 50]; let p = a.as_ptr(); - let r = _mm256_loadu_epi32(black_box(p)); + let r = unsafe { _mm256_loadu_epi32(black_box(p)) }; let e = _mm256_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_loadu_epi32() { + const fn test_mm_loadu_epi32() { let a = &[4, 3, 2, 5]; let p = a.as_ptr(); - let r = _mm_loadu_epi32(black_box(p)); + let r = unsafe { _mm_loadu_epi32(black_box(p)) }; let e = _mm_setr_epi32(4, 3, 2, 5); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtepi32_storeu_epi16() { + fn test_mm512_mask_cvtepi32_storeu_epi16() { let a = _mm512_set1_epi32(9); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111_11111111, a); + unsafe { + _mm512_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111_11111111, a); + } let e = _mm256_set1_epi16(9); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtepi32_storeu_epi16() { + fn test_mm256_mask_cvtepi32_storeu_epi16() { let a = _mm256_set1_epi32(9); let mut r = _mm_undefined_si128(); - _mm256_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm256_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set1_epi16(9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtepi32_storeu_epi16() { + fn test_mm_mask_cvtepi32_storeu_epi16() { let a = _mm_set1_epi32(9); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm_mask_cvtepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, 9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtsepi32_storeu_epi16() { + fn test_mm512_mask_cvtsepi32_storeu_epi16() { let a = _mm512_set1_epi32(i32::MAX); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111_11111111, a); + unsafe { + _mm512_mask_cvtsepi32_storeu_epi16( + &mut r as *mut _ as *mut i16, + 0b11111111_11111111, + a, + ); + } let e = _mm256_set1_epi16(i16::MAX); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtsepi32_storeu_epi16() { + fn test_mm256_mask_cvtsepi32_storeu_epi16() { let a = _mm256_set1_epi32(i32::MAX); let mut r = _mm_undefined_si128(); - _mm256_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm256_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set1_epi16(i16::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtsepi32_storeu_epi16() { + fn test_mm_mask_cvtsepi32_storeu_epi16() { let a = _mm_set1_epi32(i32::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm_mask_cvtsepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtusepi32_storeu_epi16() { + fn test_mm512_mask_cvtusepi32_storeu_epi16() { let a = _mm512_set1_epi32(i32::MAX); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111_11111111, a); + unsafe { + _mm512_mask_cvtusepi32_storeu_epi16( + &mut r as *mut _ as *mut i16, + 0b11111111_11111111, + a, + ); + } let e = _mm256_set1_epi16(u16::MAX as i16); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtusepi32_storeu_epi16() { + fn test_mm256_mask_cvtusepi32_storeu_epi16() { let a = _mm256_set1_epi32(i32::MAX); let mut r = _mm_undefined_si128(); - _mm256_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm256_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set1_epi16(u16::MAX as i16); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtusepi32_storeu_epi16() { + fn test_mm_mask_cvtusepi32_storeu_epi16() { let a = _mm_set1_epi32(i32::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm_mask_cvtusepi32_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16( 0, 0, @@ -58943,46 +59087,56 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtepi32_storeu_epi8() { + fn test_mm512_mask_cvtepi32_storeu_epi8() { let a = _mm512_set1_epi32(9); let mut r = _mm_undefined_si128(); - _mm512_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + unsafe { + _mm512_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + } let e = _mm_set1_epi8(9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtepi32_storeu_epi8() { + fn test_mm256_mask_cvtepi32_storeu_epi8() { let a = _mm256_set1_epi32(9); let mut r = _mm_set1_epi8(0); - _mm256_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm256_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtepi32_storeu_epi8() { + fn test_mm_mask_cvtepi32_storeu_epi8() { let a = _mm_set1_epi32(9); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtsepi32_storeu_epi8() { + fn test_mm512_mask_cvtsepi32_storeu_epi8() { let a = _mm512_set1_epi32(i32::MAX); let mut r = _mm_undefined_si128(); - _mm512_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + unsafe { + _mm512_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + } let e = _mm_set1_epi8(i8::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtsepi32_storeu_epi8() { + fn test_mm256_mask_cvtsepi32_storeu_epi8() { let a = _mm256_set1_epi32(i32::MAX); let mut r = _mm_set1_epi8(0); - _mm256_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm256_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -58994,10 +59148,12 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtsepi32_storeu_epi8() { + fn test_mm_mask_cvtsepi32_storeu_epi8() { let a = _mm_set1_epi32(i32::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtsepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -59009,19 +59165,23 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtusepi32_storeu_epi8() { + fn test_mm512_mask_cvtusepi32_storeu_epi8() { let a = _mm512_set1_epi32(i32::MAX); let mut r = _mm_undefined_si128(); - _mm512_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + unsafe { + _mm512_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111_11111111, a); + } let e = _mm_set1_epi8(u8::MAX as i8); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtusepi32_storeu_epi8() { + fn test_mm256_mask_cvtusepi32_storeu_epi8() { let a = _mm256_set1_epi32(i32::MAX); let mut r = _mm_set1_epi8(0); - _mm256_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm256_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -59033,10 +59193,12 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtusepi32_storeu_epi8() { + fn test_mm_mask_cvtusepi32_storeu_epi8() { let a = _mm_set1_epi32(i32::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtusepi32_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -59048,48 +59210,56 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_storeu_epi32() { + const fn test_mm512_storeu_epi32() { let a = _mm512_set1_epi32(9); let mut r = _mm512_undefined_epi32(); - _mm512_storeu_epi32(&mut r as *mut _ as *mut i32, a); + unsafe { + _mm512_storeu_epi32(&mut r as *mut _ as *mut i32, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_storeu_epi32() { + const fn test_mm256_storeu_epi32() { let a = _mm256_set1_epi32(9); let mut r = _mm256_undefined_si256(); - _mm256_storeu_epi32(&mut r as *mut _ as *mut i32, a); + unsafe { + _mm256_storeu_epi32(&mut r as *mut _ as *mut i32, a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_storeu_epi32() { + const fn test_mm_storeu_epi32() { let a = _mm_set1_epi32(9); let mut r = _mm_undefined_si128(); - _mm_storeu_epi32(&mut r as *mut _ as *mut i32, a); + unsafe { + _mm_storeu_epi32(&mut r as *mut _ as *mut i32, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_loadu_si512() { + const fn test_mm512_loadu_si512() { let a = &[4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50]; let p = a.as_ptr().cast(); - let r = _mm512_loadu_si512(black_box(p)); + let r = unsafe { _mm512_loadu_si512(black_box(p)) }; let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_storeu_si512() { + const fn test_mm512_storeu_si512() { let a = _mm512_set1_epi32(9); let mut r = _mm512_undefined_epi32(); - _mm512_storeu_si512(&mut r as *mut _, a); + unsafe { + _mm512_storeu_si512(&mut r as *mut _, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_load_si512() { + const fn test_mm512_load_si512() { #[repr(align(64))] struct Align { data: [i32; 16], // 64 bytes @@ -59098,21 +59268,23 @@ mod tests { data: [4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50], }; let p = (a.data).as_ptr().cast(); - let r = _mm512_load_si512(black_box(p)); + let r = unsafe { _mm512_load_si512(black_box(p)) }; let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_store_si512() { + const fn test_mm512_store_si512() { let a = _mm512_set1_epi32(9); let mut r = _mm512_undefined_epi32(); - _mm512_store_si512(&mut r as *mut _, a); + unsafe { + _mm512_store_si512(&mut r as *mut _, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_load_epi32() { + const fn test_mm512_load_epi32() { #[repr(align(64))] struct Align { data: [i32; 16], // 64 bytes @@ -59121,13 +59293,13 @@ mod tests { data: [4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50], }; let p = (a.data).as_ptr(); - let r = _mm512_load_epi32(black_box(p)); + let r = unsafe { _mm512_load_epi32(black_box(p)) }; let e = _mm512_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50, -4, -3, -2, -5, -8, -9, -64, -50); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_load_epi32() { + const fn test_mm256_load_epi32() { #[repr(align(64))] struct Align { data: [i32; 8], @@ -59136,50 +59308,56 @@ mod tests { data: [4, 3, 2, 5, 8, 9, 64, 50], }; let p = (a.data).as_ptr(); - let r = _mm256_load_epi32(black_box(p)); + let r = unsafe { _mm256_load_epi32(black_box(p)) }; let e = _mm256_setr_epi32(4, 3, 2, 5, 8, 9, 64, 50); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_load_epi32() { + const fn test_mm_load_epi32() { #[repr(align(64))] struct Align { data: [i32; 4], } let a = Align { data: [4, 3, 2, 5] }; let p = (a.data).as_ptr(); - let r = _mm_load_epi32(black_box(p)); + let r = unsafe { _mm_load_epi32(black_box(p)) }; let e = _mm_setr_epi32(4, 3, 2, 5); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_store_epi32() { + const fn test_mm512_store_epi32() { let a = _mm512_set1_epi32(9); let mut r = _mm512_undefined_epi32(); - _mm512_store_epi32(&mut r as *mut _ as *mut i32, a); + unsafe { + _mm512_store_epi32(&mut r as *mut _ as *mut i32, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_store_epi32() { + const fn test_mm256_store_epi32() { let a = _mm256_set1_epi32(9); let mut r = _mm256_undefined_si256(); - _mm256_store_epi32(&mut r as *mut _ as *mut i32, a); + unsafe { + _mm256_store_epi32(&mut r as *mut _ as *mut i32, a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_store_epi32() { + const fn test_mm_store_epi32() { let a = _mm_set1_epi32(9); let mut r = _mm_undefined_si128(); - _mm_store_epi32(&mut r as *mut _ as *mut i32, a); + unsafe { + _mm_store_epi32(&mut r as *mut _ as *mut i32, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_load_ps() { + const fn test_mm512_load_ps() { #[repr(align(64))] struct Align { data: [f32; 16], // 64 bytes @@ -59190,7 +59368,7 @@ mod tests { ], }; let p = (a.data).as_ptr(); - let r = _mm512_load_ps(black_box(p)); + let r = unsafe { _mm512_load_ps(black_box(p)) }; let e = _mm512_setr_ps( 4., 3., 2., 5., 8., 9., 64., 50., -4., -3., -2., -5., -8., -9., -64., -50., ); @@ -59198,10 +59376,12 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_store_ps() { + const fn test_mm512_store_ps() { let a = _mm512_set1_ps(9.); let mut r = _mm512_undefined_ps(); - _mm512_store_ps(&mut r as *mut _ as *mut f32, a); + unsafe { + _mm512_store_ps(&mut r as *mut _ as *mut f32, a); + } assert_eq_m512(r, a); } @@ -62191,140 +62371,140 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_expandloadu_epi32() { + fn test_mm512_mask_expandloadu_epi32() { let src = _mm512_set1_epi32(42); let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_mask_expandloadu_epi32(src, m, black_box(p)); + let r = unsafe { _mm512_mask_expandloadu_epi32(src, m, black_box(p)) }; let e = _mm512_set_epi32(8, 7, 6, 42, 5, 42, 42, 42, 4, 3, 42, 42, 2, 42, 1, 42); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_maskz_expandloadu_epi32() { + fn test_mm512_maskz_expandloadu_epi32() { let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_maskz_expandloadu_epi32(m, black_box(p)); + let r = unsafe { _mm512_maskz_expandloadu_epi32(m, black_box(p)) }; let e = _mm512_set_epi32(8, 7, 6, 0, 5, 0, 0, 0, 4, 3, 0, 0, 2, 0, 1, 0); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_expandloadu_epi32() { + fn test_mm256_mask_expandloadu_epi32() { let src = _mm256_set1_epi32(42); let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_mask_expandloadu_epi32(src, m, black_box(p)); + let r = unsafe { _mm256_mask_expandloadu_epi32(src, m, black_box(p)) }; let e = _mm256_set_epi32(4, 3, 2, 42, 1, 42, 42, 42); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_maskz_expandloadu_epi32() { + fn test_mm256_maskz_expandloadu_epi32() { let a = &[1_i32, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_maskz_expandloadu_epi32(m, black_box(p)); + let r = unsafe { _mm256_maskz_expandloadu_epi32(m, black_box(p)) }; let e = _mm256_set_epi32(4, 3, 2, 0, 1, 0, 0, 0); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_expandloadu_epi32() { + fn test_mm_mask_expandloadu_epi32() { let src = _mm_set1_epi32(42); let a = &[1_i32, 2, 3, 4]; let p = a.as_ptr(); let m = 0b11111000; - let r = _mm_mask_expandloadu_epi32(src, m, black_box(p)); + let r = unsafe { _mm_mask_expandloadu_epi32(src, m, black_box(p)) }; let e = _mm_set_epi32(1, 42, 42, 42); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_maskz_expandloadu_epi32() { + fn test_mm_maskz_expandloadu_epi32() { let a = &[1_i32, 2, 3, 4]; let p = a.as_ptr(); let m = 0b11111000; - let r = _mm_maskz_expandloadu_epi32(m, black_box(p)); + let r = unsafe { _mm_maskz_expandloadu_epi32(m, black_box(p)) }; let e = _mm_set_epi32(1, 0, 0, 0); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_expandloadu_epi64() { + fn test_mm512_mask_expandloadu_epi64() { let src = _mm512_set1_epi64(42); let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm512_mask_expandloadu_epi64(src, m, black_box(p)); + let r = unsafe { _mm512_mask_expandloadu_epi64(src, m, black_box(p)) }; let e = _mm512_set_epi64(4, 3, 2, 42, 1, 42, 42, 42); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_maskz_expandloadu_epi64() { + fn test_mm512_maskz_expandloadu_epi64() { let a = &[1_i64, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm512_maskz_expandloadu_epi64(m, black_box(p)); + let r = unsafe { _mm512_maskz_expandloadu_epi64(m, black_box(p)) }; let e = _mm512_set_epi64(4, 3, 2, 0, 1, 0, 0, 0); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_expandloadu_epi64() { + fn test_mm256_mask_expandloadu_epi64() { let src = _mm256_set1_epi64x(42); let a = &[1_i64, 2, 3, 4]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_mask_expandloadu_epi64(src, m, black_box(p)); + let r = unsafe { _mm256_mask_expandloadu_epi64(src, m, black_box(p)) }; let e = _mm256_set_epi64x(1, 42, 42, 42); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_maskz_expandloadu_epi64() { + fn test_mm256_maskz_expandloadu_epi64() { let a = &[1_i64, 2, 3, 4]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_maskz_expandloadu_epi64(m, black_box(p)); + let r = unsafe { _mm256_maskz_expandloadu_epi64(m, black_box(p)) }; let e = _mm256_set_epi64x(1, 0, 0, 0); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_expandloadu_epi64() { + fn test_mm_mask_expandloadu_epi64() { let src = _mm_set1_epi64x(42); let a = &[1_i64, 2]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_mask_expandloadu_epi64(src, m, black_box(p)); + let r = unsafe { _mm_mask_expandloadu_epi64(src, m, black_box(p)) }; let e = _mm_set_epi64x(42, 42); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_maskz_expandloadu_epi64() { + fn test_mm_maskz_expandloadu_epi64() { let a = &[1_i64, 2]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_maskz_expandloadu_epi64(m, black_box(p)); + let r = unsafe { _mm_maskz_expandloadu_epi64(m, black_box(p)) }; let e = _mm_set_epi64x(0, 0); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_expandloadu_ps() { + fn test_mm512_mask_expandloadu_ps() { let src = _mm512_set1_ps(42.); let a = &[ 1.0f32, 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_mask_expandloadu_ps(src, m, black_box(p)); + let r = unsafe { _mm512_mask_expandloadu_ps(src, m, black_box(p)) }; let e = _mm512_set_ps( 8., 7., 6., 42., 5., 42., 42., 42., 4., 3., 42., 42., 2., 42., 1., 42., ); @@ -62332,13 +62512,13 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_maskz_expandloadu_ps() { + fn test_mm512_maskz_expandloadu_ps() { let a = &[ 1.0f32, 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm512_maskz_expandloadu_ps(m, black_box(p)); + let r = unsafe { _mm512_maskz_expandloadu_ps(m, black_box(p)) }; let e = _mm512_set_ps( 8., 7., 6., 0., 5., 0., 0., 0., 4., 3., 0., 0., 2., 0., 1., 0., ); @@ -62346,106 +62526,106 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_expandloadu_ps() { + fn test_mm256_mask_expandloadu_ps() { let src = _mm256_set1_ps(42.); let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_mask_expandloadu_ps(src, m, black_box(p)); + let r = unsafe { _mm256_mask_expandloadu_ps(src, m, black_box(p)) }; let e = _mm256_set_ps(4., 3., 2., 42., 1., 42., 42., 42.); assert_eq_m256(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_maskz_expandloadu_ps() { + fn test_mm256_maskz_expandloadu_ps() { let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_maskz_expandloadu_ps(m, black_box(p)); + let r = unsafe { _mm256_maskz_expandloadu_ps(m, black_box(p)) }; let e = _mm256_set_ps(4., 3., 2., 0., 1., 0., 0., 0.); assert_eq_m256(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_expandloadu_ps() { + fn test_mm_mask_expandloadu_ps() { let src = _mm_set1_ps(42.); let a = &[1.0f32, 2., 3., 4.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_mask_expandloadu_ps(src, m, black_box(p)); + let r = unsafe { _mm_mask_expandloadu_ps(src, m, black_box(p)) }; let e = _mm_set_ps(1., 42., 42., 42.); assert_eq_m128(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_maskz_expandloadu_ps() { + fn test_mm_maskz_expandloadu_ps() { let a = &[1.0f32, 2., 3., 4.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_maskz_expandloadu_ps(m, black_box(p)); + let r = unsafe { _mm_maskz_expandloadu_ps(m, black_box(p)) }; let e = _mm_set_ps(1., 0., 0., 0.); assert_eq_m128(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_expandloadu_pd() { + fn test_mm512_mask_expandloadu_pd() { let src = _mm512_set1_pd(42.); let a = &[1.0f64, 2., 3., 4., 5., 6., 7., 8.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm512_mask_expandloadu_pd(src, m, black_box(p)); + let r = unsafe { _mm512_mask_expandloadu_pd(src, m, black_box(p)) }; let e = _mm512_set_pd(4., 3., 2., 42., 1., 42., 42., 42.); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_maskz_expandloadu_pd() { + fn test_mm512_maskz_expandloadu_pd() { let a = &[1.0f64, 2., 3., 4., 5., 6., 7., 8.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm512_maskz_expandloadu_pd(m, black_box(p)); + let r = unsafe { _mm512_maskz_expandloadu_pd(m, black_box(p)) }; let e = _mm512_set_pd(4., 3., 2., 0., 1., 0., 0., 0.); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_expandloadu_pd() { + fn test_mm256_mask_expandloadu_pd() { let src = _mm256_set1_pd(42.); let a = &[1.0f64, 2., 3., 4.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_mask_expandloadu_pd(src, m, black_box(p)); + let r = unsafe { _mm256_mask_expandloadu_pd(src, m, black_box(p)) }; let e = _mm256_set_pd(1., 42., 42., 42.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_maskz_expandloadu_pd() { + fn test_mm256_maskz_expandloadu_pd() { let a = &[1.0f64, 2., 3., 4.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm256_maskz_expandloadu_pd(m, black_box(p)); + let r = unsafe { _mm256_maskz_expandloadu_pd(m, black_box(p)) }; let e = _mm256_set_pd(1., 0., 0., 0.); assert_eq_m256d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_expandloadu_pd() { + fn test_mm_mask_expandloadu_pd() { let src = _mm_set1_pd(42.); let a = &[1.0f64, 2.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_mask_expandloadu_pd(src, m, black_box(p)); + let r = unsafe { _mm_mask_expandloadu_pd(src, m, black_box(p)) }; let e = _mm_set_pd(42., 42.); assert_eq_m128d(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_maskz_expandloadu_pd() { + fn test_mm_maskz_expandloadu_pd() { let a = &[1.0f64, 2.]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_maskz_expandloadu_pd(m, black_box(p)); + let r = unsafe { _mm_maskz_expandloadu_pd(m, black_box(p)) }; let e = _mm_set_pd(0., 0.); assert_eq_m128d(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs b/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs index 57d47c0bb010..f581711a3c99 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs @@ -16932,7 +16932,6 @@ unsafe extern "C" { mod tests { use crate::core_arch::assert_eq_const as assert_eq; use crate::core_arch::x86::*; - use crate::mem::transmute; use crate::ptr::{addr_of, addr_of_mut}; use stdarch_test::simd_test; @@ -17569,72 +17568,72 @@ mod tests { } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm_load_ph() { + const fn test_mm_load_ph() { let a = _mm_set_ph(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); - let b = _mm_load_ph(addr_of!(a).cast()); + let b = unsafe { _mm_load_ph(addr_of!(a).cast()) }; assert_eq_m128h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm256_load_ph() { + const fn test_mm256_load_ph() { let a = _mm256_set_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ); - let b = _mm256_load_ph(addr_of!(a).cast()); + let b = unsafe { _mm256_load_ph(addr_of!(a).cast()) }; assert_eq_m256h(a, b); } #[simd_test(enable = "avx512fp16")] - const unsafe fn test_mm512_load_ph() { + const fn test_mm512_load_ph() { let a = _mm512_set_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, ); - let b = _mm512_load_ph(addr_of!(a).cast()); + let b = unsafe { _mm512_load_ph(addr_of!(a).cast()) }; assert_eq_m512h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm_load_sh() { + const fn test_mm_load_sh() { let a = _mm_set_sh(1.0); - let b = _mm_load_sh(addr_of!(a).cast()); + let b = unsafe { _mm_load_sh(addr_of!(a).cast()) }; assert_eq_m128h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - unsafe fn test_mm_mask_load_sh() { + fn test_mm_mask_load_sh() { let a = _mm_set_sh(1.0); let src = _mm_set_sh(2.); - let b = _mm_mask_load_sh(src, 1, addr_of!(a).cast()); + let b = unsafe { _mm_mask_load_sh(src, 1, addr_of!(a).cast()) }; assert_eq_m128h(a, b); - let b = _mm_mask_load_sh(src, 0, addr_of!(a).cast()); + let b = unsafe { _mm_mask_load_sh(src, 0, addr_of!(a).cast()) }; assert_eq_m128h(src, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - unsafe fn test_mm_maskz_load_sh() { + fn test_mm_maskz_load_sh() { let a = _mm_set_sh(1.0); - let b = _mm_maskz_load_sh(1, addr_of!(a).cast()); + let b = unsafe { _mm_maskz_load_sh(1, addr_of!(a).cast()) }; assert_eq_m128h(a, b); - let b = _mm_maskz_load_sh(0, addr_of!(a).cast()); + let b = unsafe { _mm_maskz_load_sh(0, addr_of!(a).cast()) }; assert_eq_m128h(_mm_setzero_ph(), b); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm_loadu_ph() { + const fn test_mm_loadu_ph() { let array = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; - let r = _mm_loadu_ph(array.as_ptr()); + let r = unsafe { _mm_loadu_ph(array.as_ptr()) }; let e = _mm_setr_ph(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); assert_eq_m128h(r, e); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm256_loadu_ph() { + const fn test_mm256_loadu_ph() { let array = [ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ]; - let r = _mm256_loadu_ph(array.as_ptr()); + let r = unsafe { _mm256_loadu_ph(array.as_ptr()) }; let e = _mm256_setr_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ); @@ -17642,13 +17641,13 @@ mod tests { } #[simd_test(enable = "avx512fp16")] - const unsafe fn test_mm512_loadu_ph() { + const fn test_mm512_loadu_ph() { let array = [ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, ]; - let r = _mm512_loadu_ph(array.as_ptr()); + let r = unsafe { _mm512_loadu_ph(array.as_ptr()) }; let e = _mm512_setr_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, @@ -17686,81 +17685,99 @@ mod tests { } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm_store_ph() { + const fn test_mm_store_ph() { let a = _mm_set_ph(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); let mut b = _mm_setzero_ph(); - _mm_store_ph(addr_of_mut!(b).cast(), a); + unsafe { + _mm_store_ph(addr_of_mut!(b).cast(), a); + } assert_eq_m128h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm256_store_ph() { + const fn test_mm256_store_ph() { let a = _mm256_set_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ); let mut b = _mm256_setzero_ph(); - _mm256_store_ph(addr_of_mut!(b).cast(), a); + unsafe { + _mm256_store_ph(addr_of_mut!(b).cast(), a); + } assert_eq_m256h(a, b); } #[simd_test(enable = "avx512fp16")] - const unsafe fn test_mm512_store_ph() { + const fn test_mm512_store_ph() { let a = _mm512_set_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, ); let mut b = _mm512_setzero_ph(); - _mm512_store_ph(addr_of_mut!(b).cast(), a); + unsafe { + _mm512_store_ph(addr_of_mut!(b).cast(), a); + } assert_eq_m512h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm_store_sh() { + const fn test_mm_store_sh() { let a = _mm_set_sh(1.0); let mut b = _mm_setzero_ph(); - _mm_store_sh(addr_of_mut!(b).cast(), a); + unsafe { + _mm_store_sh(addr_of_mut!(b).cast(), a); + } assert_eq_m128h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - unsafe fn test_mm_mask_store_sh() { + fn test_mm_mask_store_sh() { let a = _mm_set_sh(1.0); let mut b = _mm_setzero_ph(); - _mm_mask_store_sh(addr_of_mut!(b).cast(), 0, a); + unsafe { + _mm_mask_store_sh(addr_of_mut!(b).cast(), 0, a); + } assert_eq_m128h(_mm_setzero_ph(), b); - _mm_mask_store_sh(addr_of_mut!(b).cast(), 1, a); + unsafe { + _mm_mask_store_sh(addr_of_mut!(b).cast(), 1, a); + } assert_eq_m128h(a, b); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm_storeu_ph() { + const fn test_mm_storeu_ph() { let a = _mm_set_ph(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0); let mut array = [0.0; 8]; - _mm_storeu_ph(array.as_mut_ptr(), a); - assert_eq_m128h(a, _mm_loadu_ph(array.as_ptr())); + unsafe { + _mm_storeu_ph(array.as_mut_ptr(), a); + } + assert_eq_m128h(a, unsafe { _mm_loadu_ph(array.as_ptr()) }); } #[simd_test(enable = "avx512fp16,avx512vl")] - const unsafe fn test_mm256_storeu_ph() { + const fn test_mm256_storeu_ph() { let a = _mm256_set_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ); let mut array = [0.0; 16]; - _mm256_storeu_ph(array.as_mut_ptr(), a); - assert_eq_m256h(a, _mm256_loadu_ph(array.as_ptr())); + unsafe { + _mm256_storeu_ph(array.as_mut_ptr(), a); + } + assert_eq_m256h(a, unsafe { _mm256_loadu_ph(array.as_ptr()) }); } #[simd_test(enable = "avx512fp16")] - const unsafe fn test_mm512_storeu_ph() { + const fn test_mm512_storeu_ph() { let a = _mm512_set_ph( 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, ); let mut array = [0.0; 32]; - _mm512_storeu_ph(array.as_mut_ptr(), a); - assert_eq_m512h(a, _mm512_loadu_ph(array.as_ptr())); + unsafe { + _mm512_storeu_ph(array.as_mut_ptr(), a); + } + assert_eq_m512h(a, unsafe { _mm512_loadu_ph(array.as_ptr()) }); } #[simd_test(enable = "avx512fp16,avx512vl")] diff --git a/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs b/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs index 26cef5814e9c..78a50b90c861 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs @@ -3932,7 +3932,7 @@ mod tests { } #[simd_test(enable = "avx512vbmi2")] - unsafe fn test_mm512_mask_expandloadu_epi16() { + fn test_mm512_mask_expandloadu_epi16() { let src = _mm512_set1_epi16(42); let a = &[ 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -3940,7 +3940,7 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b11101000_11001010_11110000_00001111; - let r = _mm512_mask_expandloadu_epi16(src, m, black_box(p)); + let r = unsafe { _mm512_mask_expandloadu_epi16(src, m, black_box(p)) }; let e = _mm512_set_epi16( 16, 15, 14, 42, 13, 42, 42, 42, 12, 11, 42, 42, 10, 42, 9, 42, 8, 7, 6, 5, 42, 42, 42, 42, 42, 42, 42, 42, 4, 3, 2, 1, @@ -3949,14 +3949,14 @@ mod tests { } #[simd_test(enable = "avx512vbmi2")] - unsafe fn test_mm512_maskz_expandloadu_epi16() { + fn test_mm512_maskz_expandloadu_epi16() { let a = &[ 1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ]; let p = a.as_ptr(); let m = 0b11101000_11001010_11110000_00001111; - let r = _mm512_maskz_expandloadu_epi16(m, black_box(p)); + let r = unsafe { _mm512_maskz_expandloadu_epi16(m, black_box(p)) }; let e = _mm512_set_epi16( 16, 15, 14, 0, 13, 0, 0, 0, 12, 11, 0, 0, 10, 0, 9, 0, 8, 7, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 1, @@ -3965,49 +3965,49 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm256_mask_expandloadu_epi16() { + fn test_mm256_mask_expandloadu_epi16() { let src = _mm256_set1_epi16(42); let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm256_mask_expandloadu_epi16(src, m, black_box(p)); + let r = unsafe { _mm256_mask_expandloadu_epi16(src, m, black_box(p)) }; let e = _mm256_set_epi16(8, 7, 6, 42, 5, 42, 42, 42, 4, 3, 42, 42, 2, 42, 1, 42); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm256_maskz_expandloadu_epi16() { + fn test_mm256_maskz_expandloadu_epi16() { let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm256_maskz_expandloadu_epi16(m, black_box(p)); + let r = unsafe { _mm256_maskz_expandloadu_epi16(m, black_box(p)) }; let e = _mm256_set_epi16(8, 7, 6, 0, 5, 0, 0, 0, 4, 3, 0, 0, 2, 0, 1, 0); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm_mask_expandloadu_epi16() { + fn test_mm_mask_expandloadu_epi16() { let src = _mm_set1_epi16(42); let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_mask_expandloadu_epi16(src, m, black_box(p)); + let r = unsafe { _mm_mask_expandloadu_epi16(src, m, black_box(p)) }; let e = _mm_set_epi16(4, 3, 2, 42, 1, 42, 42, 42); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm_maskz_expandloadu_epi16() { + fn test_mm_maskz_expandloadu_epi16() { let a = &[1_i16, 2, 3, 4, 5, 6, 7, 8]; let p = a.as_ptr(); let m = 0b11101000; - let r = _mm_maskz_expandloadu_epi16(m, black_box(p)); + let r = unsafe { _mm_maskz_expandloadu_epi16(m, black_box(p)) }; let e = _mm_set_epi16(4, 3, 2, 0, 1, 0, 0, 0); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512vbmi2")] - unsafe fn test_mm512_mask_expandloadu_epi8() { + fn test_mm512_mask_expandloadu_epi8() { let src = _mm512_set1_epi8(42); let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -4016,7 +4016,7 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b11101000_11001010_11110000_00001111_11111111_00000000_10101010_01010101; - let r = _mm512_mask_expandloadu_epi8(src, m, black_box(p)); + let r = unsafe { _mm512_mask_expandloadu_epi8(src, m, black_box(p)) }; let e = _mm512_set_epi8( 32, 31, 30, 42, 29, 42, 42, 42, 28, 27, 42, 42, 26, 42, 25, 42, 24, 23, 22, 21, 42, 42, 42, 42, 42, 42, 42, 42, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 42, 42, 42, 42, @@ -4026,7 +4026,7 @@ mod tests { } #[simd_test(enable = "avx512vbmi2")] - unsafe fn test_mm512_maskz_expandloadu_epi8() { + fn test_mm512_maskz_expandloadu_epi8() { let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, @@ -4034,7 +4034,7 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b11101000_11001010_11110000_00001111_11111111_00000000_10101010_01010101; - let r = _mm512_maskz_expandloadu_epi8(m, black_box(p)); + let r = unsafe { _mm512_maskz_expandloadu_epi8(m, black_box(p)) }; let e = _mm512_set_epi8( 32, 31, 30, 0, 29, 0, 0, 0, 28, 27, 0, 0, 26, 0, 25, 0, 24, 23, 22, 21, 0, 0, 0, 0, 0, 0, 0, 0, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, @@ -4044,7 +4044,7 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm256_mask_expandloadu_epi8() { + fn test_mm256_mask_expandloadu_epi8() { let src = _mm256_set1_epi8(42); let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -4052,7 +4052,7 @@ mod tests { ]; let p = a.as_ptr(); let m = 0b11101000_11001010_11110000_00001111; - let r = _mm256_mask_expandloadu_epi8(src, m, black_box(p)); + let r = unsafe { _mm256_mask_expandloadu_epi8(src, m, black_box(p)) }; let e = _mm256_set_epi8( 16, 15, 14, 42, 13, 42, 42, 42, 12, 11, 42, 42, 10, 42, 9, 42, 8, 7, 6, 5, 42, 42, 42, 42, 42, 42, 42, 42, 4, 3, 2, 1, @@ -4061,14 +4061,14 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm256_maskz_expandloadu_epi8() { + fn test_mm256_maskz_expandloadu_epi8() { let a = &[ 1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ]; let p = a.as_ptr(); let m = 0b11101000_11001010_11110000_00001111; - let r = _mm256_maskz_expandloadu_epi8(m, black_box(p)); + let r = unsafe { _mm256_maskz_expandloadu_epi8(m, black_box(p)) }; let e = _mm256_set_epi8( 16, 15, 14, 0, 13, 0, 0, 0, 12, 11, 0, 0, 10, 0, 9, 0, 8, 7, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 1, @@ -4077,36 +4077,44 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm_mask_expandloadu_epi8() { + fn test_mm_mask_expandloadu_epi8() { let src = _mm_set1_epi8(42); let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm_mask_expandloadu_epi8(src, m, black_box(p)); + let r = unsafe { _mm_mask_expandloadu_epi8(src, m, black_box(p)) }; let e = _mm_set_epi8(8, 7, 6, 42, 5, 42, 42, 42, 4, 3, 42, 42, 2, 42, 1, 42); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm_maskz_expandloadu_epi8() { + fn test_mm_maskz_expandloadu_epi8() { let a = &[1_i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let p = a.as_ptr(); let m = 0b11101000_11001010; - let r = _mm_maskz_expandloadu_epi8(m, black_box(p)); + let r = unsafe { _mm_maskz_expandloadu_epi8(m, black_box(p)) }; let e = _mm_set_epi8(8, 7, 6, 0, 5, 0, 0, 0, 4, 3, 0, 0, 2, 0, 1, 0); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512vbmi2")] - unsafe fn test_mm512_mask_compressstoreu_epi16() { + fn test_mm512_mask_compressstoreu_epi16() { let a = _mm512_set_epi16( 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, ); let mut r = [0_i16; 32]; - _mm512_mask_compressstoreu_epi16(r.as_mut_ptr(), 0, a); + unsafe { + _mm512_mask_compressstoreu_epi16(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i16; 32]); - _mm512_mask_compressstoreu_epi16(r.as_mut_ptr(), 0b11110000_11001010_11111111_00000000, a); + unsafe { + _mm512_mask_compressstoreu_epi16( + r.as_mut_ptr(), + 0b11110000_11001010_11111111_00000000, + a, + ); + } assert_eq!( &r, &[ @@ -4117,40 +4125,52 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm256_mask_compressstoreu_epi16() { + fn test_mm256_mask_compressstoreu_epi16() { let a = _mm256_set_epi16(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); let mut r = [0_i16; 16]; - _mm256_mask_compressstoreu_epi16(r.as_mut_ptr(), 0, a); + unsafe { + _mm256_mask_compressstoreu_epi16(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i16; 16]); - _mm256_mask_compressstoreu_epi16(r.as_mut_ptr(), 0b11110000_11001010, a); + unsafe { + _mm256_mask_compressstoreu_epi16(r.as_mut_ptr(), 0b11110000_11001010, a); + } assert_eq!(&r, &[2, 4, 7, 8, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0]); } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm_mask_compressstoreu_epi16() { + fn test_mm_mask_compressstoreu_epi16() { let a = _mm_set_epi16(8, 7, 6, 5, 4, 3, 2, 1); let mut r = [0_i16; 8]; - _mm_mask_compressstoreu_epi16(r.as_mut_ptr(), 0, a); + unsafe { + _mm_mask_compressstoreu_epi16(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i16; 8]); - _mm_mask_compressstoreu_epi16(r.as_mut_ptr(), 0b11110000, a); + unsafe { + _mm_mask_compressstoreu_epi16(r.as_mut_ptr(), 0b11110000, a); + } assert_eq!(&r, &[5, 6, 7, 8, 0, 0, 0, 0]); } #[simd_test(enable = "avx512vbmi2")] - unsafe fn test_mm512_mask_compressstoreu_epi8() { + fn test_mm512_mask_compressstoreu_epi8() { let a = _mm512_set_epi8( 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, ); let mut r = [0_i8; 64]; - _mm512_mask_compressstoreu_epi8(r.as_mut_ptr(), 0, a); + unsafe { + _mm512_mask_compressstoreu_epi8(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i8; 64]); - _mm512_mask_compressstoreu_epi8( - r.as_mut_ptr(), - 0b11110000_11001010_11111111_00000000_10101010_01010101_11110000_00001111, - a, - ); + unsafe { + _mm512_mask_compressstoreu_epi8( + r.as_mut_ptr(), + 0b11110000_11001010_11111111_00000000_10101010_01010101_11110000_00001111, + a, + ); + } assert_eq!( &r, &[ @@ -4162,15 +4182,23 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm256_mask_compressstoreu_epi8() { + fn test_mm256_mask_compressstoreu_epi8() { let a = _mm256_set_epi8( 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, ); let mut r = [0_i8; 32]; - _mm256_mask_compressstoreu_epi8(r.as_mut_ptr(), 0, a); + unsafe { + _mm256_mask_compressstoreu_epi8(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i8; 32]); - _mm256_mask_compressstoreu_epi8(r.as_mut_ptr(), 0b11110000_11001010_11111111_00000000, a); + unsafe { + _mm256_mask_compressstoreu_epi8( + r.as_mut_ptr(), + 0b11110000_11001010_11111111_00000000, + a, + ); + } assert_eq!( &r, &[ @@ -4181,12 +4209,16 @@ mod tests { } #[simd_test(enable = "avx512vbmi2,avx512vl")] - unsafe fn test_mm_mask_compressstoreu_epi8() { + fn test_mm_mask_compressstoreu_epi8() { let a = _mm_set_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); let mut r = [0_i8; 16]; - _mm_mask_compressstoreu_epi8(r.as_mut_ptr(), 0, a); + unsafe { + _mm_mask_compressstoreu_epi8(r.as_mut_ptr(), 0, a); + } assert_eq!(&r, &[0_i8; 16]); - _mm_mask_compressstoreu_epi8(r.as_mut_ptr(), 0b11110000_11001010, a); + unsafe { + _mm_mask_compressstoreu_epi8(r.as_mut_ptr(), 0b11110000_11001010, a); + } assert_eq!(&r, &[2, 4, 7, 8, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0]); } } diff --git a/library/stdarch/crates/core_arch/src/x86/avxneconvert.rs b/library/stdarch/crates/core_arch/src/x86/avxneconvert.rs index a53d6d97e8a0..91b6be2b09d7 100644 --- a/library/stdarch/crates/core_arch/src/x86/avxneconvert.rs +++ b/library/stdarch/crates/core_arch/src/x86/avxneconvert.rs @@ -242,127 +242,127 @@ mod tests { const BF16_EIGHT: u16 = 0b0_10000010_0000000; #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_bcstnebf16_ps() { + fn test_mm_bcstnebf16_ps() { let a = bf16::from_bits(BF16_ONE); - let r = _mm_bcstnebf16_ps(addr_of!(a)); + let r = unsafe { _mm_bcstnebf16_ps(addr_of!(a)) }; let e = _mm_set_ps(1., 1., 1., 1.); assert_eq_m128(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_bcstnebf16_ps() { + fn test_mm256_bcstnebf16_ps() { let a = bf16::from_bits(BF16_ONE); - let r = _mm256_bcstnebf16_ps(addr_of!(a)); + let r = unsafe { _mm256_bcstnebf16_ps(addr_of!(a)) }; let e = _mm256_set_ps(1., 1., 1., 1., 1., 1., 1., 1.); assert_eq_m256(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_bcstnesh_ps() { + fn test_mm_bcstnesh_ps() { let a = 1.0_f16; - let r = _mm_bcstnesh_ps(addr_of!(a)); + let r = unsafe { _mm_bcstnesh_ps(addr_of!(a)) }; let e = _mm_set_ps(1., 1., 1., 1.); assert_eq_m128(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_bcstnesh_ps() { + fn test_mm256_bcstnesh_ps() { let a = 1.0_f16; - let r = _mm256_bcstnesh_ps(addr_of!(a)); + let r = unsafe { _mm256_bcstnesh_ps(addr_of!(a)) }; let e = _mm256_set_ps(1., 1., 1., 1., 1., 1., 1., 1.); assert_eq_m256(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_cvtneebf16_ps() { + fn test_mm_cvtneebf16_ps() { let a = __m128bh([ BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, ]); - let r = _mm_cvtneebf16_ps(addr_of!(a)); + let r = unsafe { _mm_cvtneebf16_ps(addr_of!(a)) }; let e = _mm_setr_ps(1., 3., 5., 7.); assert_eq_m128(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_cvtneebf16_ps() { + fn test_mm256_cvtneebf16_ps() { let a = __m256bh([ BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, ]); - let r = _mm256_cvtneebf16_ps(addr_of!(a)); + let r = unsafe { _mm256_cvtneebf16_ps(addr_of!(a)) }; let e = _mm256_setr_ps(1., 3., 5., 7., 1., 3., 5., 7.); assert_eq_m256(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_cvtneeph_ps() { + fn test_mm_cvtneeph_ps() { let a = __m128h([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); - let r = _mm_cvtneeph_ps(addr_of!(a)); + let r = unsafe { _mm_cvtneeph_ps(addr_of!(a)) }; let e = _mm_setr_ps(1., 3., 5., 7.); assert_eq_m128(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_cvtneeph_ps() { + fn test_mm256_cvtneeph_ps() { let a = __m256h([ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ]); - let r = _mm256_cvtneeph_ps(addr_of!(a)); + let r = unsafe { _mm256_cvtneeph_ps(addr_of!(a)) }; let e = _mm256_setr_ps(1., 3., 5., 7., 9., 11., 13., 15.); assert_eq_m256(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_cvtneobf16_ps() { + fn test_mm_cvtneobf16_ps() { let a = __m128bh([ BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, ]); - let r = _mm_cvtneobf16_ps(addr_of!(a)); + let r = unsafe { _mm_cvtneobf16_ps(addr_of!(a)) }; let e = _mm_setr_ps(2., 4., 6., 8.); assert_eq_m128(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_cvtneobf16_ps() { + fn test_mm256_cvtneobf16_ps() { let a = __m256bh([ BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, ]); - let r = _mm256_cvtneobf16_ps(addr_of!(a)); + let r = unsafe { _mm256_cvtneobf16_ps(addr_of!(a)) }; let e = _mm256_setr_ps(2., 4., 6., 8., 2., 4., 6., 8.); assert_eq_m256(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_cvtneoph_ps() { + fn test_mm_cvtneoph_ps() { let a = __m128h([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); - let r = _mm_cvtneoph_ps(addr_of!(a)); + let r = unsafe { _mm_cvtneoph_ps(addr_of!(a)) }; let e = _mm_setr_ps(2., 4., 6., 8.); assert_eq_m128(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_cvtneoph_ps() { + fn test_mm256_cvtneoph_ps() { let a = __m256h([ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, ]); - let r = _mm256_cvtneoph_ps(addr_of!(a)); + let r = unsafe { _mm256_cvtneoph_ps(addr_of!(a)) }; let e = _mm256_setr_ps(2., 4., 6., 8., 10., 12., 14., 16.); assert_eq_m256(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm_cvtneps_avx_pbh() { + fn test_mm_cvtneps_avx_pbh() { let a = _mm_setr_ps(1., 2., 3., 4.); - let r: u16x4 = transmute_copy(&_mm_cvtneps_avx_pbh(a)); + let r: u16x4 = unsafe { transmute_copy(&_mm_cvtneps_avx_pbh(a)) }; let e = u16x4::new(BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR); assert_eq!(r, e); } #[simd_test(enable = "avxneconvert")] - unsafe fn test_mm256_cvtneps_avx_pbh() { + fn test_mm256_cvtneps_avx_pbh() { let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - let r: u16x8 = transmute(_mm256_cvtneps_avx_pbh(a)); + let r: u16x8 = _mm256_cvtneps_avx_pbh(a).as_u16x8(); let e = u16x8::new( BF16_ONE, BF16_TWO, BF16_THREE, BF16_FOUR, BF16_FIVE, BF16_SIX, BF16_SEVEN, BF16_EIGHT, ); diff --git a/library/stdarch/crates/core_arch/src/x86/f16c.rs b/library/stdarch/crates/core_arch/src/x86/f16c.rs index 0a26a9ff8d25..a0bb992bb9d4 100644 --- a/library/stdarch/crates/core_arch/src/x86/f16c.rs +++ b/library/stdarch/crates/core_arch/src/x86/f16c.rs @@ -106,7 +106,7 @@ pub fn _mm256_cvtps_ph(a: __m256) -> __m128i { #[cfg(test)] mod tests { use crate::core_arch::assert_eq_const as assert_eq; - use crate::{core_arch::x86::*, mem::transmute}; + use crate::core_arch::x86::*; use stdarch_test::simd_test; const F16_ONE: i16 = 0x3c00; diff --git a/library/stdarch/crates/core_arch/src/x86/fxsr.rs b/library/stdarch/crates/core_arch/src/x86/fxsr.rs index 71fd52ca1496..08619efe7c9e 100644 --- a/library/stdarch/crates/core_arch/src/x86/fxsr.rs +++ b/library/stdarch/crates/core_arch/src/x86/fxsr.rs @@ -77,12 +77,14 @@ mod tests { #[simd_test(enable = "fxsr")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_fxsave() { + fn test_fxsave() { let mut a = FxsaveArea::new(); let mut b = FxsaveArea::new(); - fxsr::_fxsave(a.ptr()); - fxsr::_fxrstor(a.ptr()); - fxsr::_fxsave(b.ptr()); + unsafe { + fxsr::_fxsave(a.ptr()); + fxsr::_fxrstor(a.ptr()); + fxsr::_fxsave(b.ptr()); + } } } diff --git a/library/stdarch/crates/core_arch/src/x86/gfni.rs b/library/stdarch/crates/core_arch/src/x86/gfni.rs index 681b8ae330d4..e9ee27a7b823 100644 --- a/library/stdarch/crates/core_arch/src/x86/gfni.rs +++ b/library/stdarch/crates/core_arch/src/x86/gfni.rs @@ -898,25 +898,25 @@ mod tests { } #[simd_test(enable = "gfni,avx512f")] - unsafe fn test_mm512_gf2p8mul_epi8() { + fn test_mm512_gf2p8mul_epi8() { let (left, right, expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_512 { - let left = load_m512i_word(&left, i); - let right = load_m512i_word(&right, i); - let expected = load_m512i_word(&expected, i); + let left = unsafe { load_m512i_word(&left, i) }; + let right = unsafe { load_m512i_word(&right, i) }; + let expected = unsafe { load_m512i_word(&expected, i) }; let result = _mm512_gf2p8mul_epi8(left, right); assert_eq_m512i(result, expected); } } #[simd_test(enable = "gfni,avx512bw")] - unsafe fn test_mm512_maskz_gf2p8mul_epi8() { + fn test_mm512_maskz_gf2p8mul_epi8() { let (left, right, _expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_512 { - let left = load_m512i_word(&left, i); - let right = load_m512i_word(&right, i); + let left = unsafe { load_m512i_word(&left, i) }; + let right = unsafe { load_m512i_word(&right, i) }; let result_zero = _mm512_maskz_gf2p8mul_epi8(0, left, right); assert_eq_m512i(result_zero, _mm512_setzero_si512()); let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; @@ -930,12 +930,12 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw")] - unsafe fn test_mm512_mask_gf2p8mul_epi8() { + fn test_mm512_mask_gf2p8mul_epi8() { let (left, right, _expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_512 { - let left = load_m512i_word(&left, i); - let right = load_m512i_word(&right, i); + let left = unsafe { load_m512i_word(&left, i) }; + let right = unsafe { load_m512i_word(&right, i) }; let result_left = _mm512_mask_gf2p8mul_epi8(left, 0, left, right); assert_eq_m512i(result_left, left); let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; @@ -948,25 +948,25 @@ mod tests { } #[simd_test(enable = "gfni,avx")] - unsafe fn test_mm256_gf2p8mul_epi8() { + fn test_mm256_gf2p8mul_epi8() { let (left, right, expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_256 { - let left = load_m256i_word(&left, i); - let right = load_m256i_word(&right, i); - let expected = load_m256i_word(&expected, i); + let left = unsafe { load_m256i_word(&left, i) }; + let right = unsafe { load_m256i_word(&right, i) }; + let expected = unsafe { load_m256i_word(&expected, i) }; let result = _mm256_gf2p8mul_epi8(left, right); assert_eq_m256i(result, expected); } } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm256_maskz_gf2p8mul_epi8() { + fn test_mm256_maskz_gf2p8mul_epi8() { let (left, right, _expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_256 { - let left = load_m256i_word(&left, i); - let right = load_m256i_word(&right, i); + let left = unsafe { load_m256i_word(&left, i) }; + let right = unsafe { load_m256i_word(&right, i) }; let result_zero = _mm256_maskz_gf2p8mul_epi8(0, left, right); assert_eq_m256i(result_zero, _mm256_setzero_si256()); let mask_bytes: __mmask32 = 0x0F_F0_FF_00; @@ -980,12 +980,12 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm256_mask_gf2p8mul_epi8() { + fn test_mm256_mask_gf2p8mul_epi8() { let (left, right, _expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_256 { - let left = load_m256i_word(&left, i); - let right = load_m256i_word(&right, i); + let left = unsafe { load_m256i_word(&left, i) }; + let right = unsafe { load_m256i_word(&right, i) }; let result_left = _mm256_mask_gf2p8mul_epi8(left, 0, left, right); assert_eq_m256i(result_left, left); let mask_bytes: __mmask32 = 0x0F_F0_FF_00; @@ -998,25 +998,25 @@ mod tests { } #[simd_test(enable = "gfni")] - unsafe fn test_mm_gf2p8mul_epi8() { + fn test_mm_gf2p8mul_epi8() { let (left, right, expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_128 { - let left = load_m128i_word(&left, i); - let right = load_m128i_word(&right, i); - let expected = load_m128i_word(&expected, i); + let left = unsafe { load_m128i_word(&left, i) }; + let right = unsafe { load_m128i_word(&right, i) }; + let expected = unsafe { load_m128i_word(&expected, i) }; let result = _mm_gf2p8mul_epi8(left, right); assert_eq_m128i(result, expected); } } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm_maskz_gf2p8mul_epi8() { + fn test_mm_maskz_gf2p8mul_epi8() { let (left, right, _expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_128 { - let left = load_m128i_word(&left, i); - let right = load_m128i_word(&right, i); + let left = unsafe { load_m128i_word(&left, i) }; + let right = unsafe { load_m128i_word(&right, i) }; let result_zero = _mm_maskz_gf2p8mul_epi8(0, left, right); assert_eq_m128i(result_zero, _mm_setzero_si128()); let mask_bytes: __mmask16 = 0x0F_F0; @@ -1030,12 +1030,12 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm_mask_gf2p8mul_epi8() { + fn test_mm_mask_gf2p8mul_epi8() { let (left, right, _expected) = generate_byte_mul_test_data(); for i in 0..NUM_TEST_WORDS_128 { - let left = load_m128i_word(&left, i); - let right = load_m128i_word(&right, i); + let left = unsafe { load_m128i_word(&left, i) }; + let right = unsafe { load_m128i_word(&right, i) }; let result_left = _mm_mask_gf2p8mul_epi8(left, 0, left, right); assert_eq_m128i(result_left, left); let mask_bytes: __mmask16 = 0x0F_F0; @@ -1048,7 +1048,7 @@ mod tests { } #[simd_test(enable = "gfni,avx512f")] - unsafe fn test_mm512_gf2p8affine_epi64_epi8() { + fn test_mm512_gf2p8affine_epi64_epi8() { let identity: i64 = 0x01_02_04_08_10_20_40_80; const IDENTITY_BYTE: i32 = 0; let constant: i64 = 0; @@ -1061,20 +1061,20 @@ mod tests { let (matrices, vectors, references) = generate_affine_mul_test_data(IDENTITY_BYTE as u8); for i in 0..NUM_TEST_WORDS_512 { - let data = load_m512i_word(&bytes, i); + let data = unsafe { load_m512i_word(&bytes, i) }; let result = _mm512_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m512i(result, data); let result = _mm512_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m512i(result, constant_reference); - let data = load_m512i_word(&more_bytes, i); + let data = unsafe { load_m512i_word(&more_bytes, i) }; let result = _mm512_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m512i(result, data); let result = _mm512_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m512i(result, constant_reference); - let matrix = load_m512i_word(&matrices, i); - let vector = load_m512i_word(&vectors, i); - let reference = load_m512i_word(&references, i); + let matrix = unsafe { load_m512i_word(&matrices, i) }; + let vector = unsafe { load_m512i_word(&vectors, i) }; + let reference = unsafe { load_m512i_word(&references, i) }; let result = _mm512_gf2p8affine_epi64_epi8::(vector, matrix); assert_eq_m512i(result, reference); @@ -1082,13 +1082,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw")] - unsafe fn test_mm512_maskz_gf2p8affine_epi64_epi8() { + fn test_mm512_maskz_gf2p8affine_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_512 { - let matrix = load_m512i_word(&matrices, i); - let vector = load_m512i_word(&vectors, i); + let matrix = unsafe { load_m512i_word(&matrices, i) }; + let vector = unsafe { load_m512i_word(&vectors, i) }; let result_zero = _mm512_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); assert_eq_m512i(result_zero, _mm512_setzero_si512()); @@ -1104,13 +1104,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw")] - unsafe fn test_mm512_mask_gf2p8affine_epi64_epi8() { + fn test_mm512_mask_gf2p8affine_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_512 { - let left = load_m512i_word(&vectors, i); - let right = load_m512i_word(&matrices, i); + let left = unsafe { load_m512i_word(&vectors, i) }; + let right = unsafe { load_m512i_word(&matrices, i) }; let result_left = _mm512_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); assert_eq_m512i(result_left, left); @@ -1125,7 +1125,7 @@ mod tests { } #[simd_test(enable = "gfni,avx")] - unsafe fn test_mm256_gf2p8affine_epi64_epi8() { + fn test_mm256_gf2p8affine_epi64_epi8() { let identity: i64 = 0x01_02_04_08_10_20_40_80; const IDENTITY_BYTE: i32 = 0; let constant: i64 = 0; @@ -1138,20 +1138,20 @@ mod tests { let (matrices, vectors, references) = generate_affine_mul_test_data(IDENTITY_BYTE as u8); for i in 0..NUM_TEST_WORDS_256 { - let data = load_m256i_word(&bytes, i); + let data = unsafe { load_m256i_word(&bytes, i) }; let result = _mm256_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m256i(result, data); let result = _mm256_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m256i(result, constant_reference); - let data = load_m256i_word(&more_bytes, i); + let data = unsafe { load_m256i_word(&more_bytes, i) }; let result = _mm256_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m256i(result, data); let result = _mm256_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m256i(result, constant_reference); - let matrix = load_m256i_word(&matrices, i); - let vector = load_m256i_word(&vectors, i); - let reference = load_m256i_word(&references, i); + let matrix = unsafe { load_m256i_word(&matrices, i) }; + let vector = unsafe { load_m256i_word(&vectors, i) }; + let reference = unsafe { load_m256i_word(&references, i) }; let result = _mm256_gf2p8affine_epi64_epi8::(vector, matrix); assert_eq_m256i(result, reference); @@ -1159,13 +1159,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm256_maskz_gf2p8affine_epi64_epi8() { + fn test_mm256_maskz_gf2p8affine_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_256 { - let matrix = load_m256i_word(&matrices, i); - let vector = load_m256i_word(&vectors, i); + let matrix = unsafe { load_m256i_word(&matrices, i) }; + let vector = unsafe { load_m256i_word(&vectors, i) }; let result_zero = _mm256_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); assert_eq_m256i(result_zero, _mm256_setzero_si256()); @@ -1181,13 +1181,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm256_mask_gf2p8affine_epi64_epi8() { + fn test_mm256_mask_gf2p8affine_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_256 { - let left = load_m256i_word(&vectors, i); - let right = load_m256i_word(&matrices, i); + let left = unsafe { load_m256i_word(&vectors, i) }; + let right = unsafe { load_m256i_word(&matrices, i) }; let result_left = _mm256_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); assert_eq_m256i(result_left, left); @@ -1202,7 +1202,7 @@ mod tests { } #[simd_test(enable = "gfni")] - unsafe fn test_mm_gf2p8affine_epi64_epi8() { + fn test_mm_gf2p8affine_epi64_epi8() { let identity: i64 = 0x01_02_04_08_10_20_40_80; const IDENTITY_BYTE: i32 = 0; let constant: i64 = 0; @@ -1215,20 +1215,20 @@ mod tests { let (matrices, vectors, references) = generate_affine_mul_test_data(IDENTITY_BYTE as u8); for i in 0..NUM_TEST_WORDS_128 { - let data = load_m128i_word(&bytes, i); + let data = unsafe { load_m128i_word(&bytes, i) }; let result = _mm_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m128i(result, data); let result = _mm_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m128i(result, constant_reference); - let data = load_m128i_word(&more_bytes, i); + let data = unsafe { load_m128i_word(&more_bytes, i) }; let result = _mm_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m128i(result, data); let result = _mm_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m128i(result, constant_reference); - let matrix = load_m128i_word(&matrices, i); - let vector = load_m128i_word(&vectors, i); - let reference = load_m128i_word(&references, i); + let matrix = unsafe { load_m128i_word(&matrices, i) }; + let vector = unsafe { load_m128i_word(&vectors, i) }; + let reference = unsafe { load_m128i_word(&references, i) }; let result = _mm_gf2p8affine_epi64_epi8::(vector, matrix); assert_eq_m128i(result, reference); @@ -1236,13 +1236,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm_maskz_gf2p8affine_epi64_epi8() { + fn test_mm_maskz_gf2p8affine_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_128 { - let matrix = load_m128i_word(&matrices, i); - let vector = load_m128i_word(&vectors, i); + let matrix = unsafe { load_m128i_word(&matrices, i) }; + let vector = unsafe { load_m128i_word(&vectors, i) }; let result_zero = _mm_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); assert_eq_m128i(result_zero, _mm_setzero_si128()); let mask_bytes: __mmask16 = 0x0F_F0; @@ -1257,13 +1257,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm_mask_gf2p8affine_epi64_epi8() { + fn test_mm_mask_gf2p8affine_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_128 { - let left = load_m128i_word(&vectors, i); - let right = load_m128i_word(&matrices, i); + let left = unsafe { load_m128i_word(&vectors, i) }; + let right = unsafe { load_m128i_word(&matrices, i) }; let result_left = _mm_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); assert_eq_m128i(result_left, left); @@ -1278,7 +1278,7 @@ mod tests { } #[simd_test(enable = "gfni,avx512f")] - unsafe fn test_mm512_gf2p8affineinv_epi64_epi8() { + fn test_mm512_gf2p8affineinv_epi64_epi8() { let identity: i64 = 0x01_02_04_08_10_20_40_80; const IDENTITY_BYTE: i32 = 0; const CONSTANT_BYTE: i32 = 0x63; @@ -1288,8 +1288,8 @@ mod tests { let (inputs, results) = generate_inv_tests_data(); for i in 0..NUM_BYTES_WORDS_512 { - let input = load_m512i_word(&inputs, i); - let reference = load_m512i_word(&results, i); + let input = unsafe { load_m512i_word(&inputs, i) }; + let reference = unsafe { load_m512i_word(&results, i) }; let result = _mm512_gf2p8affineinv_epi64_epi8::(input, identity); let remultiplied = _mm512_gf2p8mul_epi8(result, input); assert_eq_m512i(remultiplied, reference); @@ -1300,8 +1300,8 @@ mod tests { generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_512 { - let vector = load_m512i_word(&vectors, i); - let matrix = load_m512i_word(&matrices, i); + let vector = unsafe { load_m512i_word(&vectors, i) }; + let matrix = unsafe { load_m512i_word(&matrices, i) }; let inv_vec = _mm512_gf2p8affineinv_epi64_epi8::(vector, identity); let reference = _mm512_gf2p8affine_epi64_epi8::(inv_vec, matrix); @@ -1314,21 +1314,21 @@ mod tests { let sbox_matrix = _mm512_set1_epi64(AES_S_BOX_MATRIX); for i in 0..NUM_BYTES_WORDS_512 { - let reference = load_m512i_word(&AES_S_BOX, i); - let input = load_m512i_word(&inputs, i); + let reference = unsafe { load_m512i_word(&AES_S_BOX, i) }; + let input = unsafe { load_m512i_word(&inputs, i) }; let result = _mm512_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); assert_eq_m512i(result, reference); } } #[simd_test(enable = "gfni,avx512bw")] - unsafe fn test_mm512_maskz_gf2p8affineinv_epi64_epi8() { + fn test_mm512_maskz_gf2p8affineinv_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_512 { - let matrix = load_m512i_word(&matrices, i); - let vector = load_m512i_word(&vectors, i); + let matrix = unsafe { load_m512i_word(&matrices, i) }; + let vector = unsafe { load_m512i_word(&vectors, i) }; let result_zero = _mm512_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); assert_eq_m512i(result_zero, _mm512_setzero_si512()); @@ -1344,13 +1344,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw")] - unsafe fn test_mm512_mask_gf2p8affineinv_epi64_epi8() { + fn test_mm512_mask_gf2p8affineinv_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_512 { - let left = load_m512i_word(&vectors, i); - let right = load_m512i_word(&matrices, i); + let left = unsafe { load_m512i_word(&vectors, i) }; + let right = unsafe { load_m512i_word(&matrices, i) }; let result_left = _mm512_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); assert_eq_m512i(result_left, left); @@ -1366,7 +1366,7 @@ mod tests { } #[simd_test(enable = "gfni,avx")] - unsafe fn test_mm256_gf2p8affineinv_epi64_epi8() { + fn test_mm256_gf2p8affineinv_epi64_epi8() { let identity: i64 = 0x01_02_04_08_10_20_40_80; const IDENTITY_BYTE: i32 = 0; const CONSTANT_BYTE: i32 = 0x63; @@ -1376,8 +1376,8 @@ mod tests { let (inputs, results) = generate_inv_tests_data(); for i in 0..NUM_BYTES_WORDS_256 { - let input = load_m256i_word(&inputs, i); - let reference = load_m256i_word(&results, i); + let input = unsafe { load_m256i_word(&inputs, i) }; + let reference = unsafe { load_m256i_word(&results, i) }; let result = _mm256_gf2p8affineinv_epi64_epi8::(input, identity); let remultiplied = _mm256_gf2p8mul_epi8(result, input); assert_eq_m256i(remultiplied, reference); @@ -1388,8 +1388,8 @@ mod tests { generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_256 { - let vector = load_m256i_word(&vectors, i); - let matrix = load_m256i_word(&matrices, i); + let vector = unsafe { load_m256i_word(&vectors, i) }; + let matrix = unsafe { load_m256i_word(&matrices, i) }; let inv_vec = _mm256_gf2p8affineinv_epi64_epi8::(vector, identity); let reference = _mm256_gf2p8affine_epi64_epi8::(inv_vec, matrix); @@ -1402,21 +1402,21 @@ mod tests { let sbox_matrix = _mm256_set1_epi64x(AES_S_BOX_MATRIX); for i in 0..NUM_BYTES_WORDS_256 { - let reference = load_m256i_word(&AES_S_BOX, i); - let input = load_m256i_word(&inputs, i); + let reference = unsafe { load_m256i_word(&AES_S_BOX, i) }; + let input = unsafe { load_m256i_word(&inputs, i) }; let result = _mm256_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); assert_eq_m256i(result, reference); } } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm256_maskz_gf2p8affineinv_epi64_epi8() { + fn test_mm256_maskz_gf2p8affineinv_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_256 { - let matrix = load_m256i_word(&matrices, i); - let vector = load_m256i_word(&vectors, i); + let matrix = unsafe { load_m256i_word(&matrices, i) }; + let vector = unsafe { load_m256i_word(&vectors, i) }; let result_zero = _mm256_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); assert_eq_m256i(result_zero, _mm256_setzero_si256()); @@ -1432,13 +1432,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm256_mask_gf2p8affineinv_epi64_epi8() { + fn test_mm256_mask_gf2p8affineinv_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_256 { - let left = load_m256i_word(&vectors, i); - let right = load_m256i_word(&matrices, i); + let left = unsafe { load_m256i_word(&vectors, i) }; + let right = unsafe { load_m256i_word(&matrices, i) }; let result_left = _mm256_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); assert_eq_m256i(result_left, left); @@ -1454,7 +1454,7 @@ mod tests { } #[simd_test(enable = "gfni")] - unsafe fn test_mm_gf2p8affineinv_epi64_epi8() { + fn test_mm_gf2p8affineinv_epi64_epi8() { let identity: i64 = 0x01_02_04_08_10_20_40_80; const IDENTITY_BYTE: i32 = 0; const CONSTANT_BYTE: i32 = 0x63; @@ -1464,8 +1464,8 @@ mod tests { let (inputs, results) = generate_inv_tests_data(); for i in 0..NUM_BYTES_WORDS_128 { - let input = load_m128i_word(&inputs, i); - let reference = load_m128i_word(&results, i); + let input = unsafe { load_m128i_word(&inputs, i) }; + let reference = unsafe { load_m128i_word(&results, i) }; let result = _mm_gf2p8affineinv_epi64_epi8::(input, identity); let remultiplied = _mm_gf2p8mul_epi8(result, input); assert_eq_m128i(remultiplied, reference); @@ -1476,8 +1476,8 @@ mod tests { generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_128 { - let vector = load_m128i_word(&vectors, i); - let matrix = load_m128i_word(&matrices, i); + let vector = unsafe { load_m128i_word(&vectors, i) }; + let matrix = unsafe { load_m128i_word(&matrices, i) }; let inv_vec = _mm_gf2p8affineinv_epi64_epi8::(vector, identity); let reference = _mm_gf2p8affine_epi64_epi8::(inv_vec, matrix); @@ -1490,21 +1490,21 @@ mod tests { let sbox_matrix = _mm_set1_epi64x(AES_S_BOX_MATRIX); for i in 0..NUM_BYTES_WORDS_128 { - let reference = load_m128i_word(&AES_S_BOX, i); - let input = load_m128i_word(&inputs, i); + let reference = unsafe { load_m128i_word(&AES_S_BOX, i) }; + let input = unsafe { load_m128i_word(&inputs, i) }; let result = _mm_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); assert_eq_m128i(result, reference); } } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm_maskz_gf2p8affineinv_epi64_epi8() { + fn test_mm_maskz_gf2p8affineinv_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_128 { - let matrix = load_m128i_word(&matrices, i); - let vector = load_m128i_word(&vectors, i); + let matrix = unsafe { load_m128i_word(&matrices, i) }; + let vector = unsafe { load_m128i_word(&vectors, i) }; let result_zero = _mm_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); assert_eq_m128i(result_zero, _mm_setzero_si128()); @@ -1520,13 +1520,13 @@ mod tests { } #[simd_test(enable = "gfni,avx512bw,avx512vl")] - unsafe fn test_mm_mask_gf2p8affineinv_epi64_epi8() { + fn test_mm_mask_gf2p8affineinv_epi64_epi8() { const CONSTANT_BYTE: i32 = 0x63; let (matrices, vectors, _expected) = generate_affine_mul_test_data(CONSTANT_BYTE as u8); for i in 0..NUM_TEST_WORDS_128 { - let left = load_m128i_word(&vectors, i); - let right = load_m128i_word(&matrices, i); + let left = unsafe { load_m128i_word(&vectors, i) }; + let right = unsafe { load_m128i_word(&matrices, i) }; let result_left = _mm_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); assert_eq_m128i(result_left, left); diff --git a/library/stdarch/crates/core_arch/src/x86/kl.rs b/library/stdarch/crates/core_arch/src/x86/kl.rs index 26e5a46c6293..7cb52847f50d 100644 --- a/library/stdarch/crates/core_arch/src/x86/kl.rs +++ b/library/stdarch/crates/core_arch/src/x86/kl.rs @@ -352,45 +352,47 @@ mod tests { use stdarch_test::simd_test; #[target_feature(enable = "kl")] - unsafe fn encodekey128() -> [u8; 48] { + fn encodekey128() -> [u8; 48] { let mut handle = [0; 48]; - let _ = _mm_encodekey128_u32(0, _mm_setzero_si128(), handle.as_mut_ptr()); + let _ = unsafe { _mm_encodekey128_u32(0, _mm_setzero_si128(), handle.as_mut_ptr()) }; handle } #[target_feature(enable = "kl")] - unsafe fn encodekey256() -> [u8; 64] { + fn encodekey256() -> [u8; 64] { let mut handle = [0; 64]; - let _ = _mm_encodekey256_u32( - 0, - _mm_setzero_si128(), - _mm_setzero_si128(), - handle.as_mut_ptr(), - ); + let _ = unsafe { + _mm_encodekey256_u32( + 0, + _mm_setzero_si128(), + _mm_setzero_si128(), + handle.as_mut_ptr(), + ) + }; handle } #[simd_test(enable = "kl")] - unsafe fn test_mm_encodekey128_u32() { + fn test_mm_encodekey128_u32() { encodekey128(); } #[simd_test(enable = "kl")] - unsafe fn test_mm_encodekey256_u32() { + fn test_mm_encodekey256_u32() { encodekey256(); } #[simd_test(enable = "kl")] - unsafe fn test_mm_aesenc128kl_u8() { + fn test_mm_aesenc128kl_u8() { let mut buffer = _mm_setzero_si128(); let key = encodekey128(); for _ in 0..100 { - let status = _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } @@ -398,16 +400,16 @@ mod tests { } #[simd_test(enable = "kl")] - unsafe fn test_mm_aesdec128kl_u8() { + fn test_mm_aesdec128kl_u8() { let mut buffer = _mm_setzero_si128(); let key = encodekey128(); for _ in 0..100 { - let status = _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } @@ -415,16 +417,16 @@ mod tests { } #[simd_test(enable = "kl")] - unsafe fn test_mm_aesenc256kl_u8() { + fn test_mm_aesenc256kl_u8() { let mut buffer = _mm_setzero_si128(); let key = encodekey256(); for _ in 0..100 { - let status = _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } @@ -432,16 +434,16 @@ mod tests { } #[simd_test(enable = "kl")] - unsafe fn test_mm_aesdec256kl_u8() { + fn test_mm_aesdec256kl_u8() { let mut buffer = _mm_setzero_si128(); let key = encodekey256(); for _ in 0..100 { - let status = _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()); + let status = unsafe { _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()) }; assert_eq!(status, 0); } @@ -449,16 +451,20 @@ mod tests { } #[simd_test(enable = "widekl")] - unsafe fn test_mm_aesencwide128kl_u8() { + fn test_mm_aesencwide128kl_u8() { let mut buffer = [_mm_setzero_si128(); 8]; let key = encodekey128(); for _ in 0..100 { - let status = _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } @@ -468,16 +474,20 @@ mod tests { } #[simd_test(enable = "widekl")] - unsafe fn test_mm_aesdecwide128kl_u8() { + fn test_mm_aesdecwide128kl_u8() { let mut buffer = [_mm_setzero_si128(); 8]; let key = encodekey128(); for _ in 0..100 { - let status = _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } @@ -487,16 +497,20 @@ mod tests { } #[simd_test(enable = "widekl")] - unsafe fn test_mm_aesencwide256kl_u8() { + fn test_mm_aesencwide256kl_u8() { let mut buffer = [_mm_setzero_si128(); 8]; let key = encodekey256(); for _ in 0..100 { - let status = _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } @@ -506,16 +520,20 @@ mod tests { } #[simd_test(enable = "widekl")] - unsafe fn test_mm_aesdecwide256kl_u8() { + fn test_mm_aesdecwide256kl_u8() { let mut buffer = [_mm_setzero_si128(); 8]; let key = encodekey256(); for _ in 0..100 { - let status = _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } for _ in 0..100 { - let status = _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()); + let status = unsafe { + _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr()) + }; assert_eq!(status, 0); } diff --git a/library/stdarch/crates/core_arch/src/x86/rtm.rs b/library/stdarch/crates/core_arch/src/x86/rtm.rs index b37e7571eb8f..c88bd6592d78 100644 --- a/library/stdarch/crates/core_arch/src/x86/rtm.rs +++ b/library/stdarch/crates/core_arch/src/x86/rtm.rs @@ -120,13 +120,15 @@ mod tests { use crate::core_arch::x86::*; #[simd_test(enable = "rtm")] - unsafe fn test_xbegin() { + fn test_xbegin() { let mut x = 0; for _ in 0..10 { - let code = _xbegin(); + let code = unsafe { _xbegin() }; if code == _XBEGIN_STARTED { x += 1; - _xend(); + unsafe { + _xend(); + } assert_eq!(x, 1); break; } @@ -135,19 +137,23 @@ mod tests { } #[simd_test(enable = "rtm")] - unsafe fn test_xabort() { + fn test_xabort() { const ABORT_CODE: u32 = 42; // aborting outside a transactional region does nothing - _xabort::(); + unsafe { + _xabort::(); + } for _ in 0..10 { let mut x = 0; - let code = rtm::_xbegin(); + let code = unsafe { _xbegin() }; if code == _XBEGIN_STARTED { x += 1; - rtm::_xabort::(); + unsafe { + _xabort::(); + } } else if code & _XABORT_EXPLICIT != 0 { - let test_abort_code = rtm::_xabort_code(code); + let test_abort_code = _xabort_code(code); assert_eq!(test_abort_code, ABORT_CODE); } assert_eq!(x, 0); @@ -155,14 +161,16 @@ mod tests { } #[simd_test(enable = "rtm")] - unsafe fn test_xtest() { - assert_eq!(_xtest(), 0); + fn test_xtest() { + assert_eq!(unsafe { _xtest() }, 0); for _ in 0..10 { - let code = rtm::_xbegin(); + let code = unsafe { _xbegin() }; if code == _XBEGIN_STARTED { - let in_tx = _xtest(); - rtm::_xend(); + let in_tx = unsafe { _xtest() }; + unsafe { + _xend(); + } // putting the assert inside the transaction would abort the transaction on fail // without any output/panic/etc diff --git a/library/stdarch/crates/core_arch/src/x86/sse.rs b/library/stdarch/crates/core_arch/src/x86/sse.rs index f167e8381d27..1812721db4d3 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse.rs @@ -3147,21 +3147,21 @@ mod tests { } #[simd_test(enable = "sse")] - const unsafe fn test_mm_load_ss() { + const fn test_mm_load_ss() { let a = 42.0f32; - let r = _mm_load_ss(ptr::addr_of!(a)); + let r = unsafe { _mm_load_ss(ptr::addr_of!(a)) }; assert_eq_m128(r, _mm_setr_ps(42.0, 0.0, 0.0, 0.0)); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_load1_ps() { + const fn test_mm_load1_ps() { let a = 42.0f32; - let r = _mm_load1_ps(ptr::addr_of!(a)); + let r = unsafe { _mm_load1_ps(ptr::addr_of!(a)) }; assert_eq_m128(r, _mm_setr_ps(42.0, 42.0, 42.0, 42.0)); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_load_ps() { + const fn test_mm_load_ps() { let vals = Memory { data: [1.0f32, 2.0, 3.0, 4.0], }; @@ -3169,21 +3169,21 @@ mod tests { // guaranteed to be aligned to 16 bytes let p = vals.data.as_ptr(); - let r = _mm_load_ps(p); + let r = unsafe { _mm_load_ps(p) }; let e = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); assert_eq_m128(r, e); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_loadu_ps() { + const fn test_mm_loadu_ps() { let vals = &[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; - let p = vals.as_ptr().add(3); - let r = _mm_loadu_ps(black_box(p)); + let p = unsafe { vals.as_ptr().add(3) }; + let r = unsafe { _mm_loadu_ps(black_box(p)) }; assert_eq_m128(r, _mm_setr_ps(4.0, 5.0, 6.0, 7.0)); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_loadr_ps() { + const fn test_mm_loadr_ps() { let vals = Memory { data: [1.0f32, 2.0, 3.0, 4.0], }; @@ -3191,16 +3191,18 @@ mod tests { // guaranteed to be aligned to 16 bytes let p = vals.data.as_ptr(); - let r = _mm_loadr_ps(p); + let r = unsafe { _mm_loadr_ps(p) }; let e = _mm_setr_ps(4.0, 3.0, 2.0, 1.0); assert_eq_m128(r, e); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_store_ss() { + const fn test_mm_store_ss() { let mut vals = [0.0f32; 8]; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - _mm_store_ss(vals.as_mut_ptr().add(1), a); + unsafe { + _mm_store_ss(vals.as_mut_ptr().add(1), a); + } assert_eq!(vals[0], 0.0); assert_eq!(vals[1], 1.0); @@ -3208,46 +3210,52 @@ mod tests { } #[simd_test(enable = "sse")] - const unsafe fn test_mm_store1_ps() { + const fn test_mm_store1_ps() { let mut vals = Memory { data: [0.0f32; 4] }; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); // guaranteed to be aligned to 16 bytes let p = vals.data.as_mut_ptr(); - _mm_store1_ps(p, *black_box(&a)); + unsafe { + _mm_store1_ps(p, *black_box(&a)); + } assert_eq!(vals.data, [1.0, 1.0, 1.0, 1.0]); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_store_ps() { + const fn test_mm_store_ps() { let mut vals = Memory { data: [0.0f32; 4] }; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); // guaranteed to be aligned to 16 bytes let p = vals.data.as_mut_ptr(); - _mm_store_ps(p, *black_box(&a)); + unsafe { + _mm_store_ps(p, *black_box(&a)); + } assert_eq!(vals.data, [1.0, 2.0, 3.0, 4.0]); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_storer_ps() { + const fn test_mm_storer_ps() { let mut vals = Memory { data: [0.0f32; 4] }; let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); // guaranteed to be aligned to 16 bytes let p = vals.data.as_mut_ptr(); - _mm_storer_ps(p, *black_box(&a)); + unsafe { + _mm_storer_ps(p, *black_box(&a)); + } assert_eq!(vals.data, [4.0, 3.0, 2.0, 1.0]); } #[simd_test(enable = "sse")] - const unsafe fn test_mm_storeu_ps() { + const fn test_mm_storeu_ps() { #[repr(align(16))] struct Memory8 { data: [f32; 8], @@ -3258,9 +3266,11 @@ mod tests { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); // guaranteed to be *not* aligned to 16 bytes - let p = vals.data.as_mut_ptr().offset(1); + let p = unsafe { vals.data.as_mut_ptr().offset(1) }; - _mm_storeu_ps(p, *black_box(&a)); + unsafe { + _mm_storeu_ps(p, *black_box(&a)); + } assert_eq!(vals.data, [0.0, 1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0]); } @@ -3315,11 +3325,13 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_ps() { + fn test_mm_stream_ps() { let a = _mm_set1_ps(7.0); let mut mem = Memory { data: [-1.0; 4] }; - _mm_stream_ps(ptr::addr_of_mut!(mem.data[0]), a); + unsafe { + _mm_stream_ps(ptr::addr_of_mut!(mem.data[0]), a); + } _mm_sfence(); for i in 0..4 { assert_eq!(mem.data[i], get_m128(a, i)); diff --git a/library/stdarch/crates/core_arch/src/x86/sse2.rs b/library/stdarch/crates/core_arch/src/x86/sse2.rs index fc010e8467bb..c58e1b86a470 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse2.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse2.rs @@ -3291,9 +3291,11 @@ mod tests { } #[simd_test(enable = "sse2")] - unsafe fn test_mm_clflush() { + fn test_mm_clflush() { let x = 0_u8; - _mm_clflush(ptr::addr_of!(x)); + unsafe { + _mm_clflush(ptr::addr_of!(x)); + } } #[simd_test(enable = "sse2")] @@ -3725,7 +3727,7 @@ mod tests { } #[simd_test(enable = "sse2")] - unsafe fn test_mm_sll_epi16() { + fn test_mm_sll_epi16() { let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); let r = _mm_sll_epi16(a, _mm_set_epi64x(0, 4)); assert_eq_m128i( @@ -4071,7 +4073,7 @@ mod tests { } #[simd_test(enable = "sse2")] - unsafe fn test_mm_cvtps_epi32() { + fn test_mm_cvtps_epi32() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let r = _mm_cvtps_epi32(a); assert_eq_m128i(r, _mm_setr_epi32(1, 2, 3, 4)); @@ -4178,23 +4180,23 @@ mod tests { } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadl_epi64() { + const fn test_mm_loadl_epi64() { let a = _mm_setr_epi64x(6, 5); - let r = _mm_loadl_epi64(ptr::addr_of!(a)); + let r = unsafe { _mm_loadl_epi64(ptr::addr_of!(a)) }; assert_eq_m128i(r, _mm_setr_epi64x(6, 0)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_load_si128() { + const fn test_mm_load_si128() { let a = _mm_set_epi64x(5, 6); - let r = _mm_load_si128(ptr::addr_of!(a) as *const _); + let r = unsafe { _mm_load_si128(ptr::addr_of!(a) as *const _) }; assert_eq_m128i(a, r); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadu_si128() { + const fn test_mm_loadu_si128() { let a = _mm_set_epi64x(5, 6); - let r = _mm_loadu_si128(ptr::addr_of!(a) as *const _); + let r = unsafe { _mm_loadu_si128(ptr::addr_of!(a) as *const _) }; assert_eq_m128i(a, r); } @@ -4202,7 +4204,7 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_maskmoveu_si128() { + fn test_mm_maskmoveu_si128() { let a = _mm_set1_epi8(9); #[rustfmt::skip] let mask = _mm_set_epi8( @@ -4210,33 +4212,41 @@ mod tests { 0, 0, 0, 0, 0, 0, 0, 0, ); let mut r = _mm_set1_epi8(0); - _mm_maskmoveu_si128(a, mask, ptr::addr_of_mut!(r) as *mut i8); + unsafe { + _mm_maskmoveu_si128(a, mask, ptr::addr_of_mut!(r) as *mut i8); + } _mm_sfence(); let e = _mm_set_epi8(0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m128i(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_store_si128() { + const fn test_mm_store_si128() { let a = _mm_set1_epi8(9); let mut r = _mm_set1_epi8(0); - _mm_store_si128(&mut r, a); + unsafe { + _mm_store_si128(&mut r, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storeu_si128() { + const fn test_mm_storeu_si128() { let a = _mm_set1_epi8(9); let mut r = _mm_set1_epi8(0); - _mm_storeu_si128(&mut r, a); + unsafe { + _mm_storeu_si128(&mut r, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storel_epi64() { + const fn test_mm_storel_epi64() { let a = _mm_setr_epi64x(2, 9); let mut r = _mm_set1_epi8(0); - _mm_storel_epi64(&mut r, a); + unsafe { + _mm_storel_epi64(&mut r, a); + } assert_eq_m128i(r, _mm_setr_epi64x(2, 0)); } @@ -4244,10 +4254,12 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_si128() { + fn test_mm_stream_si128() { let a = _mm_setr_epi32(1, 2, 3, 4); let mut r = _mm_undefined_si128(); - _mm_stream_si128(ptr::addr_of_mut!(r), a); + unsafe { + _mm_stream_si128(ptr::addr_of_mut!(r), a); + } _mm_sfence(); assert_eq_m128i(r, a); } @@ -4256,10 +4268,12 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_si32() { + fn test_mm_stream_si32() { let a: i32 = 7; let mut mem = boxed::Box::::new(-1); - _mm_stream_si32(ptr::addr_of_mut!(*mem), a); + unsafe { + _mm_stream_si32(ptr::addr_of_mut!(*mem), a); + } _mm_sfence(); assert_eq!(a, *mem); } @@ -4909,40 +4923,40 @@ mod tests { } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_load_pd() { + const fn test_mm_load_pd() { let mem = Memory { data: [1.0f64, 2.0, 3.0, 4.0], }; let vals = &mem.data; let d = vals.as_ptr(); - let r = _mm_load_pd(d); + let r = unsafe { _mm_load_pd(d) }; assert_eq_m128d(r, _mm_setr_pd(1.0, 2.0)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_load_sd() { + const fn test_mm_load_sd() { let a = 1.; let expected = _mm_setr_pd(a, 0.); - let r = _mm_load_sd(&a); + let r = unsafe { _mm_load_sd(&a) }; assert_eq_m128d(r, expected); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadh_pd() { + const fn test_mm_loadh_pd() { let a = _mm_setr_pd(1., 2.); let b = 3.; let expected = _mm_setr_pd(_mm_cvtsd_f64(a), 3.); - let r = _mm_loadh_pd(a, &b); + let r = unsafe { _mm_loadh_pd(a, &b) }; assert_eq_m128d(r, expected); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadl_pd() { + const fn test_mm_loadl_pd() { let a = _mm_setr_pd(1., 2.); let b = 3.; let expected = _mm_setr_pd(3., get_m128d(a, 1)); - let r = _mm_loadl_pd(a, &b); + let r = unsafe { _mm_loadl_pd(a, &b) }; assert_eq_m128d(r, expected); } @@ -4950,7 +4964,7 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_pd() { + fn test_mm_stream_pd() { #[repr(align(128))] struct Memory { pub data: [f64; 2], @@ -4958,7 +4972,9 @@ mod tests { let a = _mm_set1_pd(7.0); let mut mem = Memory { data: [-1.0; 2] }; - _mm_stream_pd(ptr::addr_of_mut!(mem.data[0]), a); + unsafe { + _mm_stream_pd(ptr::addr_of_mut!(mem.data[0]), a); + } _mm_sfence(); for i in 0..2 { assert_eq!(mem.data[i], get_m128d(a, i)); @@ -4966,132 +4982,154 @@ mod tests { } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_store_sd() { + const fn test_mm_store_sd() { let mut dest = 0.; let a = _mm_setr_pd(1., 2.); - _mm_store_sd(&mut dest, a); + unsafe { + _mm_store_sd(&mut dest, a); + } assert_eq!(dest, _mm_cvtsd_f64(a)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_store_pd() { + const fn test_mm_store_pd() { let mut mem = Memory { data: [0.0f64; 4] }; let vals = &mut mem.data; let a = _mm_setr_pd(1.0, 2.0); let d = vals.as_mut_ptr(); - _mm_store_pd(d, *black_box(&a)); + unsafe { + _mm_store_pd(d, *black_box(&a)); + } assert_eq!(vals[0], 1.0); assert_eq!(vals[1], 2.0); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storeu_pd() { + const fn test_mm_storeu_pd() { // guaranteed to be aligned to 16 bytes let mut mem = Memory { data: [0.0f64; 4] }; let vals = &mut mem.data; let a = _mm_setr_pd(1.0, 2.0); // so p is *not* aligned to 16 bytes - let p = vals.as_mut_ptr().offset(1); - _mm_storeu_pd(p, *black_box(&a)); + unsafe { + let p = vals.as_mut_ptr().offset(1); + _mm_storeu_pd(p, *black_box(&a)); + } assert_eq!(*vals, [0.0, 1.0, 2.0, 0.0]); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storeu_si16() { + const fn test_mm_storeu_si16() { let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); let mut r = _mm_setr_epi16(9, 10, 11, 12, 13, 14, 15, 16); - _mm_storeu_si16(ptr::addr_of_mut!(r).cast(), a); + unsafe { + _mm_storeu_si16(ptr::addr_of_mut!(r).cast(), a); + } let e = _mm_setr_epi16(1, 10, 11, 12, 13, 14, 15, 16); assert_eq_m128i(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storeu_si32() { + const fn test_mm_storeu_si32() { let a = _mm_setr_epi32(1, 2, 3, 4); let mut r = _mm_setr_epi32(5, 6, 7, 8); - _mm_storeu_si32(ptr::addr_of_mut!(r).cast(), a); + unsafe { + _mm_storeu_si32(ptr::addr_of_mut!(r).cast(), a); + } let e = _mm_setr_epi32(1, 6, 7, 8); assert_eq_m128i(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storeu_si64() { + const fn test_mm_storeu_si64() { let a = _mm_setr_epi64x(1, 2); let mut r = _mm_setr_epi64x(3, 4); - _mm_storeu_si64(ptr::addr_of_mut!(r).cast(), a); + unsafe { + _mm_storeu_si64(ptr::addr_of_mut!(r).cast(), a); + } let e = _mm_setr_epi64x(1, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_store1_pd() { + const fn test_mm_store1_pd() { let mut mem = Memory { data: [0.0f64; 4] }; let vals = &mut mem.data; let a = _mm_setr_pd(1.0, 2.0); let d = vals.as_mut_ptr(); - _mm_store1_pd(d, *black_box(&a)); + unsafe { + _mm_store1_pd(d, *black_box(&a)); + } assert_eq!(vals[0], 1.0); assert_eq!(vals[1], 1.0); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_store_pd1() { + const fn test_mm_store_pd1() { let mut mem = Memory { data: [0.0f64; 4] }; let vals = &mut mem.data; let a = _mm_setr_pd(1.0, 2.0); let d = vals.as_mut_ptr(); - _mm_store_pd1(d, *black_box(&a)); + unsafe { + _mm_store_pd1(d, *black_box(&a)); + } assert_eq!(vals[0], 1.0); assert_eq!(vals[1], 1.0); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storer_pd() { + const fn test_mm_storer_pd() { let mut mem = Memory { data: [0.0f64; 4] }; let vals = &mut mem.data; let a = _mm_setr_pd(1.0, 2.0); let d = vals.as_mut_ptr(); - _mm_storer_pd(d, *black_box(&a)); + unsafe { + _mm_storer_pd(d, *black_box(&a)); + } assert_eq!(vals[0], 2.0); assert_eq!(vals[1], 1.0); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storeh_pd() { + const fn test_mm_storeh_pd() { let mut dest = 0.; let a = _mm_setr_pd(1., 2.); - _mm_storeh_pd(&mut dest, a); + unsafe { + _mm_storeh_pd(&mut dest, a); + } assert_eq!(dest, get_m128d(a, 1)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_storel_pd() { + const fn test_mm_storel_pd() { let mut dest = 0.; let a = _mm_setr_pd(1., 2.); - _mm_storel_pd(&mut dest, a); + unsafe { + _mm_storel_pd(&mut dest, a); + } assert_eq!(dest, _mm_cvtsd_f64(a)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadr_pd() { + const fn test_mm_loadr_pd() { let mut mem = Memory { data: [1.0f64, 2.0, 3.0, 4.0], }; let vals = &mut mem.data; let d = vals.as_ptr(); - let r = _mm_loadr_pd(d); + let r = unsafe { _mm_loadr_pd(d) }; assert_eq_m128d(r, _mm_setr_pd(2.0, 1.0)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadu_pd() { + const fn test_mm_loadu_pd() { // guaranteed to be aligned to 16 bytes let mut mem = Memory { data: [1.0f64, 2.0, 3.0, 4.0], @@ -5099,31 +5137,31 @@ mod tests { let vals = &mut mem.data; // so this will *not* be aligned to 16 bytes - let d = vals.as_ptr().offset(1); + let d = unsafe { vals.as_ptr().offset(1) }; - let r = _mm_loadu_pd(d); + let r = unsafe { _mm_loadu_pd(d) }; let e = _mm_setr_pd(2.0, 3.0); assert_eq_m128d(r, e); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadu_si16() { + const fn test_mm_loadu_si16() { let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm_loadu_si16(ptr::addr_of!(a) as *const _); + let r = unsafe { _mm_loadu_si16(ptr::addr_of!(a) as *const _) }; assert_eq_m128i(r, _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadu_si32() { + const fn test_mm_loadu_si32() { let a = _mm_setr_epi32(1, 2, 3, 4); - let r = _mm_loadu_si32(ptr::addr_of!(a) as *const _); + let r = unsafe { _mm_loadu_si32(ptr::addr_of!(a) as *const _) }; assert_eq_m128i(r, _mm_setr_epi32(1, 0, 0, 0)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_loadu_si64() { + const fn test_mm_loadu_si64() { let a = _mm_setr_epi64x(5, 6); - let r = _mm_loadu_si64(ptr::addr_of!(a) as *const _); + let r = unsafe { _mm_loadu_si64(ptr::addr_of!(a) as *const _) }; assert_eq_m128i(r, _mm_setr_epi64x(5, 0)); } @@ -5302,16 +5340,16 @@ mod tests { } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_load1_pd() { + const fn test_mm_load1_pd() { let d = -5.0; - let r = _mm_load1_pd(&d); + let r = unsafe { _mm_load1_pd(&d) }; assert_eq_m128d(r, _mm_setr_pd(d, d)); } #[simd_test(enable = "sse2")] - const unsafe fn test_mm_load_pd1() { + const fn test_mm_load_pd1() { let d = -5.0; - let r = _mm_load_pd1(&d); + let r = unsafe { _mm_load_pd1(&d) }; assert_eq_m128d(r, _mm_setr_pd(d, d)); } diff --git a/library/stdarch/crates/core_arch/src/x86/sse3.rs b/library/stdarch/crates/core_arch/src/x86/sse3.rs index 68817856f44a..e4c75702544d 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse3.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse3.rs @@ -239,7 +239,7 @@ mod tests { } #[simd_test(enable = "sse3")] - unsafe fn test_mm_lddqu_si128() { + fn test_mm_lddqu_si128() { #[rustfmt::skip] let a = _mm_setr_epi8( 1, 2, 3, 4, @@ -247,7 +247,7 @@ mod tests { 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_lddqu_si128(&a); + let r = unsafe { _mm_lddqu_si128(&a) }; assert_eq_m128i(a, r); } @@ -273,9 +273,9 @@ mod tests { } #[simd_test(enable = "sse3")] - const unsafe fn test_mm_loaddup_pd() { + const fn test_mm_loaddup_pd() { let d = -5.0; - let r = _mm_loaddup_pd(&d); + let r = unsafe { _mm_loaddup_pd(&d) }; assert_eq_m128d(r, _mm_setr_pd(d, d)); } } diff --git a/library/stdarch/crates/core_arch/src/x86/sse41.rs b/library/stdarch/crates/core_arch/src/x86/sse41.rs index a499bf898b80..7ad4306f36f2 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse41.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse41.rs @@ -1219,20 +1219,20 @@ mod tests { } #[simd_test(enable = "sse4.1")] - const unsafe fn test_mm_blendv_pd() { + const fn test_mm_blendv_pd() { let a = _mm_set1_pd(0.0); let b = _mm_set1_pd(1.0); - let mask = transmute(_mm_setr_epi64x(0, -1)); + let mask = _mm_castsi128_pd(_mm_setr_epi64x(0, -1)); let r = _mm_blendv_pd(a, b, mask); let e = _mm_setr_pd(0.0, 1.0); assert_eq_m128d(r, e); } #[simd_test(enable = "sse4.1")] - const unsafe fn test_mm_blendv_ps() { + const fn test_mm_blendv_ps() { let a = _mm_set1_ps(0.0); let b = _mm_set1_ps(1.0); - let mask = transmute(_mm_setr_epi32(0, -1, 0, -1)); + let mask = _mm_castsi128_ps(_mm_setr_epi32(0, -1, 0, -1)); let r = _mm_blendv_ps(a, b, mask); let e = _mm_setr_ps(0.0, 1.0, 0.0, 1.0); assert_eq_m128(r, e); @@ -1949,9 +1949,9 @@ mod tests { } #[simd_test(enable = "sse4.1")] - unsafe fn test_mm_stream_load_si128() { + fn test_mm_stream_load_si128() { let a = _mm_set_epi64x(5, 6); - let r = _mm_stream_load_si128(core::ptr::addr_of!(a) as *const _); + let r = unsafe { _mm_stream_load_si128(core::ptr::addr_of!(a) as *const _) }; assert_eq_m128i(a, r); } } diff --git a/library/stdarch/crates/core_arch/src/x86/sse42.rs b/library/stdarch/crates/core_arch/src/x86/sse42.rs index 65d1fe4d6233..55e22592637f 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse42.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse42.rs @@ -613,6 +613,7 @@ mod tests { use crate::core_arch::assert_eq_const as assert_eq; use stdarch_test::simd_test; + use crate::core_arch::simd::*; use crate::core_arch::x86::*; use std::ptr; @@ -625,7 +626,7 @@ mod tests { assert!(s.len() <= 16); let mut array = [0u8; 16]; array[..s.len()].copy_from_slice(s); - unsafe { transmute(array) } + u8x16::from_array(array).as_m128i() } #[simd_test(enable = "sse4.2")] diff --git a/library/stdarch/crates/core_arch/src/x86/sse4a.rs b/library/stdarch/crates/core_arch/src/x86/sse4a.rs index 020baeff152d..f36b879a030e 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse4a.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse4a.rs @@ -206,7 +206,7 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_sd() { + fn test_mm_stream_sd() { let mut mem = MemoryF64 { data: [1.0_f64, 2.0], }; @@ -216,7 +216,9 @@ mod tests { let x = _mm_setr_pd(3.0, 4.0); - _mm_stream_sd(d, x); + unsafe { + _mm_stream_sd(d, x); + } _mm_sfence(); } assert_eq!(mem.data[0], 3.0); @@ -232,7 +234,7 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_ss() { + fn test_mm_stream_ss() { let mut mem = MemoryF32 { data: [1.0_f32, 2.0, 3.0, 4.0], }; @@ -242,7 +244,9 @@ mod tests { let x = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); - _mm_stream_ss(d, x); + unsafe { + _mm_stream_ss(d, x); + } _mm_sfence(); } assert_eq!(mem.data[0], 5.0); diff --git a/library/stdarch/crates/core_arch/src/x86/xsave.rs b/library/stdarch/crates/core_arch/src/x86/xsave.rs index 653eb28c4268..e22d3580ff46 100644 --- a/library/stdarch/crates/core_arch/src/x86/xsave.rs +++ b/library/stdarch/crates/core_arch/src/x86/xsave.rs @@ -197,47 +197,53 @@ mod tests { #[simd_test(enable = "xsave")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xsave() { + fn test_xsave() { let m = 0xFFFFFFFFFFFFFFFF_u64; //< all registers let mut a = XsaveArea::new(); let mut b = XsaveArea::new(); - _xsave(a.ptr(), m); - _xrstor(a.ptr(), m); - _xsave(b.ptr(), m); + unsafe { + _xsave(a.ptr(), m); + _xrstor(a.ptr(), m); + _xsave(b.ptr(), m); + } } #[simd_test(enable = "xsave")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xgetbv() { + fn test_xgetbv() { let xcr_n: u32 = _XCR_XFEATURE_ENABLED_MASK; - let xcr: u64 = _xgetbv(xcr_n); - let xcr_cpy: u64 = _xgetbv(xcr_n); + let xcr: u64 = unsafe { _xgetbv(xcr_n) }; + let xcr_cpy: u64 = unsafe { _xgetbv(xcr_n) }; assert_eq!(xcr, xcr_cpy); } #[simd_test(enable = "xsave,xsaveopt")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xsaveopt() { + fn test_xsaveopt() { let m = 0xFFFFFFFFFFFFFFFF_u64; //< all registers let mut a = XsaveArea::new(); let mut b = XsaveArea::new(); - _xsaveopt(a.ptr(), m); - _xrstor(a.ptr(), m); - _xsaveopt(b.ptr(), m); + unsafe { + _xsaveopt(a.ptr(), m); + _xrstor(a.ptr(), m); + _xsaveopt(b.ptr(), m); + } } #[simd_test(enable = "xsave,xsavec")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xsavec() { + fn test_xsavec() { let m = 0xFFFFFFFFFFFFFFFF_u64; //< all registers let mut a = XsaveArea::new(); let mut b = XsaveArea::new(); - _xsavec(a.ptr(), m); - _xrstor(a.ptr(), m); - _xsavec(b.ptr(), m); + unsafe { + _xsavec(a.ptr(), m); + _xrstor(a.ptr(), m); + _xsavec(b.ptr(), m); + } } } diff --git a/library/stdarch/crates/core_arch/src/x86_64/amx.rs b/library/stdarch/crates/core_arch/src/x86_64/amx.rs index f372f7066d51..4e20e014cf20 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/amx.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/amx.rs @@ -581,267 +581,297 @@ mod tests { } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_loadconfig() { - let config = __tilecfg::default(); - _tile_loadconfig(config.as_ptr()); - _tile_release(); + fn test_tile_loadconfig() { + unsafe { + let config = __tilecfg::default(); + _tile_loadconfig(config.as_ptr()); + _tile_release(); + } } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_storeconfig() { - let config = __tilecfg::new(1, 0, [32; 8], [8; 8]); - _tile_loadconfig(config.as_ptr()); - let mut _config = __tilecfg::default(); - _tile_storeconfig(_config.as_mut_ptr()); - _tile_release(); - assert_eq!(config, _config); + fn test_tile_storeconfig() { + unsafe { + let config = __tilecfg::new(1, 0, [32; 8], [8; 8]); + _tile_loadconfig(config.as_ptr()); + let mut _config = __tilecfg::default(); + _tile_storeconfig(_config.as_mut_ptr()); + _tile_release(); + assert_eq!(config, _config); + } } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_zero() { - _init_amx(); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - let mut out = [[1_i8; 64]; 16]; - _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); - _tile_release(); - assert_eq!(out, [[0; 64]; 16]); + fn test_tile_zero() { + unsafe { + _init_amx(); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + let mut out = [[1_i8; 64]; 16]; + _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); + _tile_release(); + assert_eq!(out, [[0; 64]; 16]); + } } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_stored() { - _init_amx(); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - let mut out = [[1_i8; 64]; 16]; - _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); - _tile_release(); - assert_eq!(out, [[0; 64]; 16]); + fn test_tile_stored() { + unsafe { + _init_amx(); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + let mut out = [[1_i8; 64]; 16]; + _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); + _tile_release(); + assert_eq!(out, [[0; 64]; 16]); + } } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_loadd() { - _init_amx(); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - let mat = [1_i8; 1024]; - _tile_loadd::<0>(&mat as *const i8 as *const u8, 64); - let mut out = [[0_i8; 64]; 16]; - _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); - _tile_release(); - assert_eq!(out, [[1; 64]; 16]); + fn test_tile_loadd() { + unsafe { + _init_amx(); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + let mat = [1_i8; 1024]; + _tile_loadd::<0>(&mat as *const i8 as *const u8, 64); + let mut out = [[0_i8; 64]; 16]; + _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); + _tile_release(); + assert_eq!(out, [[1; 64]; 16]); + } } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_stream_loadd() { - _init_amx(); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - let mat = [1_i8; 1024]; - _tile_stream_loadd::<0>(&mat as *const i8 as *const u8, 64); - let mut out = [[0_i8; 64]; 16]; - _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); - _tile_release(); - assert_eq!(out, [[1; 64]; 16]); + fn test_tile_stream_loadd() { + unsafe { + _init_amx(); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + let mat = [1_i8; 1024]; + _tile_stream_loadd::<0>(&mat as *const i8 as *const u8, 64); + let mut out = [[0_i8; 64]; 16]; + _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); + _tile_release(); + assert_eq!(out, [[1; 64]; 16]); + } } #[simd_test(enable = "amx-tile")] - unsafe fn test_tile_release() { - _tile_release(); + fn test_tile_release() { + unsafe { + _tile_release(); + } } #[simd_test(enable = "amx-bf16,avx512f")] - unsafe fn test_tile_dpbf16ps() { - _init_amx(); - let bf16_1: u16 = _mm_cvtness_sbh(1.0).to_bits(); - let bf16_2: u16 = _mm_cvtness_sbh(2.0).to_bits(); - let ones: [u8; 1024] = transmute([bf16_1; 512]); - let twos: [u8; 1024] = transmute([bf16_2; 512]); - let mut res = [[0f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dpbf16ps::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[64f32; 16]; 16]); + fn test_tile_dpbf16ps() { + unsafe { + _init_amx(); + let bf16_1: u16 = _mm_cvtness_sbh(1.0).to_bits(); + let bf16_2: u16 = _mm_cvtness_sbh(2.0).to_bits(); + let ones: [u8; 1024] = transmute([bf16_1; 512]); + let twos: [u8; 1024] = transmute([bf16_2; 512]); + let mut res = [[0f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dpbf16ps::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[64f32; 16]; 16]); + } } #[simd_test(enable = "amx-int8")] - unsafe fn test_tile_dpbssd() { - _init_amx(); - let ones = [-1_i8; 1024]; - let twos = [-2_i8; 1024]; - let mut res = [[0_i32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const i8 as *const u8, 64); - _tile_loadd::<2>(&twos as *const i8 as *const u8, 64); - _tile_dpbssd::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[128_i32; 16]; 16]); + fn test_tile_dpbssd() { + unsafe { + _init_amx(); + let ones = [-1_i8; 1024]; + let twos = [-2_i8; 1024]; + let mut res = [[0_i32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const i8 as *const u8, 64); + _tile_loadd::<2>(&twos as *const i8 as *const u8, 64); + _tile_dpbssd::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[128_i32; 16]; 16]); + } } #[simd_test(enable = "amx-int8")] - unsafe fn test_tile_dpbsud() { - _init_amx(); - let ones = [-1_i8; 1024]; - let twos = [2_u8; 1024]; - let mut res = [[0_i32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const i8 as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dpbsud::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[-128_i32; 16]; 16]); + fn test_tile_dpbsud() { + unsafe { + _init_amx(); + let ones = [-1_i8; 1024]; + let twos = [2_u8; 1024]; + let mut res = [[0_i32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const i8 as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dpbsud::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[-128_i32; 16]; 16]); + } } #[simd_test(enable = "amx-int8")] - unsafe fn test_tile_dpbusd() { - _init_amx(); - let ones = [1_u8; 1024]; - let twos = [-2_i8; 1024]; - let mut res = [[0_i32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const i8 as *const u8, 64); - _tile_dpbusd::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[-128_i32; 16]; 16]); + fn test_tile_dpbusd() { + unsafe { + _init_amx(); + let ones = [1_u8; 1024]; + let twos = [-2_i8; 1024]; + let mut res = [[0_i32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const i8 as *const u8, 64); + _tile_dpbusd::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[-128_i32; 16]; 16]); + } } #[simd_test(enable = "amx-int8")] - unsafe fn test_tile_dpbuud() { - _init_amx(); - let ones = [1_u8; 1024]; - let twos = [2_u8; 1024]; - let mut res = [[0_i32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dpbuud::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[128_i32; 16]; 16]); + fn test_tile_dpbuud() { + unsafe { + _init_amx(); + let ones = [1_u8; 1024]; + let twos = [2_u8; 1024]; + let mut res = [[0_i32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dpbuud::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [i32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[128_i32; 16]; 16]); + } } #[simd_test(enable = "amx-fp16")] - unsafe fn test_tile_dpfp16ps() { - _init_amx(); - let ones = [1f16; 512]; - let twos = [2f16; 512]; - let mut res = [[0f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const f16 as *const u8, 64); - _tile_loadd::<2>(&twos as *const f16 as *const u8, 64); - _tile_dpfp16ps::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[64f32; 16]; 16]); + fn test_tile_dpfp16ps() { + unsafe { + _init_amx(); + let ones = [1f16; 512]; + let twos = [2f16; 512]; + let mut res = [[0f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const f16 as *const u8, 64); + _tile_loadd::<2>(&twos as *const f16 as *const u8, 64); + _tile_dpfp16ps::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[64f32; 16]; 16]); + } } #[simd_test(enable = "amx-complex")] - unsafe fn test_tile_cmmimfp16ps() { - _init_amx(); - let ones = [1f16; 512]; - let twos = [2f16; 512]; - let mut res = [[0f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const f16 as *const u8, 64); - _tile_loadd::<2>(&twos as *const f16 as *const u8, 64); - _tile_cmmimfp16ps::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[64f32; 16]; 16]); + fn test_tile_cmmimfp16ps() { + unsafe { + _init_amx(); + let ones = [1f16; 512]; + let twos = [2f16; 512]; + let mut res = [[0f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const f16 as *const u8, 64); + _tile_loadd::<2>(&twos as *const f16 as *const u8, 64); + _tile_cmmimfp16ps::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[64f32; 16]; 16]); + } } #[simd_test(enable = "amx-complex")] - unsafe fn test_tile_cmmrlfp16ps() { - _init_amx(); - let ones = [1f16; 512]; - let twos = [2f16; 512]; - let mut res = [[0f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const f16 as *const u8, 64); - _tile_loadd::<2>(&twos as *const f16 as *const u8, 64); - _tile_cmmrlfp16ps::<0, 1, 2>(); - _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); - _tile_release(); - assert_eq!(res, [[0f32; 16]; 16]); + fn test_tile_cmmrlfp16ps() { + unsafe { + _init_amx(); + let ones = [1f16; 512]; + let twos = [2f16; 512]; + let mut res = [[0f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const f16 as *const u8, 64); + _tile_loadd::<2>(&twos as *const f16 as *const u8, 64); + _tile_cmmrlfp16ps::<0, 1, 2>(); + _tile_stored::<0>(&mut res as *mut [f32; 16] as *mut u8, 64); + _tile_release(); + assert_eq!(res, [[0f32; 16]; 16]); + } } const BF8_ONE: u8 = 0x3c; @@ -850,223 +880,245 @@ mod tests { const HF8_TWO: u8 = 0x40; #[simd_test(enable = "amx-fp8")] - unsafe fn test_tile_dpbf8ps() { - _init_amx(); - let ones = [BF8_ONE; 1024]; - let twos = [BF8_TWO; 1024]; - let mut res = [[0.0_f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dpbf8ps::<0, 1, 2>(); - _tile_stored::<0>(res.as_mut_ptr().cast(), 64); - _tile_release(); - assert_eq!(res, [[128.0_f32; 16]; 16]); + fn test_tile_dpbf8ps() { + unsafe { + _init_amx(); + let ones = [BF8_ONE; 1024]; + let twos = [BF8_TWO; 1024]; + let mut res = [[0.0_f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dpbf8ps::<0, 1, 2>(); + _tile_stored::<0>(res.as_mut_ptr().cast(), 64); + _tile_release(); + assert_eq!(res, [[128.0_f32; 16]; 16]); + } } #[simd_test(enable = "amx-fp8")] - unsafe fn test_tile_dpbhf8ps() { - _init_amx(); - let ones = [BF8_ONE; 1024]; - let twos = [HF8_TWO; 1024]; - let mut res = [[0.0_f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dpbhf8ps::<0, 1, 2>(); - _tile_stored::<0>(res.as_mut_ptr().cast(), 64); - _tile_release(); - assert_eq!(res, [[128.0_f32; 16]; 16]); + fn test_tile_dpbhf8ps() { + unsafe { + _init_amx(); + let ones = [BF8_ONE; 1024]; + let twos = [HF8_TWO; 1024]; + let mut res = [[0.0_f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dpbhf8ps::<0, 1, 2>(); + _tile_stored::<0>(res.as_mut_ptr().cast(), 64); + _tile_release(); + assert_eq!(res, [[128.0_f32; 16]; 16]); + } } #[simd_test(enable = "amx-fp8")] - unsafe fn test_tile_dphbf8ps() { - _init_amx(); - let ones = [HF8_ONE; 1024]; - let twos = [BF8_TWO; 1024]; - let mut res = [[0.0_f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dphbf8ps::<0, 1, 2>(); - _tile_stored::<0>(res.as_mut_ptr().cast(), 64); - _tile_release(); - assert_eq!(res, [[128.0_f32; 16]; 16]); + fn test_tile_dphbf8ps() { + unsafe { + _init_amx(); + let ones = [HF8_ONE; 1024]; + let twos = [BF8_TWO; 1024]; + let mut res = [[0.0_f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dphbf8ps::<0, 1, 2>(); + _tile_stored::<0>(res.as_mut_ptr().cast(), 64); + _tile_release(); + assert_eq!(res, [[128.0_f32; 16]; 16]); + } } #[simd_test(enable = "amx-fp8")] - unsafe fn test_tile_dphf8ps() { - _init_amx(); - let ones = [HF8_ONE; 1024]; - let twos = [HF8_TWO; 1024]; - let mut res = [[0.0_f32; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(&ones as *const u8, 64); - _tile_loadd::<2>(&twos as *const u8, 64); - _tile_dphf8ps::<0, 1, 2>(); - _tile_stored::<0>(res.as_mut_ptr().cast(), 64); - _tile_release(); - assert_eq!(res, [[128.0_f32; 16]; 16]); + fn test_tile_dphf8ps() { + unsafe { + _init_amx(); + let ones = [HF8_ONE; 1024]; + let twos = [HF8_TWO; 1024]; + let mut res = [[0.0_f32; 16]; 16]; + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(&ones as *const u8, 64); + _tile_loadd::<2>(&twos as *const u8, 64); + _tile_dphf8ps::<0, 1, 2>(); + _tile_stored::<0>(res.as_mut_ptr().cast(), 64); + _tile_release(); + assert_eq!(res, [[128.0_f32; 16]; 16]); + } } #[simd_test(enable = "amx-movrs")] - unsafe fn test_tile_loaddrs() { - _init_amx(); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - let mat = [1_i8; 1024]; - _tile_loaddrs::<0>(&mat as *const i8 as *const u8, 64); - let mut out = [[0_i8; 64]; 16]; - _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); - _tile_release(); - assert_eq!(out, [[1; 64]; 16]); + fn test_tile_loaddrs() { + unsafe { + _init_amx(); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + let mat = [1_i8; 1024]; + _tile_loaddrs::<0>(&mat as *const i8 as *const u8, 64); + let mut out = [[0_i8; 64]; 16]; + _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); + _tile_release(); + assert_eq!(out, [[1; 64]; 16]); + } } #[simd_test(enable = "amx-movrs")] - unsafe fn test_tile_stream_loaddrs() { - _init_amx(); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - let mat = [1_i8; 1024]; - _tile_stream_loaddrs::<0>(&mat as *const i8 as *const u8, 64); - let mut out = [[0_i8; 64]; 16]; - _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); - _tile_release(); - assert_eq!(out, [[1; 64]; 16]); - } - - #[simd_test(enable = "amx-avx512,avx10.2")] - unsafe fn test_tile_movrow() { - _init_amx(); - let array: [[u8; 64]; 16] = array::from_fn(|i| [i as _; _]); - - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_loadd::<0>(array.as_ptr().cast(), 64); - for i in 0..16 { - let row = _tile_movrow::<0>(i); - assert_eq!(*row.as_u8x64().as_array(), [i as _; _]); + fn test_tile_stream_loaddrs() { + unsafe { + _init_amx(); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + let mat = [1_i8; 1024]; + _tile_stream_loaddrs::<0>(&mat as *const i8 as *const u8, 64); + let mut out = [[0_i8; 64]; 16]; + _tile_stored::<0>(&mut out as *mut [i8; 64] as *mut u8, 64); + _tile_release(); + assert_eq!(out, [[1; 64]; 16]); } } #[simd_test(enable = "amx-avx512,avx10.2")] - unsafe fn test_tile_cvtrowd2ps() { - _init_amx(); - let array: [[u32; 16]; 16] = array::from_fn(|i| [i as _; _]); + fn test_tile_movrow() { + unsafe { + _init_amx(); + let array: [[u8; 64]; 16] = array::from_fn(|i| [i as _; _]); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_loadd::<0>(array.as_ptr().cast(), 64); - for i in 0..16 { - let row = _tile_cvtrowd2ps::<0>(i); - assert_eq!(*row.as_f32x16().as_array(), [i as _; _]); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_loadd::<0>(array.as_ptr().cast(), 64); + for i in 0..16 { + let row = _tile_movrow::<0>(i); + assert_eq!(*row.as_u8x64().as_array(), [i as _; _]); + } } } #[simd_test(enable = "amx-avx512,avx10.2")] - unsafe fn test_tile_cvtrowps2phh() { - _init_amx(); - let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]); + fn test_tile_cvtrowd2ps() { + unsafe { + _init_amx(); + let array: [[u32; 16]; 16] = array::from_fn(|i| [i as _; _]); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_loadd::<0>(array.as_ptr().cast(), 64); - for i in 0..16 { - let row = _tile_cvtrowps2phh::<0>(i); - assert_eq!( - *row.as_f16x32().as_array(), - array::from_fn(|j| if j & 1 == 0 { 0.0 } else { i as _ }) - ); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_loadd::<0>(array.as_ptr().cast(), 64); + for i in 0..16 { + let row = _tile_cvtrowd2ps::<0>(i); + assert_eq!(*row.as_f32x16().as_array(), [i as _; _]); + } } } #[simd_test(enable = "amx-avx512,avx10.2")] - unsafe fn test_tile_cvtrowps2phl() { - _init_amx(); - let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]); + fn test_tile_cvtrowps2phh() { + unsafe { + _init_amx(); + let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]); - let mut config = __tilecfg::default(); - config.palette = 1; - config.colsb[0] = 64; - config.rows[0] = 16; - _tile_loadconfig(config.as_ptr()); - _tile_loadd::<0>(array.as_ptr().cast(), 64); - for i in 0..16 { - let row = _tile_cvtrowps2phl::<0>(i); - assert_eq!( - *row.as_f16x32().as_array(), - array::from_fn(|j| if j & 1 == 0 { i as _ } else { 0.0 }) - ); + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_loadd::<0>(array.as_ptr().cast(), 64); + for i in 0..16 { + let row = _tile_cvtrowps2phh::<0>(i); + assert_eq!( + *row.as_f16x32().as_array(), + array::from_fn(|j| if j & 1 == 0 { 0.0 } else { i as _ }) + ); + } + } + } + + #[simd_test(enable = "amx-avx512,avx10.2")] + fn test_tile_cvtrowps2phl() { + unsafe { + _init_amx(); + let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]); + + let mut config = __tilecfg::default(); + config.palette = 1; + config.colsb[0] = 64; + config.rows[0] = 16; + _tile_loadconfig(config.as_ptr()); + _tile_loadd::<0>(array.as_ptr().cast(), 64); + for i in 0..16 { + let row = _tile_cvtrowps2phl::<0>(i); + assert_eq!( + *row.as_f16x32().as_array(), + array::from_fn(|j| if j & 1 == 0 { i as _ } else { 0.0 }) + ); + } } } #[simd_test(enable = "amx-tf32")] - unsafe fn test_tile_mmultf32ps() { - _init_amx(); - let a: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]); - let b: [[f32; 16]; 16] = [array::from_fn(|j| j as _); _]; - let mut res = [[0.0; 16]; 16]; + fn test_tile_mmultf32ps() { + unsafe { + _init_amx(); + let a: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]); + let b: [[f32; 16]; 16] = [array::from_fn(|j| j as _); _]; + let mut res = [[0.0; 16]; 16]; - let mut config = __tilecfg::default(); - config.palette = 1; - (0..=2).for_each(|i| { - config.colsb[i] = 64; - config.rows[i] = 16; - }); - _tile_loadconfig(config.as_ptr()); - _tile_zero::<0>(); - _tile_loadd::<1>(a.as_ptr().cast(), 64); - _tile_loadd::<2>(b.as_ptr().cast(), 64); - _tile_mmultf32ps::<0, 1, 2>(); - _tile_stored::<0>(res.as_mut_ptr().cast(), 64); - _tile_release(); + let mut config = __tilecfg::default(); + config.palette = 1; + (0..=2).for_each(|i| { + config.colsb[i] = 64; + config.rows[i] = 16; + }); + _tile_loadconfig(config.as_ptr()); + _tile_zero::<0>(); + _tile_loadd::<1>(a.as_ptr().cast(), 64); + _tile_loadd::<2>(b.as_ptr().cast(), 64); + _tile_mmultf32ps::<0, 1, 2>(); + _tile_stored::<0>(res.as_mut_ptr().cast(), 64); + _tile_release(); - let expected = array::from_fn(|i| array::from_fn(|j| 16.0 * i as f32 * j as f32)); - assert_eq!(res, expected); + let expected = array::from_fn(|i| array::from_fn(|j| 16.0 * i as f32 * j as f32)); + assert_eq!(res, expected); + } } } diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs index 368fb0c238e4..0fd9b09363d4 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs @@ -7446,81 +7446,81 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32gather_pd() { + fn test_mm512_i32gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i32gather_pd::<8>(index, arr.as_ptr()); + let r = unsafe { _mm512_i32gather_pd::<8>(index, arr.as_ptr()) }; assert_eq_m512d(r, _mm512_setr_pd(0., 16., 32., 48., 64., 80., 96., 112.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32gather_pd() { + fn test_mm512_mask_i32gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); let src = _mm512_set1_pd(2.); let mask = 0b10101010; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i32gather_pd::<8>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i32gather_pd::<8>(src, mask, index, arr.as_ptr()) }; assert_eq_m512d(r, _mm512_setr_pd(2., 16., 2., 48., 2., 80., 2., 112.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64gather_pd() { + fn test_mm512_i64gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); // A multiplier of 8 is word-addressing let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_pd::<8>(index, arr.as_ptr()); + let r = unsafe { _mm512_i64gather_pd::<8>(index, arr.as_ptr()) }; assert_eq_m512d(r, _mm512_setr_pd(0., 16., 32., 48., 64., 80., 96., 112.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64gather_pd() { + fn test_mm512_mask_i64gather_pd() { let arr: [f64; 128] = core::array::from_fn(|i| i as f64); let src = _mm512_set1_pd(2.); let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i64gather_pd::<8>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i64gather_pd::<8>(src, mask, index, arr.as_ptr()) }; assert_eq_m512d(r, _mm512_setr_pd(2., 16., 2., 48., 2., 80., 2., 112.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64gather_ps() { + fn test_mm512_i64gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); // A multiplier of 4 is word-addressing #[rustfmt::skip] let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_ps::<4>(index, arr.as_ptr()); + let r = unsafe { _mm512_i64gather_ps::<4>(index, arr.as_ptr()) }; assert_eq_m256(r, _mm256_setr_ps(0., 16., 32., 48., 64., 80., 96., 112.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64gather_ps() { + fn test_mm512_mask_i64gather_ps() { let arr: [f32; 128] = core::array::from_fn(|i| i as f32); let src = _mm256_set1_ps(2.); let mask = 0b10101010; #[rustfmt::skip] let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 4 is word-addressing - let r = _mm512_mask_i64gather_ps::<4>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i64gather_ps::<4>(src, mask, index, arr.as_ptr()) }; assert_eq_m256(r, _mm256_setr_ps(2., 16., 2., 48., 2., 80., 2., 112.)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32gather_epi64() { + fn test_mm512_i32gather_epi64() { let mut arr = [0i64; 128]; for i in 0..128i64 { arr[i as usize] = i; } // A multiplier of 8 is word-addressing let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i32gather_epi64::<8>(index, arr.as_ptr()); + let r = unsafe { _mm512_i32gather_epi64::<8>(index, arr.as_ptr()) }; assert_eq_m512i(r, _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32gather_epi64() { + fn test_mm512_mask_i32gather_epi64() { let mut arr = [0i64; 128]; for i in 0..128i64 { arr[i as usize] = i; @@ -7529,24 +7529,24 @@ mod tests { let mask = 0b10101010; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i32gather_epi64::<8>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i32gather_epi64::<8>(src, mask, index, arr.as_ptr()) }; assert_eq_m512i(r, _mm512_setr_epi64(2, 16, 2, 48, 2, 80, 2, 112)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64gather_epi64() { + fn test_mm512_i64gather_epi64() { let mut arr = [0i64; 128]; for i in 0..128i64 { arr[i as usize] = i; } // A multiplier of 8 is word-addressing let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_epi64::<8>(index, arr.as_ptr()); + let r = unsafe { _mm512_i64gather_epi64::<8>(index, arr.as_ptr()) }; assert_eq_m512i(r, _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64gather_epi64() { + fn test_mm512_mask_i64gather_epi64() { let mut arr = [0i64; 128]; for i in 0..128i64 { arr[i as usize] = i; @@ -7555,24 +7555,24 @@ mod tests { let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i64gather_epi64::<8>(src, mask, index, arr.as_ptr()); + let r = unsafe { _mm512_mask_i64gather_epi64::<8>(src, mask, index, arr.as_ptr()) }; assert_eq_m512i(r, _mm512_setr_epi64(2, 16, 2, 48, 2, 80, 2, 112)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64gather_epi32() { + fn test_mm512_i64gather_epi32() { let mut arr = [0i64; 128]; for i in 0..128i64 { arr[i as usize] = i; } // A multiplier of 8 is word-addressing let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_epi32::<8>(index, arr.as_ptr() as *const i32); + let r = unsafe { _mm512_i64gather_epi32::<8>(index, arr.as_ptr() as *const i32) }; assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64gather_epi32() { + fn test_mm512_mask_i64gather_epi32() { let mut arr = [0i64; 128]; for i in 0..128i64 { arr[i as usize] = i; @@ -7581,17 +7581,21 @@ mod tests { let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i64gather_epi32::<8>(src, mask, index, arr.as_ptr() as *const i32); + let r = unsafe { + _mm512_mask_i64gather_epi32::<8>(src, mask, index, arr.as_ptr() as *const i32) + }; assert_eq_m256i(r, _mm256_setr_epi32(2, 16, 2, 48, 2, 80, 2, 112)); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32scatter_pd() { + fn test_mm512_i32scatter_pd() { let mut arr = [0f64; 128]; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_i32scatter_pd::<8>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i32scatter_pd::<8>(arr.as_mut_ptr(), index, src); + } let mut expected = [0f64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as f64; @@ -7600,13 +7604,15 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32scatter_pd() { + fn test_mm512_mask_i32scatter_pd() { let mut arr = [0f64; 128]; let mask = 0b10101010; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_mask_i32scatter_pd::<8>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i32scatter_pd::<8>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0f64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2. * (i + 1) as f64; @@ -7615,12 +7621,14 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64scatter_pd() { + fn test_mm512_i64scatter_pd() { let mut arr = [0f64; 128]; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_i64scatter_pd::<8>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i64scatter_pd::<8>(arr.as_mut_ptr(), index, src); + } let mut expected = [0f64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as f64; @@ -7629,13 +7637,15 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64scatter_pd() { + fn test_mm512_mask_i64scatter_pd() { let mut arr = [0f64; 128]; let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_mask_i64scatter_pd::<8>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i64scatter_pd::<8>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0f64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2. * (i + 1) as f64; @@ -7644,12 +7654,14 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64scatter_ps() { + fn test_mm512_i64scatter_ps() { let mut arr = [0f32; 128]; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 4 is word-addressing - _mm512_i64scatter_ps::<4>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i64scatter_ps::<4>(arr.as_mut_ptr(), index, src); + } let mut expected = [0f32; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as f32; @@ -7658,13 +7670,15 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64scatter_ps() { + fn test_mm512_mask_i64scatter_ps() { let mut arr = [0f32; 128]; let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 4 is word-addressing - _mm512_mask_i64scatter_ps::<4>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i64scatter_ps::<4>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0f32; 128]; for i in 0..4 { expected[i * 32 + 16] = 2. * (i + 1) as f32; @@ -7673,12 +7687,14 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32scatter_epi64() { + fn test_mm512_i32scatter_epi64() { let mut arr = [0i64; 128]; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_i32scatter_epi64::<8>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i32scatter_epi64::<8>(arr.as_mut_ptr(), index, src); + } let mut expected = [0i64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as i64; @@ -7687,13 +7703,15 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32scatter_epi64() { + fn test_mm512_mask_i32scatter_epi64() { let mut arr = [0i64; 128]; let mask = 0b10101010; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_mask_i32scatter_epi64::<8>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i32scatter_epi64::<8>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0i64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2 * (i + 1) as i64; @@ -7702,12 +7720,14 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64scatter_epi64() { + fn test_mm512_i64scatter_epi64() { let mut arr = [0i64; 128]; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_i64scatter_epi64::<8>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i64scatter_epi64::<8>(arr.as_mut_ptr(), index, src); + } let mut expected = [0i64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as i64; @@ -7716,13 +7736,15 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64scatter_epi64() { + fn test_mm512_mask_i64scatter_epi64() { let mut arr = [0i64; 128]; let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_mask_i64scatter_epi64::<8>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i64scatter_epi64::<8>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0i64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2 * (i + 1) as i64; @@ -7731,12 +7753,14 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i64scatter_epi32() { + fn test_mm512_i64scatter_epi32() { let mut arr = [0i32; 128]; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 4 is word-addressing - _mm512_i64scatter_epi32::<4>(arr.as_mut_ptr(), index, src); + unsafe { + _mm512_i64scatter_epi32::<4>(arr.as_mut_ptr(), index, src); + } let mut expected = [0i32; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as i32; @@ -7745,13 +7769,15 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i64scatter_epi32() { + fn test_mm512_mask_i64scatter_epi32() { let mut arr = [0i32; 128]; let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 4 is word-addressing - _mm512_mask_i64scatter_epi32::<4>(arr.as_mut_ptr(), mask, index, src); + unsafe { + _mm512_mask_i64scatter_epi32::<4>(arr.as_mut_ptr(), mask, index, src); + } let mut expected = [0i32; 128]; for i in 0..4 { expected[i * 32 + 16] = 2 * (i + 1) as i32; @@ -7760,559 +7786,640 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32logather_epi64() { + fn test_mm512_i32logather_epi64() { let base_addr: [i64; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); - let r = _mm512_i32logather_epi64::<8>(vindex, base_addr.as_ptr()); + let r = unsafe { _mm512_i32logather_epi64::<8>(vindex, base_addr.as_ptr()) }; let expected = _mm512_setr_epi64(2, 3, 4, 5, 6, 7, 8, 1); assert_eq_m512i(expected, r); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32logather_epi64() { + fn test_mm512_mask_i32logather_epi64() { let base_addr: [i64; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; let src = _mm512_setr_epi64(9, 10, 11, 12, 13, 14, 15, 16); let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); - let r = _mm512_mask_i32logather_epi64::<8>(src, 0b01010101, vindex, base_addr.as_ptr()); + let r = unsafe { + _mm512_mask_i32logather_epi64::<8>(src, 0b01010101, vindex, base_addr.as_ptr()) + }; let expected = _mm512_setr_epi64(2, 10, 4, 12, 6, 14, 8, 16); assert_eq_m512i(expected, r); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32logather_pd() { + fn test_mm512_i32logather_pd() { let base_addr: [f64; 8] = [1., 2., 3., 4., 5., 6., 7., 8.]; let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); - let r = _mm512_i32logather_pd::<8>(vindex, base_addr.as_ptr()); + let r = unsafe { _mm512_i32logather_pd::<8>(vindex, base_addr.as_ptr()) }; let expected = _mm512_setr_pd(2., 3., 4., 5., 6., 7., 8., 1.); assert_eq_m512d(expected, r); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32logather_pd() { + fn test_mm512_mask_i32logather_pd() { let base_addr: [f64; 8] = [1., 2., 3., 4., 5., 6., 7., 8.]; let src = _mm512_setr_pd(9., 10., 11., 12., 13., 14., 15., 16.); let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); - let r = _mm512_mask_i32logather_pd::<8>(src, 0b01010101, vindex, base_addr.as_ptr()); + let r = + unsafe { _mm512_mask_i32logather_pd::<8>(src, 0b01010101, vindex, base_addr.as_ptr()) }; let expected = _mm512_setr_pd(2., 10., 4., 12., 6., 14., 8., 16.); assert_eq_m512d(expected, r); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32loscatter_epi64() { + fn test_mm512_i32loscatter_epi64() { let mut base_addr: [i64; 8] = [0; 8]; let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); let src = _mm512_setr_epi64(2, 3, 4, 5, 6, 7, 8, 1); - _mm512_i32loscatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm512_i32loscatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2, 3, 4, 5, 6, 7, 8]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32loscatter_epi64() { + fn test_mm512_mask_i32loscatter_epi64() { let mut base_addr: [i64; 8] = [0; 8]; let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); let src = _mm512_setr_epi64(2, 3, 4, 5, 6, 7, 8, 1); - _mm512_mask_i32loscatter_epi64::<8>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + unsafe { + _mm512_mask_i32loscatter_epi64::<8>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + } let expected = [0, 2, 0, 4, 0, 6, 0, 8]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_i32loscatter_pd() { + fn test_mm512_i32loscatter_pd() { let mut base_addr: [f64; 8] = [0.; 8]; let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); let src = _mm512_setr_pd(2., 3., 4., 5., 6., 7., 8., 1.); - _mm512_i32loscatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm512_i32loscatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2., 3., 4., 5., 6., 7., 8.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_i32loscatter_pd() { + fn test_mm512_mask_i32loscatter_pd() { let mut base_addr: [f64; 8] = [0.; 8]; let vindex = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1); let src = _mm512_setr_pd(2., 3., 4., 5., 6., 7., 8., 1.); - _mm512_mask_i32loscatter_pd::<8>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + unsafe { + _mm512_mask_i32loscatter_pd::<8>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + } let expected = [0., 2., 0., 4., 0., 6., 0., 8.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i32gather_epi32() { + fn test_mm_mmask_i32gather_epi32() { let base_addr: [i32; 4] = [1, 2, 3, 4]; let src = _mm_setr_epi32(5, 6, 7, 8); let vindex = _mm_setr_epi32(1, 2, 3, 0); - let r = _mm_mmask_i32gather_epi32::<4>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i32gather_epi32::<4>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_epi32(2, 6, 4, 8); assert_eq_m128i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i32gather_epi64() { + fn test_mm_mmask_i32gather_epi64() { let base_addr: [i64; 2] = [1, 2]; let src = _mm_setr_epi64x(5, 6); let vindex = _mm_setr_epi32(1, 0, -1, -1); - let r = _mm_mmask_i32gather_epi64::<8>(src, 0b01, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i32gather_epi64::<8>(src, 0b01, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_epi64x(2, 6); assert_eq_m128i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i32gather_pd() { + fn test_mm_mmask_i32gather_pd() { let base_addr: [f64; 2] = [1., 2.]; let src = _mm_setr_pd(5., 6.); let vindex = _mm_setr_epi32(1, 0, -1, -1); - let r = _mm_mmask_i32gather_pd::<8>(src, 0b01, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i32gather_pd::<8>(src, 0b01, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_pd(2., 6.); assert_eq_m128d(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i32gather_ps() { + fn test_mm_mmask_i32gather_ps() { let base_addr: [f32; 4] = [1., 2., 3., 4.]; let src = _mm_setr_ps(5., 6., 7., 8.); let vindex = _mm_setr_epi32(1, 2, 3, 0); - let r = _mm_mmask_i32gather_ps::<4>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i32gather_ps::<4>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_ps(2., 6., 4., 8.); assert_eq_m128(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i64gather_epi32() { + fn test_mm_mmask_i64gather_epi32() { let base_addr: [i32; 2] = [1, 2]; let src = _mm_setr_epi32(5, 6, 7, 8); let vindex = _mm_setr_epi64x(1, 0); - let r = _mm_mmask_i64gather_epi32::<4>(src, 0b01, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i64gather_epi32::<4>(src, 0b01, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_epi32(2, 6, 0, 0); assert_eq_m128i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i64gather_epi64() { + fn test_mm_mmask_i64gather_epi64() { let base_addr: [i64; 2] = [1, 2]; let src = _mm_setr_epi64x(5, 6); let vindex = _mm_setr_epi64x(1, 0); - let r = _mm_mmask_i64gather_epi64::<8>(src, 0b01, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i64gather_epi64::<8>(src, 0b01, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_epi64x(2, 6); assert_eq_m128i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i64gather_pd() { + fn test_mm_mmask_i64gather_pd() { let base_addr: [f64; 2] = [1., 2.]; let src = _mm_setr_pd(5., 6.); let vindex = _mm_setr_epi64x(1, 0); - let r = _mm_mmask_i64gather_pd::<8>(src, 0b01, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i64gather_pd::<8>(src, 0b01, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_pd(2., 6.); assert_eq_m128d(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mmask_i64gather_ps() { + fn test_mm_mmask_i64gather_ps() { let base_addr: [f32; 2] = [1., 2.]; let src = _mm_setr_ps(5., 6., 7., 8.); let vindex = _mm_setr_epi64x(1, 0); - let r = _mm_mmask_i64gather_ps::<4>(src, 0b01, vindex, base_addr.as_ptr()); + let r = unsafe { _mm_mmask_i64gather_ps::<4>(src, 0b01, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_ps(2., 6., 0., 0.); assert_eq_m128(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i32gather_epi32() { + fn test_mm256_mmask_i32gather_epi32() { let base_addr: [i32; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; let src = _mm256_setr_epi32(9, 10, 11, 12, 13, 14, 15, 16); let vindex = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); - let r = _mm256_mmask_i32gather_epi32::<4>(src, 0b01010101, vindex, base_addr.as_ptr()); + let r = unsafe { + _mm256_mmask_i32gather_epi32::<4>(src, 0b01010101, vindex, base_addr.as_ptr()) + }; let expected = _mm256_setr_epi32(2, 10, 4, 12, 6, 14, 8, 16); assert_eq_m256i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i32gather_epi64() { + fn test_mm256_mmask_i32gather_epi64() { let base_addr: [i64; 4] = [1, 2, 3, 4]; let src = _mm256_setr_epi64x(9, 10, 11, 12); let vindex = _mm_setr_epi32(1, 2, 3, 4); - let r = _mm256_mmask_i32gather_epi64::<8>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = + unsafe { _mm256_mmask_i32gather_epi64::<8>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm256_setr_epi64x(2, 10, 4, 12); assert_eq_m256i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i32gather_pd() { + fn test_mm256_mmask_i32gather_pd() { let base_addr: [f64; 4] = [1., 2., 3., 4.]; let src = _mm256_setr_pd(9., 10., 11., 12.); let vindex = _mm_setr_epi32(1, 2, 3, 4); - let r = _mm256_mmask_i32gather_pd::<8>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = unsafe { _mm256_mmask_i32gather_pd::<8>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm256_setr_pd(2., 10., 4., 12.); assert_eq_m256d(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i32gather_ps() { + fn test_mm256_mmask_i32gather_ps() { let base_addr: [f32; 8] = [1., 2., 3., 4., 5., 6., 7., 8.]; let src = _mm256_setr_ps(9., 10., 11., 12., 13., 14., 15., 16.); let vindex = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); - let r = _mm256_mmask_i32gather_ps::<4>(src, 0b01010101, vindex, base_addr.as_ptr()); + let r = + unsafe { _mm256_mmask_i32gather_ps::<4>(src, 0b01010101, vindex, base_addr.as_ptr()) }; let expected = _mm256_setr_ps(2., 10., 4., 12., 6., 14., 8., 16.); assert_eq_m256(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i64gather_epi32() { + fn test_mm256_mmask_i64gather_epi32() { let base_addr: [i32; 4] = [1, 2, 3, 4]; let src = _mm_setr_epi32(9, 10, 11, 12); let vindex = _mm256_setr_epi64x(1, 2, 3, 0); - let r = _mm256_mmask_i64gather_epi32::<4>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = + unsafe { _mm256_mmask_i64gather_epi32::<4>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_epi32(2, 10, 4, 12); assert_eq_m128i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i64gather_epi64() { + fn test_mm256_mmask_i64gather_epi64() { let base_addr: [i64; 4] = [1, 2, 3, 4]; let src = _mm256_setr_epi64x(9, 10, 11, 12); let vindex = _mm256_setr_epi64x(1, 2, 3, 0); - let r = _mm256_mmask_i64gather_epi64::<8>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = + unsafe { _mm256_mmask_i64gather_epi64::<8>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm256_setr_epi64x(2, 10, 4, 12); assert_eq_m256i(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i64gather_pd() { + fn test_mm256_mmask_i64gather_pd() { let base_addr: [f64; 4] = [1., 2., 3., 4.]; let src = _mm256_setr_pd(9., 10., 11., 12.); let vindex = _mm256_setr_epi64x(1, 2, 3, 0); - let r = _mm256_mmask_i64gather_pd::<8>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = unsafe { _mm256_mmask_i64gather_pd::<8>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm256_setr_pd(2., 10., 4., 12.); assert_eq_m256d(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mmask_i64gather_ps() { + fn test_mm256_mmask_i64gather_ps() { let base_addr: [f32; 4] = [1., 2., 3., 4.]; let src = _mm_setr_ps(9., 10., 11., 12.); let vindex = _mm256_setr_epi64x(1, 2, 3, 0); - let r = _mm256_mmask_i64gather_ps::<4>(src, 0b0101, vindex, base_addr.as_ptr()); + let r = unsafe { _mm256_mmask_i64gather_ps::<4>(src, 0b0101, vindex, base_addr.as_ptr()) }; let expected = _mm_setr_ps(2., 10., 4., 12.); assert_eq_m128(expected, r); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i32scatter_epi32() { + fn test_mm_i32scatter_epi32() { let mut base_addr: [i32; 4] = [0; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm_setr_epi32(2, 3, 4, 1); - _mm_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2, 3, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i32scatter_epi32() { + fn test_mm_mask_i32scatter_epi32() { let mut base_addr: [i32; 4] = [0; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm_setr_epi32(2, 3, 4, 1); - _mm_mask_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm_mask_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0, 2, 0, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i32scatter_epi64() { + fn test_mm_i32scatter_epi64() { let mut base_addr: [i64; 2] = [0; 2]; let vindex = _mm_setr_epi32(1, 0, -1, -1); let src = _mm_setr_epi64x(2, 1); - _mm_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i32scatter_epi64() { + fn test_mm_mask_i32scatter_epi64() { let mut base_addr: [i64; 2] = [0; 2]; let vindex = _mm_setr_epi32(1, 0, -1, -1); let src = _mm_setr_epi64x(2, 1); - _mm_mask_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + unsafe { + _mm_mask_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + } let expected = [0, 2]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i32scatter_pd() { + fn test_mm_i32scatter_pd() { let mut base_addr: [f64; 2] = [0.; 2]; let vindex = _mm_setr_epi32(1, 0, -1, -1); let src = _mm_setr_pd(2., 1.); - _mm_i32scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i32scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i32scatter_pd() { + fn test_mm_mask_i32scatter_pd() { let mut base_addr: [f64; 2] = [0.; 2]; let vindex = _mm_setr_epi32(1, 0, -1, -1); let src = _mm_setr_pd(2., 1.); - _mm_mask_i32scatter_pd::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + unsafe { + _mm_mask_i32scatter_pd::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + } let expected = [0., 2.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i32scatter_ps() { + fn test_mm_i32scatter_ps() { let mut base_addr: [f32; 4] = [0.; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm_setr_ps(2., 3., 4., 1.); - _mm_i32scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i32scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2., 3., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i32scatter_ps() { + fn test_mm_mask_i32scatter_ps() { let mut base_addr: [f32; 4] = [0.; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm_setr_ps(2., 3., 4., 1.); - _mm_mask_i32scatter_ps::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm_mask_i32scatter_ps::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0., 2., 0., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i64scatter_epi32() { + fn test_mm_i64scatter_epi32() { let mut base_addr: [i32; 2] = [0; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_epi32(2, 1, -1, -1); - _mm_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i64scatter_epi32() { + fn test_mm_mask_i64scatter_epi32() { let mut base_addr: [i32; 2] = [0; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_epi32(2, 1, -1, -1); - _mm_mask_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b01, vindex, src); + unsafe { + _mm_mask_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b01, vindex, src); + } let expected = [0, 2]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i64scatter_epi64() { + fn test_mm_i64scatter_epi64() { let mut base_addr: [i64; 2] = [0; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_epi64x(2, 1); - _mm_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i64scatter_epi64() { + fn test_mm_mask_i64scatter_epi64() { let mut base_addr: [i64; 2] = [0; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_epi64x(2, 1); - _mm_mask_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + unsafe { + _mm_mask_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + } let expected = [0, 2]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i64scatter_pd() { + fn test_mm_i64scatter_pd() { let mut base_addr: [f64; 2] = [0.; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_pd(2., 1.); - _mm_i64scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i64scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i64scatter_pd() { + fn test_mm_mask_i64scatter_pd() { let mut base_addr: [f64; 2] = [0.; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_pd(2., 1.); - _mm_mask_i64scatter_pd::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + unsafe { + _mm_mask_i64scatter_pd::<8>(base_addr.as_mut_ptr(), 0b01, vindex, src); + } let expected = [0., 2.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_i64scatter_ps() { + fn test_mm_i64scatter_ps() { let mut base_addr: [f32; 2] = [0.; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_ps(2., 1., -1., -1.); - _mm_i64scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm_i64scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_i64scatter_ps() { + fn test_mm_mask_i64scatter_ps() { let mut base_addr: [f32; 2] = [0.; 2]; let vindex = _mm_setr_epi64x(1, 0); let src = _mm_setr_ps(2., 1., -1., -1.); - _mm_mask_i64scatter_ps::<4>(base_addr.as_mut_ptr(), 0b01, vindex, src); + unsafe { + _mm_mask_i64scatter_ps::<4>(base_addr.as_mut_ptr(), 0b01, vindex, src); + } let expected = [0., 2.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i32scatter_epi32() { + fn test_mm256_i32scatter_epi32() { let mut base_addr: [i32; 8] = [0; 8]; let vindex = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); let src = _mm256_setr_epi32(2, 3, 4, 5, 6, 7, 8, 1); - _mm256_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2, 3, 4, 5, 6, 7, 8]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i32scatter_epi32() { + fn test_mm256_mask_i32scatter_epi32() { let mut base_addr: [i32; 8] = [0; 8]; let vindex = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); let src = _mm256_setr_epi32(2, 3, 4, 5, 6, 7, 8, 1); - _mm256_mask_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + unsafe { + _mm256_mask_i32scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + } let expected = [0, 2, 0, 4, 0, 6, 0, 8]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i32scatter_epi64() { + fn test_mm256_i32scatter_epi64() { let mut base_addr: [i64; 4] = [0; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm256_setr_epi64x(2, 3, 4, 1); - _mm256_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2, 3, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i32scatter_epi64() { + fn test_mm256_mask_i32scatter_epi64() { let mut base_addr: [i64; 4] = [0; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm256_setr_epi64x(2, 3, 4, 1); - _mm256_mask_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm256_mask_i32scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0, 2, 0, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i32scatter_pd() { + fn test_mm256_i32scatter_pd() { let mut base_addr: [f64; 4] = [0.; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm256_setr_pd(2., 3., 4., 1.); - _mm256_i32scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i32scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2., 3., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i32scatter_pd() { + fn test_mm256_mask_i32scatter_pd() { let mut base_addr: [f64; 4] = [0.; 4]; let vindex = _mm_setr_epi32(1, 2, 3, 0); let src = _mm256_setr_pd(2., 3., 4., 1.); - _mm256_mask_i32scatter_pd::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm256_mask_i32scatter_pd::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0., 2., 0., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i32scatter_ps() { + fn test_mm256_i32scatter_ps() { let mut base_addr: [f32; 8] = [0.; 8]; let vindex = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); let src = _mm256_setr_ps(2., 3., 4., 5., 6., 7., 8., 1.); - _mm256_i32scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i32scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2., 3., 4., 5., 6., 7., 8.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i32scatter_ps() { + fn test_mm256_mask_i32scatter_ps() { let mut base_addr: [f32; 8] = [0.; 8]; let vindex = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); let src = _mm256_setr_ps(2., 3., 4., 5., 6., 7., 8., 1.); - _mm256_mask_i32scatter_ps::<4>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + unsafe { + _mm256_mask_i32scatter_ps::<4>(base_addr.as_mut_ptr(), 0b01010101, vindex, src); + } let expected = [0., 2., 0., 4., 0., 6., 0., 8.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i64scatter_epi32() { + fn test_mm256_i64scatter_epi32() { let mut base_addr: [i32; 4] = [0; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm_setr_epi32(2, 3, 4, 1); - _mm256_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2, 3, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i64scatter_epi32() { + fn test_mm256_mask_i64scatter_epi32() { let mut base_addr: [i32; 4] = [0; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm_setr_epi32(2, 3, 4, 1); - _mm256_mask_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm256_mask_i64scatter_epi32::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0, 2, 0, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i64scatter_epi64() { + fn test_mm256_i64scatter_epi64() { let mut base_addr: [i64; 4] = [0; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm256_setr_epi64x(2, 3, 4, 1); - _mm256_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1, 2, 3, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i64scatter_epi64() { + fn test_mm256_mask_i64scatter_epi64() { let mut base_addr: [i64; 4] = [0; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm256_setr_epi64x(2, 3, 4, 1); - _mm256_mask_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm256_mask_i64scatter_epi64::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0, 2, 0, 4]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i64scatter_pd() { + fn test_mm256_i64scatter_pd() { let mut base_addr: [f64; 4] = [0.; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm256_setr_pd(2., 3., 4., 1.); - _mm256_i64scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i64scatter_pd::<8>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2., 3., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i64scatter_pd() { + fn test_mm256_mask_i64scatter_pd() { let mut base_addr: [f64; 4] = [0.; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm256_setr_pd(2., 3., 4., 1.); - _mm256_mask_i64scatter_pd::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm256_mask_i64scatter_pd::<8>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0., 2., 0., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_i64scatter_ps() { + fn test_mm256_i64scatter_ps() { let mut base_addr: [f32; 4] = [0.; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm_setr_ps(2., 3., 4., 1.); - _mm256_i64scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + unsafe { + _mm256_i64scatter_ps::<4>(base_addr.as_mut_ptr(), vindex, src); + } let expected = [1., 2., 3., 4.]; assert_eq!(expected, base_addr); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_i64scatter_ps() { + fn test_mm256_mask_i64scatter_ps() { let mut base_addr: [f32; 4] = [0.; 4]; let vindex = _mm256_setr_epi64x(1, 2, 3, 0); let src = _mm_setr_ps(2., 3., 4., 1.); - _mm256_mask_i64scatter_ps::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + unsafe { + _mm256_mask_i64scatter_ps::<4>(base_addr.as_mut_ptr(), 0b0101, vindex, src); + } let expected = [0., 2., 0., 4.]; assert_eq!(expected, base_addr); } @@ -12168,100 +12275,116 @@ mod tests { } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_loadu_epi64() { + const fn test_mm512_loadu_epi64() { let a = &[4, 3, 2, 5, -8, -9, -64, -50]; let p = a.as_ptr(); - let r = _mm512_loadu_epi64(black_box(p)); + let r = unsafe { _mm512_loadu_epi64(black_box(p)) }; let e = _mm512_setr_epi64(4, 3, 2, 5, -8, -9, -64, -50); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_loadu_epi64() { + const fn test_mm256_loadu_epi64() { let a = &[4, 3, 2, 5]; let p = a.as_ptr(); - let r = _mm256_loadu_epi64(black_box(p)); + let r = unsafe { _mm256_loadu_epi64(black_box(p)) }; let e = _mm256_setr_epi64x(4, 3, 2, 5); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_loadu_epi64() { + const fn test_mm_loadu_epi64() { let a = &[4, 3]; let p = a.as_ptr(); - let r = _mm_loadu_epi64(black_box(p)); + let r = unsafe { _mm_loadu_epi64(black_box(p)) }; let e = _mm_setr_epi64x(4, 3); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtepi64_storeu_epi16() { + fn test_mm512_mask_cvtepi64_storeu_epi16() { let a = _mm512_set1_epi64(9); let mut r = _mm_undefined_si128(); - _mm512_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm512_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set1_epi16(9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtepi64_storeu_epi16() { + fn test_mm256_mask_cvtepi64_storeu_epi16() { let a = _mm256_set1_epi64x(9); let mut r = _mm_set1_epi16(0); - _mm256_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm256_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, 9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtepi64_storeu_epi16() { + fn test_mm_mask_cvtepi64_storeu_epi16() { let a = _mm_set1_epi64x(9); let mut r = _mm_set1_epi16(0); - _mm_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm_mask_cvtepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtsepi64_storeu_epi16() { + fn test_mm512_mask_cvtsepi64_storeu_epi16() { let a = _mm512_set1_epi64(i64::MAX); let mut r = _mm_undefined_si128(); - _mm512_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm512_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set1_epi16(i16::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtsepi64_storeu_epi16() { + fn test_mm256_mask_cvtsepi64_storeu_epi16() { let a = _mm256_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi16(0); - _mm256_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm256_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, i16::MAX, i16::MAX, i16::MAX, i16::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtsepi64_storeu_epi16() { + fn test_mm_mask_cvtsepi64_storeu_epi16() { let a = _mm_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi16(0); - _mm_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm_mask_cvtsepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, i16::MAX, i16::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtusepi64_storeu_epi16() { + fn test_mm512_mask_cvtusepi64_storeu_epi16() { let a = _mm512_set1_epi64(i64::MAX); let mut r = _mm_undefined_si128(); - _mm512_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm512_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set1_epi16(u16::MAX as i16); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtusepi64_storeu_epi16() { + fn test_mm256_mask_cvtusepi64_storeu_epi16() { let a = _mm256_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi16(0); - _mm256_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm256_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16( 0, 0, @@ -12276,46 +12399,56 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtusepi64_storeu_epi16() { + fn test_mm_mask_cvtusepi64_storeu_epi16() { let a = _mm_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi16(0); - _mm_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + unsafe { + _mm_mask_cvtusepi64_storeu_epi16(&mut r as *mut _ as *mut i16, 0b11111111, a); + } let e = _mm_set_epi16(0, 0, 0, 0, 0, 0, u16::MAX as i16, u16::MAX as i16); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtepi64_storeu_epi8() { + fn test_mm512_mask_cvtepi64_storeu_epi8() { let a = _mm512_set1_epi64(9); let mut r = _mm_set1_epi8(0); - _mm512_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm512_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtepi64_storeu_epi8() { + fn test_mm256_mask_cvtepi64_storeu_epi8() { let a = _mm256_set1_epi64x(9); let mut r = _mm_set1_epi8(0); - _mm256_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm256_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtepi64_storeu_epi8() { + fn test_mm_mask_cvtepi64_storeu_epi8() { let a = _mm_set1_epi64x(9); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtsepi64_storeu_epi8() { + fn test_mm512_mask_cvtsepi64_storeu_epi8() { let a = _mm512_set1_epi64(i64::MAX); let mut r = _mm_set1_epi8(0); - _mm512_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm512_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -12327,10 +12460,12 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtsepi64_storeu_epi8() { + fn test_mm256_mask_cvtsepi64_storeu_epi8() { let a = _mm256_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi8(0); - _mm256_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm256_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -12342,19 +12477,23 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtsepi64_storeu_epi8() { + fn test_mm_mask_cvtsepi64_storeu_epi8() { let a = _mm_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtsepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } let e = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, i8::MAX, i8::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtusepi64_storeu_epi8() { + fn test_mm512_mask_cvtusepi64_storeu_epi8() { let a = _mm512_set1_epi64(i64::MAX); let mut r = _mm_set1_epi8(0); - _mm512_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm512_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -12366,10 +12505,12 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtusepi64_storeu_epi8() { + fn test_mm256_mask_cvtusepi64_storeu_epi8() { let a = _mm256_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi8(0); - _mm256_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm256_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -12381,10 +12522,12 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtusepi64_storeu_epi8() { + fn test_mm_mask_cvtusepi64_storeu_epi8() { let a = _mm_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi8(0); - _mm_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + unsafe { + _mm_mask_cvtusepi64_storeu_epi8(&mut r as *mut _ as *mut i8, 0b11111111, a); + } #[rustfmt::skip] let e = _mm_set_epi8( 0, 0, 0, 0, @@ -12396,112 +12539,136 @@ mod tests { } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtepi64_storeu_epi32() { + fn test_mm512_mask_cvtepi64_storeu_epi32() { let a = _mm512_set1_epi64(9); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + unsafe { + _mm512_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + } let e = _mm256_set1_epi32(9); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtepi64_storeu_epi32() { + fn test_mm256_mask_cvtepi64_storeu_epi32() { let a = _mm256_set1_epi64x(9); let mut r = _mm_set1_epi32(0); - _mm256_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + unsafe { + _mm256_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + } let e = _mm_set_epi32(9, 9, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtepi64_storeu_epi32() { + fn test_mm_mask_cvtepi64_storeu_epi32() { let a = _mm_set1_epi64x(9); let mut r = _mm_set1_epi16(0); - _mm_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + unsafe { + _mm_mask_cvtepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + } let e = _mm_set_epi32(0, 0, 9, 9); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtsepi64_storeu_epi32() { + fn test_mm512_mask_cvtsepi64_storeu_epi32() { let a = _mm512_set1_epi64(i64::MAX); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + unsafe { + _mm512_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + } let e = _mm256_set1_epi32(i32::MAX); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtsepi64_storeu_epi32() { + fn test_mm256_mask_cvtsepi64_storeu_epi32() { let a = _mm256_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi32(0); - _mm256_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00001111, a); + unsafe { + _mm256_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00001111, a); + } let e = _mm_set1_epi32(i32::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtsepi64_storeu_epi32() { + fn test_mm_mask_cvtsepi64_storeu_epi32() { let a = _mm_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi16(0); - _mm_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00000011, a); + unsafe { + _mm_mask_cvtsepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00000011, a); + } let e = _mm_set_epi32(0, 0, i32::MAX, i32::MAX); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_cvtusepi64_storeu_epi32() { + fn test_mm512_mask_cvtusepi64_storeu_epi32() { let a = _mm512_set1_epi64(i64::MAX); let mut r = _mm256_undefined_si256(); - _mm512_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + unsafe { + _mm512_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b11111111, a); + } let e = _mm256_set1_epi32(u32::MAX as i32); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm256_mask_cvtusepi64_storeu_epi32() { + fn test_mm256_mask_cvtusepi64_storeu_epi32() { let a = _mm256_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi32(0); - _mm256_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00001111, a); + unsafe { + _mm256_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00001111, a); + } let e = _mm_set1_epi32(u32::MAX as i32); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - unsafe fn test_mm_mask_cvtusepi64_storeu_epi32() { + fn test_mm_mask_cvtusepi64_storeu_epi32() { let a = _mm_set1_epi64x(i64::MAX); let mut r = _mm_set1_epi16(0); - _mm_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00000011, a); + unsafe { + _mm_mask_cvtusepi64_storeu_epi32(&mut r as *mut _ as *mut i32, 0b00000011, a); + } let e = _mm_set_epi32(0, 0, u32::MAX as i32, u32::MAX as i32); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_storeu_epi64() { + const fn test_mm512_storeu_epi64() { let a = _mm512_set1_epi64(9); let mut r = _mm512_set1_epi64(0); - _mm512_storeu_epi64(&mut r as *mut _ as *mut i64, a); + unsafe { + _mm512_storeu_epi64(&mut r as *mut _ as *mut i64, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_storeu_epi64() { + const fn test_mm256_storeu_epi64() { let a = _mm256_set1_epi64x(9); let mut r = _mm256_set1_epi64x(0); - _mm256_storeu_epi64(&mut r as *mut _ as *mut i64, a); + unsafe { + _mm256_storeu_epi64(&mut r as *mut _ as *mut i64, a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_storeu_epi64() { + const fn test_mm_storeu_epi64() { let a = _mm_set1_epi64x(9); let mut r = _mm_set1_epi64x(0); - _mm_storeu_epi64(&mut r as *mut _ as *mut i64, a); + unsafe { + _mm_storeu_epi64(&mut r as *mut _ as *mut i64, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_load_epi64() { + const fn test_mm512_load_epi64() { #[repr(align(64))] struct Align { data: [i64; 8], // 64 bytes @@ -12510,63 +12677,69 @@ mod tests { data: [4, 3, 2, 5, -8, -9, -64, -50], }; let p = (a.data).as_ptr(); - let r = _mm512_load_epi64(black_box(p)); + let r = unsafe { _mm512_load_epi64(black_box(p)) }; let e = _mm512_setr_epi64(4, 3, 2, 5, -8, -9, -64, -50); assert_eq_m512i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_load_epi64() { + const fn test_mm256_load_epi64() { #[repr(align(64))] struct Align { data: [i64; 4], } let a = Align { data: [4, 3, 2, 5] }; let p = (a.data).as_ptr(); - let r = _mm256_load_epi64(black_box(p)); + let r = unsafe { _mm256_load_epi64(black_box(p)) }; let e = _mm256_set_epi64x(5, 2, 3, 4); assert_eq_m256i(r, e); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_load_epi64() { + const fn test_mm_load_epi64() { #[repr(align(64))] struct Align { data: [i64; 2], } let a = Align { data: [4, 3] }; let p = (a.data).as_ptr(); - let r = _mm_load_epi64(black_box(p)); + let r = unsafe { _mm_load_epi64(black_box(p)) }; let e = _mm_set_epi64x(3, 4); assert_eq_m128i(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_store_epi64() { + const fn test_mm512_store_epi64() { let a = _mm512_set1_epi64(9); let mut r = _mm512_set1_epi64(0); - _mm512_store_epi64(&mut r as *mut _ as *mut i64, a); + unsafe { + _mm512_store_epi64(&mut r as *mut _ as *mut i64, a); + } assert_eq_m512i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm256_store_epi64() { + const fn test_mm256_store_epi64() { let a = _mm256_set1_epi64x(9); let mut r = _mm256_set1_epi64x(0); - _mm256_store_epi64(&mut r as *mut _ as *mut i64, a); + unsafe { + _mm256_store_epi64(&mut r as *mut _ as *mut i64, a); + } assert_eq_m256i(r, a); } #[simd_test(enable = "avx512f,avx512vl")] - const unsafe fn test_mm_store_epi64() { + const fn test_mm_store_epi64() { let a = _mm_set1_epi64x(9); let mut r = _mm_set1_epi64x(0); - _mm_store_epi64(&mut r as *mut _ as *mut i64, a); + unsafe { + _mm_store_epi64(&mut r as *mut _ as *mut i64, a); + } assert_eq_m128i(r, a); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_load_pd() { + const fn test_mm512_load_pd() { #[repr(align(64))] struct Align { data: [f64; 8], // 64 bytes @@ -12575,16 +12748,18 @@ mod tests { data: [4., 3., 2., 5., -8., -9., -64., -50.], }; let p = (a.data).as_ptr(); - let r = _mm512_load_pd(black_box(p)); + let r = unsafe { _mm512_load_pd(black_box(p)) }; let e = _mm512_setr_pd(4., 3., 2., 5., -8., -9., -64., -50.); assert_eq_m512d(r, e); } #[simd_test(enable = "avx512f")] - const unsafe fn test_mm512_store_pd() { + const fn test_mm512_store_pd() { let a = _mm512_set1_pd(9.); let mut r = _mm512_undefined_pd(); - _mm512_store_pd(&mut r as *mut _ as *mut f64, a); + unsafe { + _mm512_store_pd(&mut r as *mut _ as *mut f64, a); + } assert_eq_m512d(r, a); } diff --git a/library/stdarch/crates/core_arch/src/x86_64/fxsr.rs b/library/stdarch/crates/core_arch/src/x86_64/fxsr.rs index a24b44fb1f7e..28bf1951167a 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/fxsr.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/fxsr.rs @@ -77,12 +77,14 @@ mod tests { #[simd_test(enable = "fxsr")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_fxsave64() { + fn test_fxsave64() { let mut a = FxsaveArea::new(); let mut b = FxsaveArea::new(); - fxsr::_fxsave64(a.ptr()); - fxsr::_fxrstor64(a.ptr()); - fxsr::_fxsave64(b.ptr()); + unsafe { + fxsr::_fxsave64(a.ptr()); + fxsr::_fxrstor64(a.ptr()); + fxsr::_fxsave64(b.ptr()); + } } } diff --git a/library/stdarch/crates/core_arch/src/x86_64/sse2.rs b/library/stdarch/crates/core_arch/src/x86_64/sse2.rs index b156af078a32..08dabf053d42 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/sse2.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/sse2.rs @@ -204,10 +204,12 @@ mod tests { // Miri cannot support this until it is clear how it fits in the Rust memory model // (non-temporal store) #[cfg_attr(miri, ignore)] - unsafe fn test_mm_stream_si64() { + fn test_mm_stream_si64() { let a: i64 = 7; let mut mem = boxed::Box::::new(-1); - _mm_stream_si64(ptr::addr_of_mut!(*mem), a); + unsafe { + _mm_stream_si64(ptr::addr_of_mut!(*mem), a); + } _mm_sfence(); assert_eq!(a, *mem); } diff --git a/library/stdarch/crates/core_arch/src/x86_64/xsave.rs b/library/stdarch/crates/core_arch/src/x86_64/xsave.rs index fa1454a822e3..30a7123315e5 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/xsave.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/xsave.rs @@ -132,37 +132,43 @@ mod tests { #[simd_test(enable = "xsave")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xsave64() { + fn test_xsave64() { let m = 0xFFFFFFFFFFFFFFFF_u64; //< all registers let mut a = XsaveArea::new(); let mut b = XsaveArea::new(); - _xsave64(a.ptr(), m); - _xrstor64(a.ptr(), m); - _xsave64(b.ptr(), m); + unsafe { + _xsave64(a.ptr(), m); + _xrstor64(a.ptr(), m); + _xsave64(b.ptr(), m); + } } #[simd_test(enable = "xsave,xsaveopt")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xsaveopt64() { + fn test_xsaveopt64() { let m = 0xFFFFFFFFFFFFFFFF_u64; //< all registers let mut a = XsaveArea::new(); let mut b = XsaveArea::new(); - _xsaveopt64(a.ptr(), m); - _xrstor64(a.ptr(), m); - _xsaveopt64(b.ptr(), m); + unsafe { + _xsaveopt64(a.ptr(), m); + _xrstor64(a.ptr(), m); + _xsaveopt64(b.ptr(), m); + } } #[simd_test(enable = "xsave,xsavec")] #[cfg_attr(miri, ignore)] // Register saving/restoring is not supported in Miri - unsafe fn test_xsavec64() { + fn test_xsavec64() { let m = 0xFFFFFFFFFFFFFFFF_u64; //< all registers let mut a = XsaveArea::new(); let mut b = XsaveArea::new(); - _xsavec64(a.ptr(), m); - _xrstor64(a.ptr(), m); - _xsavec64(b.ptr(), m); + unsafe { + _xsavec64(a.ptr(), m); + _xrstor64(a.ptr(), m); + _xsavec64(b.ptr(), m); + } } } From dc64aef1ed2ea86e42fe83f10e00c2f362dec114 Mon Sep 17 00:00:00 2001 From: Shoyu Vanilla Date: Sun, 4 Jan 2026 19:10:10 +0900 Subject: [PATCH 0099/1843] fix: Properly lower `SelfOnly` predicates --- .../rust-analyzer/crates/hir-ty/src/lower.rs | 369 +++++++++++++----- .../crates/hir-ty/src/lower/path.rs | 34 +- .../crates/hir-ty/src/next_solver/interner.rs | 7 +- .../crates/hir-ty/src/next_solver/util.rs | 15 + .../hir-ty/src/tests/regression/new_solver.rs | 60 +++ 5 files changed, 373 insertions(+), 112 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs b/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs index 46ec554e0a65..9da32464c86f 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs @@ -77,6 +77,7 @@ pub struct ImplTraits { #[derive(PartialEq, Eq, Debug, Hash)] pub struct ImplTrait { pub(crate) predicates: StoredClauses, + pub(crate) assoc_ty_bounds_start: u32, } pub type ImplTraitIdx = Idx; @@ -166,6 +167,12 @@ impl<'db> LifetimeElisionKind<'db> { } } +#[derive(Clone, Copy, PartialEq, Debug)] +pub(crate) enum GenericPredicateSource { + SelfOnly, + AssocTyBound, +} + #[derive(Debug)] pub struct TyLoweringContext<'db, 'a> { pub db: &'db dyn HirDatabase, @@ -465,10 +472,10 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { // this dance is to make sure the data is in the right // place even if we encounter more opaque types while // lowering the bounds - let idx = self - .impl_trait_mode - .opaque_type_data - .alloc(ImplTrait { predicates: Clauses::empty(interner).store() }); + let idx = self.impl_trait_mode.opaque_type_data.alloc(ImplTrait { + predicates: Clauses::empty(interner).store(), + assoc_ty_bounds_start: 0, + }); let impl_trait_id = origin.either( |f| ImplTraitId::ReturnTypeImplTrait(f, idx), @@ -608,7 +615,7 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { ignore_bindings: bool, generics: &Generics, predicate_filter: PredicateFilter, - ) -> impl Iterator> + use<'a, 'b, 'db> { + ) -> impl Iterator, GenericPredicateSource)> + use<'a, 'b, 'db> { match where_predicate { WherePredicate::ForLifetime { target, bound, .. } | WherePredicate::TypeBound { target, bound } => { @@ -634,8 +641,8 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { let self_ty = self.lower_ty(*target); Either::Left(Either::Right(self.lower_type_bound(bound, self_ty, ignore_bindings))) } - &WherePredicate::Lifetime { bound, target } => { - Either::Right(iter::once(Clause(Predicate::new( + &WherePredicate::Lifetime { bound, target } => Either::Right(iter::once(( + Clause(Predicate::new( self.interner, Binder::dummy(rustc_type_ir::PredicateKind::Clause( rustc_type_ir::ClauseKind::RegionOutlives(OutlivesPredicate( @@ -643,8 +650,9 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { self.lower_lifetime(target), )), )), - )))) - } + )), + GenericPredicateSource::SelfOnly, + ))), } .into_iter() } @@ -654,7 +662,7 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { bound: &'b TypeBound, self_ty: Ty<'db>, ignore_bindings: bool, - ) -> impl Iterator> + use<'b, 'a, 'db> { + ) -> impl Iterator, GenericPredicateSource)> + use<'b, 'a, 'db> { let interner = self.interner; let meta_sized = self.lang_items.MetaSized; let pointee_sized = self.lang_items.PointeeSized; @@ -712,7 +720,10 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { } TypeBound::Use(_) | TypeBound::Error => {} } - clause.into_iter().chain(assoc_bounds.into_iter().flatten()) + clause + .into_iter() + .map(|pred| (pred, GenericPredicateSource::SelfOnly)) + .chain(assoc_bounds.into_iter().flatten()) } fn lower_dyn_trait(&mut self, bounds: &[TypeBound]) -> Ty<'db> { @@ -732,7 +743,7 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { for b in bounds { let db = ctx.db; - ctx.lower_type_bound(b, dummy_self_ty, false).for_each(|b| { + ctx.lower_type_bound(b, dummy_self_ty, false).for_each(|(b, _)| { match b.kind().skip_binder() { rustc_type_ir::ClauseKind::Trait(t) => { let id = t.def_id(); @@ -990,35 +1001,49 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { rustc_type_ir::AliasTyKind::Opaque, AliasTy::new_from_args(interner, def_id, args), ); - let predicates = self.with_shifted_in(DebruijnIndex::from_u32(1), |ctx| { - let mut predicates = Vec::new(); - for b in bounds { - predicates.extend(ctx.lower_type_bound(b, self_ty, false)); - } + let (predicates, assoc_ty_bounds_start) = + self.with_shifted_in(DebruijnIndex::from_u32(1), |ctx| { + let mut predicates = Vec::new(); + let mut assoc_ty_bounds = Vec::new(); + for b in bounds { + for (pred, source) in ctx.lower_type_bound(b, self_ty, false) { + match source { + GenericPredicateSource::SelfOnly => predicates.push(pred), + GenericPredicateSource::AssocTyBound => assoc_ty_bounds.push(pred), + } + } + } - if !ctx.unsized_types.contains(&self_ty) { - let sized_trait = self.lang_items.Sized; - let sized_clause = sized_trait.map(|trait_id| { - let trait_ref = TraitRef::new_from_args( - interner, - trait_id.into(), - GenericArgs::new_from_slice(&[self_ty.into()]), - ); - Clause(Predicate::new( - interner, - Binder::dummy(rustc_type_ir::PredicateKind::Clause( - rustc_type_ir::ClauseKind::Trait(TraitPredicate { - trait_ref, - polarity: rustc_type_ir::PredicatePolarity::Positive, - }), - )), - )) - }); - predicates.extend(sized_clause); - } - predicates - }); - ImplTrait { predicates: Clauses::new_from_slice(&predicates).store() } + if !ctx.unsized_types.contains(&self_ty) { + let sized_trait = self.lang_items.Sized; + let sized_clause = sized_trait.map(|trait_id| { + let trait_ref = TraitRef::new_from_args( + interner, + trait_id.into(), + GenericArgs::new_from_slice(&[self_ty.into()]), + ); + Clause(Predicate::new( + interner, + Binder::dummy(rustc_type_ir::PredicateKind::Clause( + rustc_type_ir::ClauseKind::Trait(TraitPredicate { + trait_ref, + polarity: rustc_type_ir::PredicatePolarity::Positive, + }), + )), + )) + }); + predicates.extend(sized_clause); + } + + let assoc_ty_bounds_start = predicates.len() as u32; + predicates.extend(assoc_ty_bounds); + (predicates, assoc_ty_bounds_start) + }); + + ImplTrait { + predicates: Clauses::new_from_slice(&predicates).store(), + assoc_ty_bounds_start, + } } pub(crate) fn lower_lifetime(&mut self, lifetime: LifetimeRefId) -> Region<'db> { @@ -1139,6 +1164,31 @@ impl ImplTraitId { .expect("owner should have opaque type") .get_with(|it| it.impl_traits[idx].predicates.as_ref().as_slice()) } + + #[inline] + pub fn self_predicates<'db>( + self, + db: &'db dyn HirDatabase, + ) -> EarlyBinder<'db, &'db [Clause<'db>]> { + let (impl_traits, idx) = match self { + ImplTraitId::ReturnTypeImplTrait(owner, idx) => { + (ImplTraits::return_type_impl_traits(db, owner), idx) + } + ImplTraitId::TypeAliasImplTrait(owner, idx) => { + (ImplTraits::type_alias_impl_traits(db, owner), idx) + } + }; + let predicates = + impl_traits.as_deref().expect("owner should have opaque type").get_with(|it| { + let impl_trait = &it.impl_traits[idx]; + ( + impl_trait.predicates.as_ref().as_slice(), + impl_trait.assoc_ty_bounds_start as usize, + ) + }); + + predicates.map_bound(|(preds, len)| &preds[..len]) + } } impl InternedOpaqueTyId { @@ -1146,6 +1196,14 @@ impl InternedOpaqueTyId { pub fn predicates<'db>(self, db: &'db dyn HirDatabase) -> EarlyBinder<'db, &'db [Clause<'db>]> { self.loc(db).predicates(db) } + + #[inline] + pub fn self_predicates<'db>( + self, + db: &'db dyn HirDatabase, + ) -> EarlyBinder<'db, &'db [Clause<'db>]> { + self.loc(db).self_predicates(db) + } } #[salsa::tracked] @@ -1655,12 +1713,15 @@ pub(crate) fn generic_predicates_for_param<'db>( ctx.store = maybe_parent_generics.store(); for pred in maybe_parent_generics.where_predicates() { if predicate(pred, &mut ctx) { - predicates.extend(ctx.lower_where_predicate( - pred, - true, - maybe_parent_generics, - PredicateFilter::All, - )); + predicates.extend( + ctx.lower_where_predicate( + pred, + true, + maybe_parent_generics, + PredicateFilter::All, + ) + .map(|(pred, _)| pred), + ); } } } @@ -1696,21 +1757,44 @@ pub(crate) fn type_alias_bounds<'db>( db: &'db dyn HirDatabase, type_alias: TypeAliasId, ) -> EarlyBinder<'db, &'db [Clause<'db>]> { - type_alias_bounds_with_diagnostics(db, type_alias).0.map_bound(|it| it.as_slice()) + type_alias_bounds_with_diagnostics(db, type_alias).0.predicates.map_bound(|it| it.as_slice()) } -pub(crate) fn type_alias_bounds_with_diagnostics<'db>( +#[inline] +pub(crate) fn type_alias_self_bounds<'db>( db: &'db dyn HirDatabase, type_alias: TypeAliasId, -) -> (EarlyBinder<'db, Clauses<'db>>, Diagnostics) { - let (bounds, diags) = type_alias_bounds_with_diagnostics_query(db, type_alias); - return (bounds.get(), diags.clone()); +) -> EarlyBinder<'db, &'db [Clause<'db>]> { + let (TypeAliasBounds { predicates, assoc_ty_bounds_start }, _) = + type_alias_bounds_with_diagnostics(db, type_alias); + predicates.map_bound(|it| &it.as_slice()[..assoc_ty_bounds_start as usize]) +} + +#[derive(PartialEq, Eq, Debug, Hash)] +struct TypeAliasBounds { + predicates: T, + assoc_ty_bounds_start: u32, +} + +fn type_alias_bounds_with_diagnostics<'db>( + db: &'db dyn HirDatabase, + type_alias: TypeAliasId, +) -> (TypeAliasBounds>>, Diagnostics) { + let (TypeAliasBounds { predicates, assoc_ty_bounds_start }, diags) = + type_alias_bounds_with_diagnostics_query(db, type_alias); + return ( + TypeAliasBounds { + predicates: predicates.get(), + assoc_ty_bounds_start: *assoc_ty_bounds_start, + }, + diags.clone(), + ); #[salsa::tracked(returns(ref))] pub fn type_alias_bounds_with_diagnostics_query<'db>( db: &'db dyn HirDatabase, type_alias: TypeAliasId, - ) -> (StoredEarlyBinder, Diagnostics) { + ) -> (TypeAliasBounds>, Diagnostics) { let type_alias_data = db.type_alias_signature(type_alias); let resolver = hir_def::resolver::HasResolver::resolver(type_alias, db); let mut ctx = TyLoweringContext::new( @@ -1727,10 +1811,18 @@ pub(crate) fn type_alias_bounds_with_diagnostics<'db>( let interner_ty = Ty::new_projection_from_args(interner, def_id, item_args); let mut bounds = Vec::new(); + let mut assoc_ty_bounds = Vec::new(); for bound in &type_alias_data.bounds { - ctx.lower_type_bound(bound, interner_ty, false).for_each(|pred| { - bounds.push(pred); - }); + ctx.lower_type_bound(bound, interner_ty, false).for_each( + |(pred, source)| match source { + GenericPredicateSource::SelfOnly => { + bounds.push(pred); + } + GenericPredicateSource::AssocTyBound => { + assoc_ty_bounds.push(pred); + } + }, + ); } if !ctx.unsized_types.contains(&interner_ty) { @@ -1745,8 +1837,14 @@ pub(crate) fn type_alias_bounds_with_diagnostics<'db>( }; } + let assoc_ty_bounds_start = bounds.len() as u32; + bounds.extend(assoc_ty_bounds); + ( - StoredEarlyBinder::bind(Clauses::new_from_slice(&bounds).store()), + TypeAliasBounds { + predicates: StoredEarlyBinder::bind(Clauses::new_from_slice(&bounds).store()), + assoc_ty_bounds_start, + }, create_diagnostics(ctx.diagnostics), ) } @@ -1754,11 +1852,15 @@ pub(crate) fn type_alias_bounds_with_diagnostics<'db>( #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct GenericPredicates { - // The order is the following: first, if `parent_is_trait == true`, comes the implicit trait predicate for the - // parent. Then come the explicit predicates for the parent, then the explicit trait predicate for the child, + // The order is the following: first, if `parent_is_trait == true`, comes the implicit trait + // predicate for the parent. Then come the bounds of the associated types of the parents, + // then the explicit, self-only predicates for the parent, then the explicit, self-only trait + // predicate for the child, then the bounds of the associated types of the child, // then the implicit trait predicate for the child, if `is_trait` is `true`. predicates: StoredEarlyBinder, + parent_explicit_self_predicates_start: u32, own_predicates_start: u32, + own_assoc_ty_bounds_start: u32, is_trait: bool, parent_is_trait: bool, } @@ -1782,7 +1884,15 @@ impl GenericPredicates { pub(crate) fn from_explicit_own_predicates( predicates: StoredEarlyBinder, ) -> Self { - Self { predicates, own_predicates_start: 0, is_trait: false, parent_is_trait: false } + let len = predicates.get().skip_binder().len() as u32; + Self { + predicates, + parent_explicit_self_predicates_start: 0, + own_predicates_start: 0, + own_assoc_ty_bounds_start: len, + is_trait: false, + parent_is_trait: false, + } } #[inline] @@ -1814,6 +1924,14 @@ impl GenericPredicates { Self::query(db, def).explicit_predicates() } + #[inline] + pub fn query_explicit_implied<'db>( + db: &'db dyn HirDatabase, + def: GenericDefId, + ) -> EarlyBinder<'db, &'db [Clause<'db>]> { + Self::query(db, def).explicit_implied_predicates() + } + #[inline] pub fn all_predicates(&self) -> EarlyBinder<'_, &[Clause<'_>]> { self.predicates.get().map_bound(|it| it.as_slice()) @@ -1824,9 +1942,18 @@ impl GenericPredicates { self.predicates.get().map_bound(|it| &it.as_slice()[self.own_predicates_start as usize..]) } - /// Returns the predicates, minus the implicit `Self: Trait` predicate for a trait. + /// Returns the predicates, minus the implicit `Self: Trait` predicate and bounds of the + /// associated types for a trait. #[inline] pub fn explicit_predicates(&self) -> EarlyBinder<'_, &[Clause<'_>]> { + self.predicates.get().map_bound(|it| { + &it.as_slice()[self.parent_explicit_self_predicates_start as usize + ..self.own_assoc_ty_bounds_start as usize] + }) + } + + #[inline] + pub fn explicit_implied_predicates(&self) -> EarlyBinder<'_, &[Clause<'_>]> { self.predicates.get().map_bound(|it| { &it.as_slice()[usize::from(self.parent_is_trait)..it.len() - usize::from(self.is_trait)] }) @@ -1902,26 +2029,22 @@ where ); let sized_trait = ctx.lang_items.Sized; - let mut predicates = Vec::new(); + // We need to lower parents and self separately - see the comment below lowering of implicit + // `Sized` predicates for why. + let mut own_predicates = Vec::new(); + let mut parent_predicates = Vec::new(); + let mut own_assoc_ty_bounds = Vec::new(); + let mut parent_assoc_ty_bounds = Vec::new(); let all_generics = std::iter::successors(Some(&generics), |generics| generics.parent_generics()) .collect::>(); - let mut is_trait = false; - let mut parent_is_trait = false; - if all_generics.len() > 1 { - add_implicit_trait_predicate( - interner, - all_generics.last().unwrap().def(), - predicate_filter, - &mut predicates, - &mut parent_is_trait, - ); - } - // We need to lower parent predicates first - see the comment below lowering of implicit `Sized` predicates - // for why. - let mut own_predicates_start = 0; + let own_implicit_trait_predicate = implicit_trait_predicate(interner, def, predicate_filter); + let parent_implicit_trait_predicate = if all_generics.len() > 1 { + implicit_trait_predicate(interner, all_generics.last().unwrap().def(), predicate_filter) + } else { + None + }; for &maybe_parent_generics in all_generics.iter().rev() { - let current_def_predicates_start = predicates.len(); // Collect only diagnostics from the child, not including parents. ctx.diagnostics.clear(); @@ -1929,15 +2052,37 @@ where ctx.store = maybe_parent_generics.store(); for pred in maybe_parent_generics.where_predicates() { tracing::debug!(?pred); - predicates.extend(ctx.lower_where_predicate( - pred, - false, - maybe_parent_generics, - predicate_filter, - )); + for (pred, source) in + ctx.lower_where_predicate(pred, false, maybe_parent_generics, predicate_filter) + { + match source { + GenericPredicateSource::SelfOnly => { + if maybe_parent_generics.def() == def { + own_predicates.push(pred); + } else { + parent_predicates.push(pred); + } + } + GenericPredicateSource::AssocTyBound => { + if maybe_parent_generics.def() == def { + own_assoc_ty_bounds.push(pred); + } else { + parent_assoc_ty_bounds.push(pred); + } + } + } + } } - push_const_arg_has_type_predicates(db, &mut predicates, maybe_parent_generics); + if maybe_parent_generics.def() == def { + push_const_arg_has_type_predicates(db, &mut own_predicates, maybe_parent_generics); + } else { + push_const_arg_has_type_predicates( + db, + &mut parent_predicates, + maybe_parent_generics, + ); + } if let Some(sized_trait) = sized_trait { let mut add_sized_clause = |param_idx, param_id, param_data| { @@ -1971,7 +2116,11 @@ where }), )), )); - predicates.push(clause); + if maybe_parent_generics.def() == def { + own_predicates.push(clause); + } else { + parent_predicates.push(clause); + } }; let parent_params_len = maybe_parent_generics.len_parent(); maybe_parent_generics.iter_self().enumerate().for_each( @@ -1990,30 +2139,55 @@ where // predicates before lowering the child, as a child cannot define a `?Sized` predicate for its parent. // But we do have to lower the parent first. } - - if maybe_parent_generics.def() == def { - own_predicates_start = current_def_predicates_start as u32; - } } - add_implicit_trait_predicate(interner, def, predicate_filter, &mut predicates, &mut is_trait); - let diagnostics = create_diagnostics(ctx.diagnostics); + + // The order is: + // + // 1. parent implicit trait pred + // 2. parent assoc bounds + // 3. parent self only preds + // 4. own self only preds + // 5. own assoc ty bounds + // 6. own implicit trait pred + // + // The purpose of this is to index the slice of the followings, without making extra `Vec`s or + // iterators: + // - explicit self only predicates, of own or own + self + // - explicit predicates, of own or own + self + let predicates = parent_implicit_trait_predicate + .iter() + .chain(parent_assoc_ty_bounds.iter()) + .chain(parent_predicates.iter()) + .chain(own_predicates.iter()) + .chain(own_assoc_ty_bounds.iter()) + .chain(own_implicit_trait_predicate.iter()) + .copied() + .collect::>(); + let parent_is_trait = parent_implicit_trait_predicate.is_some(); + let is_trait = own_implicit_trait_predicate.is_some(); + let parent_explicit_self_predicates_start = + parent_is_trait as u32 + parent_assoc_ty_bounds.len() as u32; + let own_predicates_start = + parent_explicit_self_predicates_start + parent_predicates.len() as u32; + let own_assoc_ty_bounds_start = own_predicates_start + own_predicates.len() as u32; + let predicates = GenericPredicates { + parent_explicit_self_predicates_start, own_predicates_start, + own_assoc_ty_bounds_start, is_trait, parent_is_trait, predicates: StoredEarlyBinder::bind(Clauses::new_from_slice(&predicates).store()), }; return (predicates, diagnostics); - fn add_implicit_trait_predicate<'db>( + fn implicit_trait_predicate<'db>( interner: DbInterner<'db>, def: GenericDefId, predicate_filter: PredicateFilter, - predicates: &mut Vec>, - set_is_trait: &mut bool, - ) { + ) -> Option> { // For traits, add `Self: Trait` predicate. This is // not part of the predicates that a user writes, but it // is something that one must prove in order to invoke a @@ -2029,8 +2203,9 @@ where if let GenericDefId::TraitId(def_id) = def && predicate_filter == PredicateFilter::All { - *set_is_trait = true; - predicates.push(TraitRef::identity(interner, def_id.into()).upcast(interner)); + Some(TraitRef::identity(interner, def_id.into()).upcast(interner)) + } else { + None } } } @@ -2327,7 +2502,7 @@ pub(crate) fn associated_ty_item_bounds<'db>( let mut bounds = Vec::new(); for bound in &type_alias_data.bounds { - ctx.lower_type_bound(bound, self_ty, false).for_each(|pred| { + ctx.lower_type_bound(bound, self_ty, false).for_each(|(pred, _)| { if let Some(bound) = pred .kind() .map_bound(|c| match c { diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs b/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs index a79f547c2a44..b77aeab62d15 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs @@ -32,7 +32,8 @@ use crate::{ db::HirDatabase, generics::{Generics, generics}, lower::{ - LifetimeElisionKind, PathDiagnosticCallbackData, named_associated_type_shorthand_candidates, + GenericPredicateSource, LifetimeElisionKind, PathDiagnosticCallbackData, + named_associated_type_shorthand_candidates, }, next_solver::{ Binder, Clause, Const, DbInterner, ErrorGuaranteed, GenericArg, GenericArgs, Predicate, @@ -853,7 +854,8 @@ impl<'a, 'b, 'db> PathLoweringContext<'a, 'b, 'db> { pub(super) fn assoc_type_bindings_from_type_bound<'c>( mut self, trait_ref: TraitRef<'db>, - ) -> Option> + use<'a, 'b, 'c, 'db>> { + ) -> Option, GenericPredicateSource)> + use<'a, 'b, 'c, 'db>> + { let interner = self.ctx.interner; self.current_or_prev_segment.args_and_bindings.map(|args_and_bindings| { args_and_bindings.bindings.iter().enumerate().flat_map(move |(binding_idx, binding)| { @@ -921,21 +923,29 @@ impl<'a, 'b, 'db> PathLoweringContext<'a, 'b, 'db> { ), )), )); - predicates.push(pred); + predicates.push((pred, GenericPredicateSource::SelfOnly)); } } }) } for bound in binding.bounds.iter() { - predicates.extend(self.ctx.lower_type_bound( - bound, - Ty::new_alias( - self.ctx.interner, - AliasTyKind::Projection, - AliasTy::new_from_args(self.ctx.interner, associated_ty.into(), args), - ), - false, - )); + predicates.extend( + self.ctx + .lower_type_bound( + bound, + Ty::new_alias( + self.ctx.interner, + AliasTyKind::Projection, + AliasTy::new_from_args( + self.ctx.interner, + associated_ty.into(), + args, + ), + ), + false, + ) + .map(|(pred, _)| (pred, GenericPredicateSource::AssocTyBound)), + ); } predicates }) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/interner.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/interner.rs index 2a3df1d32a30..e17bdac68cdd 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/interner.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/interner.rs @@ -41,7 +41,8 @@ use crate::{ AdtIdWrapper, AnyImplId, BoundConst, CallableIdWrapper, CanonicalVarKind, ClosureIdWrapper, CoroutineIdWrapper, Ctor, FnSig, FxIndexMap, GeneralConstIdWrapper, OpaqueTypeKey, RegionAssumptions, SimplifiedType, SolverContext, SolverDefIds, TraitIdWrapper, - TypeAliasIdWrapper, UnevaluatedConst, util::explicit_item_bounds, + TypeAliasIdWrapper, UnevaluatedConst, + util::{explicit_item_bounds, explicit_item_self_bounds}, }, }; @@ -1421,7 +1422,7 @@ impl<'db> Interner for DbInterner<'db> { self, def_id: Self::DefId, ) -> EarlyBinder> { - explicit_item_bounds(self, def_id) + explicit_item_self_bounds(self, def_id) .map_bound(|bounds| elaborate(self, bounds).filter_only_self()) } @@ -1500,7 +1501,7 @@ impl<'db> Interner for DbInterner<'db> { } } - predicates_of(self.db, def_id).explicit_predicates().map_bound(|predicates| { + predicates_of(self.db, def_id).explicit_implied_predicates().map_bound(|predicates| { predicates .iter() .copied() diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/util.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/util.rs index 34ecfed08f29..9a1b476976e3 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/util.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/util.rs @@ -455,6 +455,21 @@ pub fn explicit_item_bounds<'db>( clauses.map_bound(|clauses| clauses.iter().copied()) } +pub fn explicit_item_self_bounds<'db>( + interner: DbInterner<'db>, + def_id: SolverDefId, +) -> EarlyBinder<'db, impl DoubleEndedIterator> + ExactSizeIterator> { + let db = interner.db(); + let clauses = match def_id { + SolverDefId::TypeAliasId(type_alias) => { + crate::lower::type_alias_self_bounds(db, type_alias) + } + SolverDefId::InternedOpaqueTyId(id) => id.self_predicates(db), + _ => panic!("Unexpected GenericDefId"), + }; + clauses.map_bound(|clauses| clauses.iter().copied()) +} + pub struct ContainsTypeErrors; impl<'db> TypeVisitor> for ContainsTypeErrors { diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression/new_solver.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression/new_solver.rs index a4554673cdd5..be6ab23ad761 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression/new_solver.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression/new_solver.rs @@ -750,3 +750,63 @@ fn main() { "#]], ); } + +#[test] +fn regression_19339() { + check_infer( + r#" +trait Bar { + type Baz; + + fn baz(&self) -> Self::Baz; +} + +trait Foo { + type Bar; + + fn bar(&self) -> Self::Bar; +} + +trait FooFactory { + type Output: Foo>; + + fn foo(&self) -> Self::Output; + + fn foo_rpit(&self) -> impl Foo>; +} + +fn test1(foo: impl Foo>) { + let baz = foo.bar().baz(); +} + +fn test2(factory: T) { + let baz = factory.foo().bar().baz(); + let baz = factory.foo_rpit().bar().baz(); +} +"#, + expect![[r#" + 39..43 'self': &'? Self + 101..105 'self': &'? Self + 198..202 'self': &'? Self + 239..243 'self': &'? Self + 290..293 'foo': impl Foo + ?Sized + 325..359 '{ ...z(); }': () + 335..338 'baz': u8 + 341..344 'foo': impl Foo + ?Sized + 341..350 'foo.bar()': impl Bar + 341..356 'foo.bar().baz()': u8 + 385..392 'factory': T + 397..487 '{ ...z(); }': () + 407..410 'baz': u8 + 413..420 'factory': T + 413..426 'factory.foo()': ::Output + 413..432 'factor....bar()': <::Output as Foo>::Bar + 413..438 'factor....baz()': u8 + 448..451 'baz': u8 + 454..461 'factory': T + 454..472 'factor...rpit()': impl Foo + Bar + ?Sized + 454..478 'factor....bar()': + ?Sized as Foo>::Bar + 454..484 'factor....baz()': u8 + "#]], + ); +} From 6750919d47f8150bf53381981150b8af8571d575 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 7 Jan 2026 19:36:10 +0100 Subject: [PATCH 0100/1843] add `pmaddwd` shim --- src/tools/miri/src/shims/x86/avx2.rs | 30 +------- src/tools/miri/src/shims/x86/avx512.rs | 11 ++- src/tools/miri/src/shims/x86/mod.rs | 46 ++++++++++++ src/tools/miri/src/shims/x86/sse2.rs | 30 +------- .../pass/shims/x86/intrinsics-x86-avx512.rs | 71 +++++++++++++++++++ 5 files changed, 131 insertions(+), 57 deletions(-) diff --git a/src/tools/miri/src/shims/x86/avx2.rs b/src/tools/miri/src/shims/x86/avx2.rs index 97b9f649c158..7d8e52db73d6 100644 --- a/src/tools/miri/src/shims/x86/avx2.rs +++ b/src/tools/miri/src/shims/x86/avx2.rs @@ -6,7 +6,7 @@ use rustc_target::callconv::FnAbi; use super::{ ShiftOp, horizontal_bin_op, mpsadbw, packssdw, packsswb, packusdw, packuswb, permute, pmaddbw, - pmulhrsw, psadbw, pshufb, psign, shift_simd_by_scalar, + pmaddwd, pmulhrsw, psadbw, pshufb, psign, shift_simd_by_scalar, }; use crate::*; @@ -232,33 +232,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?; - let (left, left_len) = this.project_to_simd(left)?; - let (right, right_len) = this.project_to_simd(right)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(left_len, right_len); - assert_eq!(dest_len.strict_mul(2), left_len); - - for i in 0..dest_len { - let j1 = i.strict_mul(2); - let left1 = this.read_scalar(&this.project_index(&left, j1)?)?.to_i16()?; - let right1 = this.read_scalar(&this.project_index(&right, j1)?)?.to_i16()?; - - let j2 = j1.strict_add(1); - let left2 = this.read_scalar(&this.project_index(&left, j2)?)?.to_i16()?; - let right2 = this.read_scalar(&this.project_index(&right, j2)?)?.to_i16()?; - - let dest = this.project_index(&dest, i)?; - - // Multiplications are i16*i16->i32, which will not overflow. - let mul1 = i32::from(left1).strict_mul(right1.into()); - let mul2 = i32::from(left2).strict_mul(right2.into()); - // However, this addition can overflow in the most extreme case - // (-0x8000)*(-0x8000)+(-0x8000)*(-0x8000) = 0x80000000 - let res = mul1.wrapping_add(mul2); - - this.write_scalar(Scalar::from_i32(res), &dest)?; - } + pmaddwd(this, left, right, dest)?; } _ => return interp_ok(EmulateItemResult::NotSupported), } diff --git a/src/tools/miri/src/shims/x86/avx512.rs b/src/tools/miri/src/shims/x86/avx512.rs index 0466ba1bd6c0..9231fc446919 100644 --- a/src/tools/miri/src/shims/x86/avx512.rs +++ b/src/tools/miri/src/shims/x86/avx512.rs @@ -3,7 +3,7 @@ use rustc_middle::ty::Ty; use rustc_span::Symbol; use rustc_target::callconv::FnAbi; -use super::{permute, pmaddbw, psadbw, pshufb}; +use super::{permute, pmaddbw, pmaddwd, psadbw, pshufb}; use crate::*; impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {} @@ -88,6 +88,15 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { psadbw(this, left, right, dest)? } + // Used to implement the _mm512_madd_epi16 function. + "pmaddw.d.512" => { + this.expect_target_feature_for_intrinsic(link_name, "avx512bw")?; + + let [left, right] = + this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?; + + pmaddwd(this, left, right, dest)?; + } // Used to implement the _mm512_maddubs_epi16 function. "pmaddubs.w.512" => { let [left, right] = diff --git a/src/tools/miri/src/shims/x86/mod.rs b/src/tools/miri/src/shims/x86/mod.rs index a5164cc87ab4..dc0d8d48ac9b 100644 --- a/src/tools/miri/src/shims/x86/mod.rs +++ b/src/tools/miri/src/shims/x86/mod.rs @@ -964,6 +964,52 @@ fn psadbw<'tcx>( interp_ok(()) } +/// Multiply packed signed 16-bit integers in `left` and `right`, producing intermediate signed 32-bit integers. +/// Horizontally add adjacent pairs of intermediate 32-bit integers, and pack the results in `dest`. +/// +/// +/// +/// +fn pmaddwd<'tcx>( + ecx: &mut crate::MiriInterpCx<'tcx>, + left: &OpTy<'tcx>, + right: &OpTy<'tcx>, + dest: &MPlaceTy<'tcx>, +) -> InterpResult<'tcx, ()> { + let (left, left_len) = ecx.project_to_simd(left)?; + let (right, right_len) = ecx.project_to_simd(right)?; + let (dest, dest_len) = ecx.project_to_simd(dest)?; + + // fn pmaddwd(a: i16x8, b: i16x8) -> i32x4; + // fn pmaddwd(a: i16x16, b: i16x16) -> i32x8; + // fn vpmaddwd(a: i16x32, b: i16x32) -> i32x16; + assert_eq!(left_len, right_len); + assert_eq!(dest_len.strict_mul(2), left_len); + + for i in 0..dest_len { + let j1 = i.strict_mul(2); + let left1 = ecx.read_scalar(&ecx.project_index(&left, j1)?)?.to_i16()?; + let right1 = ecx.read_scalar(&ecx.project_index(&right, j1)?)?.to_i16()?; + + let j2 = j1.strict_add(1); + let left2 = ecx.read_scalar(&ecx.project_index(&left, j2)?)?.to_i16()?; + let right2 = ecx.read_scalar(&ecx.project_index(&right, j2)?)?.to_i16()?; + + let dest = ecx.project_index(&dest, i)?; + + // Multiplications are i16*i16->i32, which will not overflow. + let mul1 = i32::from(left1).strict_mul(right1.into()); + let mul2 = i32::from(left2).strict_mul(right2.into()); + // However, this addition can overflow in the most extreme case + // (-0x8000)*(-0x8000)+(-0x8000)*(-0x8000) = 0x80000000 + let res = mul1.wrapping_add(mul2); + + ecx.write_scalar(Scalar::from_i32(res), &dest)?; + } + + interp_ok(()) +} + /// Multiplies packed 8-bit unsigned integers from `left` and packed /// signed 8-bit integers from `right` into 16-bit signed integers. Then, /// the saturating sum of the products with indices `2*i` and `2*i+1` diff --git a/src/tools/miri/src/shims/x86/sse2.rs b/src/tools/miri/src/shims/x86/sse2.rs index 3fbab9ba789e..f712814a5eda 100644 --- a/src/tools/miri/src/shims/x86/sse2.rs +++ b/src/tools/miri/src/shims/x86/sse2.rs @@ -6,7 +6,7 @@ use rustc_target::callconv::FnAbi; use super::{ FloatBinOp, ShiftOp, bin_op_simd_float_all, bin_op_simd_float_first, convert_float_to_int, - packssdw, packsswb, packuswb, psadbw, shift_simd_by_scalar, + packssdw, packsswb, packuswb, pmaddwd, psadbw, shift_simd_by_scalar, }; use crate::*; @@ -286,33 +286,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?; - let (left, left_len) = this.project_to_simd(left)?; - let (right, right_len) = this.project_to_simd(right)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(left_len, right_len); - assert_eq!(dest_len.strict_mul(2), left_len); - - for i in 0..dest_len { - let j1 = i.strict_mul(2); - let left1 = this.read_scalar(&this.project_index(&left, j1)?)?.to_i16()?; - let right1 = this.read_scalar(&this.project_index(&right, j1)?)?.to_i16()?; - - let j2 = j1.strict_add(1); - let left2 = this.read_scalar(&this.project_index(&left, j2)?)?.to_i16()?; - let right2 = this.read_scalar(&this.project_index(&right, j2)?)?.to_i16()?; - - let dest = this.project_index(&dest, i)?; - - // Multiplications are i16*i16->i32, which will not overflow. - let mul1 = i32::from(left1).strict_mul(right1.into()); - let mul2 = i32::from(left2).strict_mul(right2.into()); - // However, this addition can overflow in the most extreme case - // (-0x8000)*(-0x8000)+(-0x8000)*(-0x8000) = 0x80000000 - let res = mul1.wrapping_add(mul2); - - this.write_scalar(Scalar::from_i32(res), &dest)?; - } + pmaddwd(this, left, right, dest)?; } _ => return interp_ok(EmulateItemResult::NotSupported), } diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs index 42acb6c3fb37..7cc554ef5a3c 100644 --- a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs @@ -100,6 +100,77 @@ unsafe fn test_avx512() { } test_mm512_maddubs_epi16(); + #[target_feature(enable = "avx512bw")] + unsafe fn test_mm512_madd_epi16() { + // Input pairs + // + // - `i16::MIN * i16::MIN + i16::MIN * i16::MIN`: the 32-bit addition overflows + // - `i16::MAX * i16::MAX + i16::MAX * i16::MAX`: check that widening happens before + // arithmetic + // - `i16::MIN * i16::MAX + i16::MAX * i16::MIN`: check that large negative values are + // handled correctly + // - `3 * 1 + 4 * 2`: A sanity check, the result should be 14. + + #[rustfmt::skip] + let a = _mm512_set_epi16( + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MIN, i16::MAX, + 3, 1, + + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MIN, i16::MAX, + 3, 1, + + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MIN, i16::MAX, + 3, 1, + + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MIN, i16::MAX, + 3, 1, + ); + + #[rustfmt::skip] + let b = _mm512_set_epi16( + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MAX, i16::MIN, + 4, 2, + + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MAX, i16::MIN, + 4, 2, + + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MAX, i16::MIN, + 4, 2, + + i16::MIN, i16::MIN, + i16::MAX, i16::MAX, + i16::MAX, i16::MIN, + 4, 2, + ); + + let r = _mm512_madd_epi16(a, b); + + #[rustfmt::skip] + let e = _mm512_set_epi32( + i32::MIN, 2_147_352_578, -2_147_418_112, 14, + i32::MIN, 2_147_352_578, -2_147_418_112, 14, + i32::MIN, 2_147_352_578, -2_147_418_112, 14, + i32::MIN, 2_147_352_578, -2_147_418_112, 14, + ); + + assert_eq_m512i(r, e); + } + test_mm512_madd_epi16(); + #[target_feature(enable = "avx512f")] unsafe fn test_mm512_permutexvar_epi32() { let a = _mm512_set_epi32( From 93f2e80f4a885dd4eacce00e7a52aeeb41b7ce70 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Wed, 7 Jan 2026 11:25:17 -0800 Subject: [PATCH 0101/1843] Add -Z large-data-threshold This flag allows specifying the threshold size for placing static data in large data sections when using the medium code model on x86-64. When using -Ccode-model=medium, data smaller than this threshold uses RIP-relative addressing (32-bit offsets), while larger data uses absolute 64-bit addressing. This allows the compiler to generate more efficient code for smaller data while still supporting data larger than 2GB. This mirrors the -mlarge-data-threshold flag available in GCC and Clang. The default threshold is 65536 bytes (64KB) if not specified, matching LLVM's default behavior. --- .../src/back/owned_target_machine.rs | 2 + compiler/rustc_codegen_llvm/src/back/write.rs | 3 + compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 7 +- compiler/rustc_session/src/options.rs | 3 + .../compiler-flags/large-data-threshold.md | 27 +++++++ tests/assembly-llvm/large_data_threshold.rs | 73 +++++++++++++++++++ 7 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/doc/unstable-book/src/compiler-flags/large-data-threshold.md create mode 100644 tests/assembly-llvm/large_data_threshold.rs diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index f88932d43d2a..65cf4cad24bd 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -39,6 +39,7 @@ impl OwnedTargetMachine { debug_info_compression: llvm::CompressionKind, use_emulated_tls: bool, use_wasm_eh: bool, + large_data_threshold: u64, ) -> Result> { // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { @@ -65,6 +66,7 @@ impl OwnedTargetMachine { debug_info_compression, use_emulated_tls, use_wasm_eh, + large_data_threshold, ) }; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index bcadb6f0de92..6c29d0470766 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -272,6 +272,8 @@ pub(crate) fn target_machine_factory( let use_wasm_eh = wants_wasm_eh(sess); + let large_data_threshold = sess.opts.unstable_opts.large_data_threshold.unwrap_or(0); + let prof = SelfProfilerRef::clone(&sess.prof); Arc::new(move |config: TargetMachineFactoryConfig| { // Self-profile timer for invoking a factory to create a target machine. @@ -313,6 +315,7 @@ pub(crate) fn target_machine_factory( debuginfo_compression, use_emulated_tls, use_wasm_eh, + large_data_threshold, ) }) } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index a90013801c8c..70fc62d08d5c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2338,6 +2338,7 @@ unsafe extern "C" { DebugInfoCompression: CompressionKind, UseEmulatedTls: bool, UseWasmEH: bool, + LargeDataThreshold: u64, ) -> *mut TargetMachine; pub(crate) fn LLVMRustAddLibraryInfo<'a>( diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 733f5fd0df0a..97f95ac01e86 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -305,7 +305,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, const char *SplitDwarfFile, const char *OutputObjFile, LLVMRustCompressionKind DebugInfoCompression, bool UseEmulatedTls, - bool UseWasmEH) { + bool UseWasmEH, uint64_t LargeDataThreshold) { auto OptLevel = fromRust(RustOptLevel); auto RM = fromRust(RustReloc); @@ -381,6 +381,11 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( TargetMachine *TM = TheTarget->createTargetMachine( Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); #endif + + if (LargeDataThreshold != 0) { + TM->setLargeDataThreshold(LargeDataThreshold); + } + return wrap(TM); } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 21fa3321a30a..0bdca21ff644 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2444,6 +2444,9 @@ options! { `=skip-entry` `=skip-exit` Multiple options can be combined with commas."), + large_data_threshold: Option = (None, parse_opt_number, [TRACKED], + "set the threshold for objects to be stored in a \"large data\" section \ + (only effective with -Ccode-model=medium, default: 65536)"), layout_seed: Option = (None, parse_opt_number, [TRACKED], "seed layout randomization"), link_directives: bool = (true, parse_bool, [TRACKED], diff --git a/src/doc/unstable-book/src/compiler-flags/large-data-threshold.md b/src/doc/unstable-book/src/compiler-flags/large-data-threshold.md new file mode 100644 index 000000000000..27c86079a6c8 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/large-data-threshold.md @@ -0,0 +1,27 @@ +# `large-data-threshold` + +----------------------- + +This flag controls the threshold for static data to be placed in large data +sections when using the `medium` code model on x86-64. + +When using `-Ccode-model=medium`, static data smaller than this threshold will +use RIP-relative addressing (32-bit offsets), while larger data will use +absolute 64-bit addressing. This allows the compiler to generate more efficient +code for smaller data while still supporting data larger than 2GB. + +The default threshold is 65536 bytes (64KB) if not specified. + +## Example + +```sh +rustc -Ccode-model=medium -Zlarge-data-threshold=1024 main.rs +``` + +This sets the threshold to 1KB, meaning only data smaller than 1024 bytes will +use RIP-relative addressing. + +## Platform Support + +This flag is only effective on x86-64 targets when using `-Ccode-model=medium`. +On other architectures or with other code models, this flag has no effect. diff --git a/tests/assembly-llvm/large_data_threshold.rs b/tests/assembly-llvm/large_data_threshold.rs new file mode 100644 index 000000000000..f3b37eb7f83d --- /dev/null +++ b/tests/assembly-llvm/large_data_threshold.rs @@ -0,0 +1,73 @@ +// Test for -Z large_data_threshold=... +// This test verifies that with the medium code model, data above the threshold +// is placed in large data sections (.ldata, .lbss, .lrodata). +//@ assembly-output: emit-asm +//@ compile-flags: -Ccode-model=medium -Zlarge-data-threshold=4 +//@ compile-flags: --target=x86_64-unknown-linux-gnu +//@ needs-llvm-components: x86 + +#![feature(no_core, lang_items)] +#![no_std] +#![no_core] +#![crate_type = "lib"] + +#[lang = "pointee_sized"] +pub trait PointeeSized {} + +#[lang = "meta_sized"] +pub trait MetaSized: PointeeSized {} + +#[lang = "sized"] +pub trait Sized: MetaSized {} + +#[lang = "drop_in_place"] +fn drop_in_place(_: *mut T) {} + +#[used] +#[no_mangle] +// U is below the threshold, should be in .data +static mut U: u16 = 123; + +#[used] +#[no_mangle] +// V is below the threshold, should be in .bss +static mut V: u16 = 0; + +#[used] +#[no_mangle] +// W is at the threshold, should be in .data +static mut W: u32 = 123; + +#[used] +#[no_mangle] +// X is at the threshold, should be in .bss +static mut X: u32 = 0; + +#[used] +#[no_mangle] +// Y is over the threshold, should be in .ldata +static mut Y: u64 = 123; + +#[used] +#[no_mangle] +// Z is over the threshold, should be in .lbss +static mut Z: u64 = 0; + +// CHECK: .section .data.U, +// CHECK-NOT: .section +// CHECK: U: +// CHECK: .section .bss.V, +// CHECK-NOT: .section +// CHECK: V: +// CHECK: .section .data.W, +// CHECK-NOT: .section +// CHECK: W: +// CHECK: .section .bss.X, +// CHECK-NOT: .section +// CHECK: X: +// CHECK: .section .ldata.Y, +// CHECK-NOT: .section +// CHECK: Y: +// CHECK: .section .lbss.Z, +// CHECK-NOT: .section +// CHECK: Z: From 0db25dbd9dfef774e4dcc2a00e37674b5bac87de Mon Sep 17 00:00:00 2001 From: yanglsh Date: Wed, 24 Sep 2025 08:53:48 +0800 Subject: [PATCH 0102/1843] fix: `map_unwrap_or` suggests wrongly for empty slice --- clippy_lints/src/derivable_impls.rs | 2 +- clippy_lints/src/methods/map_unwrap_or.rs | 231 ++++++++++-------- .../src/methods/map_unwrap_or_else.rs | 81 +++--- tests/ui/map_unwrap_or.rs | 8 + tests/ui/map_unwrap_or.stderr | 8 +- tests/ui/map_unwrap_or_fixable.fixed | 6 +- tests/ui/map_unwrap_or_fixable.rs | 6 +- tests/ui/map_unwrap_or_fixable.stderr | 46 ++-- 8 files changed, 211 insertions(+), 177 deletions(-) diff --git a/clippy_lints/src/derivable_impls.rs b/clippy_lints/src/derivable_impls.rs index 6b8a6aec92fa..992ed320ce68 100644 --- a/clippy_lints/src/derivable_impls.rs +++ b/clippy_lints/src/derivable_impls.rs @@ -105,7 +105,7 @@ fn check_struct<'tcx>( if let TyKind::Path(QPath::Resolved(_, p)) = self_ty.kind && let Some(PathSegment { args, .. }) = p.segments.last() { - let args = args.map(|a| a.args).unwrap_or(&[]); + let args = args.map(|a| a.args).unwrap_or_default(); // ty_args contains the generic parameters of the type declaration, while args contains the // arguments used at instantiation time. If both len are not equal, it means that some diff --git a/clippy_lints/src/methods/map_unwrap_or.rs b/clippy_lints/src/methods/map_unwrap_or.rs index f2e3cc5c0091..ac2f99180486 100644 --- a/clippy_lints/src/methods/map_unwrap_or.rs +++ b/clippy_lints/src/methods/map_unwrap_or.rs @@ -1,13 +1,13 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::msrvs::{self, Msrv}; -use clippy_utils::res::MaybeDef; +use clippy_utils::res::{MaybeDef as _, MaybeResPath as _}; use clippy_utils::source::snippet_with_applicability; use clippy_utils::ty::is_copy; use rustc_data_structures::fx::FxHashSet; use rustc_errors::Applicability; use rustc_hir::def::Res; -use rustc_hir::intravisit::{Visitor, walk_path}; -use rustc_hir::{ExprKind, HirId, Node, PatKind, Path, QPath}; +use rustc_hir::intravisit::{Visitor, walk_expr, walk_path}; +use rustc_hir::{ExprKind, HirId, LangItem, Node, PatKind, Path, QPath}; use rustc_lint::LateContext; use rustc_middle::hir::nested_filter; use rustc_span::{Span, sym}; @@ -28,116 +28,131 @@ pub(super) fn check<'tcx>( msrv: Msrv, ) { let recv_ty = cx.typeck_results().expr_ty(recv).peel_refs(); - let is_option = recv_ty.is_diag_item(cx, sym::Option); - let is_result = recv_ty.is_diag_item(cx, sym::Result); + let recv_ty_kind = match recv_ty.opt_diag_name(cx) { + Some(sym::Option) => sym::Option, + Some(sym::Result) if msrv.meets(cx, msrvs::RESULT_MAP_OR) => sym::Result, + _ => return, + }; - if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR) { + let unwrap_arg_ty = cx.typeck_results().expr_ty(unwrap_arg); + if !is_copy(cx, unwrap_arg_ty) { + // Replacing `.map().unwrap_or()` with `.map_or(, )` can sometimes lead to + // borrowck errors, see #10579 for one such instance. + // In particular, if `a` causes a move and `f` references that moved binding, then we cannot lint: + // ``` + // let x = vec![1, 2]; + // x.get(0..1).map(|s| s.to_vec()).unwrap_or(x); + // ``` + // This compiles, but changing it to `map_or` will produce a compile error: + // ``` + // let x = vec![1, 2]; + // x.get(0..1).map_or(x, |s| s.to_vec()) + // ^ moving `x` here + // ^^^^^^^^^^^ while it is borrowed here (and later used in the closure) + // ``` + // So, we have to check that `a` is not referenced anywhere (even outside of the `.map` closure!) + // before the call to `unwrap_or`. + + let mut unwrap_visitor = UnwrapVisitor { + cx, + identifiers: FxHashSet::default(), + }; + unwrap_visitor.visit_expr(unwrap_arg); + + let mut reference_visitor = ReferenceVisitor { + cx, + identifiers: unwrap_visitor.identifiers, + unwrap_or_span: unwrap_arg.span, + }; + + let body = cx.tcx.hir_body_owned_by(cx.tcx.hir_enclosing_body_owner(expr.hir_id)); + + // Visit the body, and return if we've found a reference + if reference_visitor.visit_body(body).is_break() { + return; + } + } + + if !unwrap_arg.span.eq_ctxt(map_span) { return; } - // lint if the caller of `map()` is an `Option` - if is_option || is_result { - if !is_copy(cx, cx.typeck_results().expr_ty(unwrap_arg)) { - // Replacing `.map().unwrap_or()` with `.map_or(, )` can sometimes lead to - // borrowck errors, see #10579 for one such instance. - // In particular, if `a` causes a move and `f` references that moved binding, then we cannot lint: - // ``` - // let x = vec![1, 2]; - // x.get(0..1).map(|s| s.to_vec()).unwrap_or(x); - // ``` - // This compiles, but changing it to `map_or` will produce a compile error: - // ``` - // let x = vec![1, 2]; - // x.get(0..1).map_or(x, |s| s.to_vec()) - // ^ moving `x` here - // ^^^^^^^^^^^ while it is borrowed here (and later used in the closure) - // ``` - // So, we have to check that `a` is not referenced anywhere (even outside of the `.map` closure!) - // before the call to `unwrap_or`. + let mut applicability = Applicability::MachineApplicable; + // get snippet for unwrap_or() + let unwrap_snippet = snippet_with_applicability(cx, unwrap_arg.span, "..", &mut applicability); + // lint message - let mut unwrap_visitor = UnwrapVisitor { - cx, - identifiers: FxHashSet::default(), - }; - unwrap_visitor.visit_expr(unwrap_arg); - - let mut reference_visitor = ReferenceVisitor { - cx, - identifiers: unwrap_visitor.identifiers, - unwrap_or_span: unwrap_arg.span, - }; - - let body = cx.tcx.hir_body_owned_by(cx.tcx.hir_enclosing_body_owner(expr.hir_id)); - - // Visit the body, and return if we've found a reference - if reference_visitor.visit_body(body).is_break() { - return; - } - } - - if !unwrap_arg.span.eq_ctxt(map_span) { - return; - } - - // is_some_and is stabilised && `unwrap_or` argument is false; suggest `is_some_and` instead - let suggest_is_some_and = matches!(&unwrap_arg.kind, ExprKind::Lit(lit) - if matches!(lit.node, rustc_ast::LitKind::Bool(false))) - && msrv.meets(cx, msrvs::OPTION_RESULT_IS_VARIANT_AND); - - let mut applicability = Applicability::MachineApplicable; - // get snippet for unwrap_or() - let unwrap_snippet = snippet_with_applicability(cx, unwrap_arg.span, "..", &mut applicability); - // lint message - // comparing the snippet from source to raw text ("None") below is safe - // because we already have checked the type. - let unwrap_snippet_none = is_option && unwrap_snippet == "None"; - let arg = if unwrap_snippet_none { - "None" - } else if suggest_is_some_and { - "false" - } else { - "" - }; - let suggest = if unwrap_snippet_none { - "and_then()" - } else if suggest_is_some_and { - if is_result { - "is_ok_and()" - } else { - "is_some_and()" - } - } else { - "map_or(, )" - }; - let msg = format!( - "called `map().unwrap_or({arg})` on an `{}` value", - if is_option { "Option" } else { "Result" } - ); - - span_lint_and_then(cx, MAP_UNWRAP_OR, expr.span, msg, |diag| { - let map_arg_span = map_arg.span; - - let mut suggestion = vec![ - ( - map_span, - String::from(if unwrap_snippet_none { - "and_then" - } else if suggest_is_some_and { - if is_result { "is_ok_and" } else { "is_some_and" } - } else { - "map_or" - }), - ), - (expr.span.with_lo(unwrap_recv.span.hi()), String::new()), - ]; - - if !unwrap_snippet_none && !suggest_is_some_and { - suggestion.push((map_arg_span.with_hi(map_arg_span.lo()), format!("{unwrap_snippet}, "))); - } - - diag.multipart_suggestion(format!("use `{suggest}` instead"), suggestion, applicability); - }); + let suggest_kind = if recv_ty_kind == sym::Option + && unwrap_arg + .basic_res() + .ctor_parent(cx) + .is_lang_item(cx, LangItem::OptionNone) + { + SuggestedKind::AndThen } + // is_some_and is stabilised && `unwrap_or` argument is false; suggest `is_some_and` instead + else if matches!(&unwrap_arg.kind, ExprKind::Lit(lit) + if matches!(lit.node, rustc_ast::LitKind::Bool(false))) + && msrv.meets(cx, msrvs::OPTION_RESULT_IS_VARIANT_AND) + { + SuggestedKind::IsVariantAnd + } else { + SuggestedKind::Other + }; + + let arg = match suggest_kind { + SuggestedKind::AndThen => "None", + SuggestedKind::IsVariantAnd => "false", + SuggestedKind::Other => "", + }; + + let suggest = match (suggest_kind, recv_ty_kind) { + (SuggestedKind::AndThen, _) => "and_then()", + (SuggestedKind::IsVariantAnd, sym::Result) => "is_ok_and()", + (SuggestedKind::IsVariantAnd, sym::Option) => "is_some_and()", + _ => "map_or(, )", + }; + + let msg = format!( + "called `map().unwrap_or({arg})` on {} `{recv_ty_kind}` value", + if recv_ty_kind == sym::Option { "an" } else { "a" } + ); + + span_lint_and_then(cx, MAP_UNWRAP_OR, expr.span, msg, |diag| { + let map_arg_span = map_arg.span; + + let mut suggestion = vec![ + ( + map_span, + String::from(match (suggest_kind, recv_ty_kind) { + (SuggestedKind::AndThen, _) => "and_then", + (SuggestedKind::IsVariantAnd, sym::Result) => "is_ok_and", + (SuggestedKind::IsVariantAnd, sym::Option) => "is_some_and", + (SuggestedKind::Other, _) + if unwrap_arg_ty.peel_refs().is_array() + && cx.typeck_results().expr_ty_adjusted(unwrap_arg).peel_refs().is_slice() => + { + return; + }, + _ => "map_or", + }), + ), + (expr.span.with_lo(unwrap_recv.span.hi()), String::new()), + ]; + + if matches!(suggest_kind, SuggestedKind::Other) { + suggestion.push((map_arg_span.with_hi(map_arg_span.lo()), format!("{unwrap_snippet}, "))); + } + + diag.multipart_suggestion(format!("use `{suggest}` instead"), suggestion, applicability); + }); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum SuggestedKind { + AndThen, + IsVariantAnd, + Other, } struct UnwrapVisitor<'a, 'tcx> { @@ -186,7 +201,7 @@ impl<'tcx> Visitor<'tcx> for ReferenceVisitor<'_, 'tcx> { { return ControlFlow::Break(()); } - rustc_hir::intravisit::walk_expr(self, expr) + walk_expr(self, expr) } fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt { diff --git a/clippy_lints/src/methods/map_unwrap_or_else.rs b/clippy_lints/src/methods/map_unwrap_or_else.rs index a2f157c0cb8a..8bb532b21635 100644 --- a/clippy_lints/src/methods/map_unwrap_or_else.rs +++ b/clippy_lints/src/methods/map_unwrap_or_else.rs @@ -1,18 +1,18 @@ use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::msrvs::{self, Msrv}; -use clippy_utils::res::MaybeDef; +use clippy_utils::res::MaybeDef as _; use clippy_utils::source::snippet; +use clippy_utils::sym; use clippy_utils::usage::mutated_variables; use rustc_errors::Applicability; use rustc_hir as hir; use rustc_lint::LateContext; -use rustc_span::symbol::sym; use super::MAP_UNWRAP_OR; /// lint use of `map().unwrap_or_else()` for `Option`s and `Result`s /// -/// Returns true if the lint was emitted +/// Is part of the `map_unwrap_or` lint, split into separate files for readability. pub(super) fn check<'tcx>( cx: &LateContext<'tcx>, expr: &'tcx hir::Expr<'_>, @@ -22,49 +22,46 @@ pub(super) fn check<'tcx>( msrv: Msrv, ) -> bool { let recv_ty = cx.typeck_results().expr_ty(recv).peel_refs(); - let is_option = recv_ty.is_diag_item(cx, sym::Option); - let is_result = recv_ty.is_diag_item(cx, sym::Result); + let recv_ty_kind = match recv_ty.opt_diag_name(cx) { + Some(sym::Option) => sym::Option, + Some(sym::Result) if msrv.meets(cx, msrvs::RESULT_MAP_OR_ELSE) => sym::Result, + _ => return false, + }; - if is_result && !msrv.meets(cx, msrvs::RESULT_MAP_OR_ELSE) { + // Don't make a suggestion that may fail to compile due to mutably borrowing + // the same variable twice. + let Some(map_mutated_vars) = mutated_variables(recv, cx) else { + return false; + }; + let Some(unwrap_mutated_vars) = mutated_variables(unwrap_arg, cx) else { + return false; + }; + if map_mutated_vars.intersection(&unwrap_mutated_vars).next().is_some() { return false; } - if is_option || is_result { - // Don't make a suggestion that may fail to compile due to mutably borrowing - // the same variable twice. - let map_mutated_vars = mutated_variables(recv, cx); - let unwrap_mutated_vars = mutated_variables(unwrap_arg, cx); - if let (Some(map_mutated_vars), Some(unwrap_mutated_vars)) = (map_mutated_vars, unwrap_mutated_vars) { - if map_mutated_vars.intersection(&unwrap_mutated_vars).next().is_some() { - return false; - } - } else { - return false; - } - - // lint message - let msg = if is_option { - "called `map().unwrap_or_else()` on an `Option` value" - } else { - "called `map().unwrap_or_else()` on a `Result` value" - }; - // get snippets for args to map() and unwrap_or_else() - let map_snippet = snippet(cx, map_arg.span, ".."); - let unwrap_snippet = snippet(cx, unwrap_arg.span, ".."); - // lint, with note if both map() and unwrap_or_else() have the same span - if map_arg.span.eq_ctxt(unwrap_arg.span) { - let var_snippet = snippet(cx, recv.span, ".."); - span_lint_and_sugg( - cx, - MAP_UNWRAP_OR, - expr.span, - msg, - "try", - format!("{var_snippet}.map_or_else({unwrap_snippet}, {map_snippet})"), - Applicability::MachineApplicable, - ); - return true; - } + // lint message + let msg = if recv_ty_kind == sym::Option { + "called `map().unwrap_or_else()` on an `Option` value" + } else { + "called `map().unwrap_or_else()` on a `Result` value" + }; + // get snippets for args to map() and unwrap_or_else() + let map_snippet = snippet(cx, map_arg.span, ".."); + let unwrap_snippet = snippet(cx, unwrap_arg.span, ".."); + // lint, with note if both map() and unwrap_or_else() have the same span + if map_arg.span.eq_ctxt(unwrap_arg.span) { + let var_snippet = snippet(cx, recv.span, ".."); + span_lint_and_sugg( + cx, + MAP_UNWRAP_OR, + expr.span, + msg, + "try", + format!("{var_snippet}.map_or_else({unwrap_snippet}, {map_snippet})"), + Applicability::MachineApplicable, + ); + return true; } false diff --git a/tests/ui/map_unwrap_or.rs b/tests/ui/map_unwrap_or.rs index fba81cb493cd..dccacd7df8af 100644 --- a/tests/ui/map_unwrap_or.rs +++ b/tests/ui/map_unwrap_or.rs @@ -156,3 +156,11 @@ mod issue_10579 { println!("{y:?}"); } } + +fn issue15752() { + struct Foo<'a>(&'a [u32]); + + let x = Some(Foo(&[1, 2, 3])); + x.map(|y| y.0).unwrap_or(&[]); + //~^ map_unwrap_or +} diff --git a/tests/ui/map_unwrap_or.stderr b/tests/ui/map_unwrap_or.stderr index df0207c420e6..bd9e0eeb0bda 100644 --- a/tests/ui/map_unwrap_or.stderr +++ b/tests/ui/map_unwrap_or.stderr @@ -232,5 +232,11 @@ LL - let _ = opt.map(|x| x > 5).unwrap_or(false); LL + let _ = opt.is_some_and(|x| x > 5); | -error: aborting due to 15 previous errors +error: called `map().unwrap_or()` on an `Option` value + --> tests/ui/map_unwrap_or.rs:164:5 + | +LL | x.map(|y| y.0).unwrap_or(&[]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 16 previous errors diff --git a/tests/ui/map_unwrap_or_fixable.fixed b/tests/ui/map_unwrap_or_fixable.fixed index 1789c7f4d487..dca2536132d7 100644 --- a/tests/ui/map_unwrap_or_fixable.fixed +++ b/tests/ui/map_unwrap_or_fixable.fixed @@ -1,7 +1,11 @@ //@aux-build:option_helpers.rs #![warn(clippy::map_unwrap_or)] -#![allow(clippy::unnecessary_lazy_evaluations)] +#![allow( + clippy::unnecessary_lazy_evaluations, + clippy::manual_is_variant_and, + clippy::unnecessary_map_or +)] #[macro_use] extern crate option_helpers; diff --git a/tests/ui/map_unwrap_or_fixable.rs b/tests/ui/map_unwrap_or_fixable.rs index 309067edce4d..c60cb082ae3c 100644 --- a/tests/ui/map_unwrap_or_fixable.rs +++ b/tests/ui/map_unwrap_or_fixable.rs @@ -1,7 +1,11 @@ //@aux-build:option_helpers.rs #![warn(clippy::map_unwrap_or)] -#![allow(clippy::unnecessary_lazy_evaluations)] +#![allow( + clippy::unnecessary_lazy_evaluations, + clippy::manual_is_variant_and, + clippy::unnecessary_map_or +)] #[macro_use] extern crate option_helpers; diff --git a/tests/ui/map_unwrap_or_fixable.stderr b/tests/ui/map_unwrap_or_fixable.stderr index 696f516ee055..33a865d6769c 100644 --- a/tests/ui/map_unwrap_or_fixable.stderr +++ b/tests/ui/map_unwrap_or_fixable.stderr @@ -1,5 +1,5 @@ error: called `map().unwrap_or_else()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:17:13 + --> tests/ui/map_unwrap_or_fixable.rs:21:13 | LL | let _ = opt.map(|x| x + 1) | _____________^ @@ -11,7 +11,7 @@ LL | | .unwrap_or_else(|| 0); = help: to override `-D warnings` add `#[allow(clippy::map_unwrap_or)]` error: called `map().unwrap_or_else()` on a `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:48:13 + --> tests/ui/map_unwrap_or_fixable.rs:52:13 | LL | let _ = res.map(|x| x + 1) | _____________^ @@ -20,7 +20,7 @@ LL | | .unwrap_or_else(|_e| 0); | |_______________________________^ help: try: `res.map_or_else(|_e| 0, |x| x + 1)` error: called `map().unwrap_or()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:65:20 + --> tests/ui/map_unwrap_or_fixable.rs:69:20 | LL | println!("{}", o.map(|y| y + 1).unwrap_or(3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -32,13 +32,13 @@ LL + println!("{}", o.map_or(3, |y| y + 1)); | error: called `map().unwrap_or_else()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:67:20 + --> tests/ui/map_unwrap_or_fixable.rs:71:20 | LL | println!("{}", o.map(|y| y + 1).unwrap_or_else(|| 3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `o.map_or_else(|| 3, |y| y + 1)` -error: called `map().unwrap_or()` on an `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:69:20 +error: called `map().unwrap_or()` on a `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:73:20 | LL | println!("{}", r.map(|y| y + 1).unwrap_or(3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -50,13 +50,13 @@ LL + println!("{}", r.map_or(3, |y| y + 1)); | error: called `map().unwrap_or_else()` on a `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:71:20 + --> tests/ui/map_unwrap_or_fixable.rs:75:20 | LL | println!("{}", r.map(|y| y + 1).unwrap_or_else(|()| 3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `r.map_or_else(|()| 3, |y| y + 1)` -error: called `map().unwrap_or(false)` on an `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:74:20 +error: called `map().unwrap_or(false)` on a `Result` value + --> tests/ui/map_unwrap_or_fixable.rs:78:20 | LL | println!("{}", r.map(|y| y == 1).unwrap_or(false)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -68,18 +68,6 @@ LL + println!("{}", r.is_ok_and(|y| y == 1)); | error: called `map().unwrap_or()` on an `Option` value - --> tests/ui/map_unwrap_or_fixable.rs:80:20 - | -LL | println!("{}", x.map(|y| y + 1).unwrap_or(3)); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | -help: use `map_or(, )` instead - | -LL - println!("{}", x.map(|y| y + 1).unwrap_or(3)); -LL + println!("{}", x.map_or(3, |y| y + 1)); - | - -error: called `map().unwrap_or()` on an `Result` value --> tests/ui/map_unwrap_or_fixable.rs:84:20 | LL | println!("{}", x.map(|y| y + 1).unwrap_or(3)); @@ -91,14 +79,26 @@ LL - println!("{}", x.map(|y| y + 1).unwrap_or(3)); LL + println!("{}", x.map_or(3, |y| y + 1)); | -error: called `map().unwrap_or_else()` on an `Option` value +error: called `map().unwrap_or()` on a `Result` value --> tests/ui/map_unwrap_or_fixable.rs:88:20 | +LL | println!("{}", x.map(|y| y + 1).unwrap_or(3)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: use `map_or(, )` instead + | +LL - println!("{}", x.map(|y| y + 1).unwrap_or(3)); +LL + println!("{}", x.map_or(3, |y| y + 1)); + | + +error: called `map().unwrap_or_else()` on an `Option` value + --> tests/ui/map_unwrap_or_fixable.rs:92:20 + | LL | println!("{}", x.map(|y| y + 1).unwrap_or_else(|| 3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `x.map_or_else(|| 3, |y| y + 1)` error: called `map().unwrap_or_else()` on a `Result` value - --> tests/ui/map_unwrap_or_fixable.rs:92:20 + --> tests/ui/map_unwrap_or_fixable.rs:96:20 | LL | println!("{}", x.map(|y| y + 1).unwrap_or_else(|_| 3)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `x.map_or_else(|_| 3, |y| y + 1)` From 4976fdf203b5122bf3b26acf02af5ae6ea25f138 Mon Sep 17 00:00:00 2001 From: Linshu Yang Date: Wed, 7 Jan 2026 23:17:33 +0000 Subject: [PATCH 0103/1843] fix: `significant_drop_tightening` suggests wrongly for non-method usage --- .../src/significant_drop_tightening.rs | 24 ++++--- tests/ui/significant_drop_tightening.fixed | 36 ++++++++++ tests/ui/significant_drop_tightening.rs | 33 +++++++++ tests/ui/significant_drop_tightening.stderr | 71 ++++++++++++++++++- 4 files changed, 152 insertions(+), 12 deletions(-) diff --git a/clippy_lints/src/significant_drop_tightening.rs b/clippy_lints/src/significant_drop_tightening.rs index fabb21f78b9e..8557e8d18d10 100644 --- a/clippy_lints/src/significant_drop_tightening.rs +++ b/clippy_lints/src/significant_drop_tightening.rs @@ -2,6 +2,7 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::MaybeResPath; use clippy_utils::source::{indent_of, snippet}; use clippy_utils::{expr_or_init, get_builtin_attr, peel_hir_expr_unary, sym}; +use rustc_ast::BindingMode; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_errors::Applicability; use rustc_hir::def::{DefKind, Res}; @@ -89,13 +90,14 @@ impl<'tcx> LateLintPass<'tcx> for SignificantDropTightening<'tcx> { |diag| { match apa.counter { 0 | 1 => {}, - 2 => { + 2 if let Some(last_method_span) = apa.last_method_span => { let indent = " ".repeat(indent_of(cx, apa.last_stmt_span).unwrap_or(0)); let init_method = snippet(cx, apa.first_method_span, ".."); - let usage_method = snippet(cx, apa.last_method_span, ".."); - let stmt = if let Some(last_bind_ident) = apa.last_bind_ident { + let usage_method = snippet(cx, last_method_span, ".."); + let stmt = if let Some((binding_mode, last_bind_ident)) = apa.last_bind_ident { format!( - "\n{indent}let {} = {init_method}.{usage_method};", + "\n{indent}let {}{} = {init_method}.{usage_method};", + binding_mode.prefix_str(), snippet(cx, last_bind_ident.span, ".."), ) } else { @@ -310,13 +312,13 @@ impl<'tcx> Visitor<'tcx> for StmtsChecker<'_, '_, '_, '_, 'tcx> { }; match self.ap.curr_stmt.kind { hir::StmtKind::Let(local) => { - if let hir::PatKind::Binding(_, _, ident, _) = local.pat.kind { - apa.last_bind_ident = Some(ident); + if let hir::PatKind::Binding(binding_mode, _, ident, _) = local.pat.kind { + apa.last_bind_ident = Some((binding_mode, ident)); } if let Some(local_init) = local.init && let hir::ExprKind::MethodCall(_, _, _, span) = local_init.kind { - apa.last_method_span = span; + apa.last_method_span = Some(span); } }, hir::StmtKind::Semi(semi_expr) => { @@ -326,7 +328,7 @@ impl<'tcx> Visitor<'tcx> for StmtsChecker<'_, '_, '_, '_, 'tcx> { return; } if let hir::ExprKind::MethodCall(_, _, _, span) = semi_expr.kind { - apa.last_method_span = span; + apa.last_method_span = Some(span); } }, _ => {}, @@ -385,9 +387,9 @@ struct AuxParamsAttr { /// The last visited binding or variable span within a block that had any referenced inner type /// marked with `#[has_significant_drop]`. - last_bind_ident: Option, + last_bind_ident: Option<(BindingMode, Ident)>, /// Similar to `last_bind_span` but encompasses the right-hand method call. - last_method_span: Span, + last_method_span: Option, /// Similar to `last_bind_span` but encompasses the whole contained statement. last_stmt_span: Span, } @@ -403,7 +405,7 @@ impl Default for AuxParamsAttr { first_method_span: DUMMY_SP, first_stmt_span: DUMMY_SP, last_bind_ident: None, - last_method_span: DUMMY_SP, + last_method_span: None, last_stmt_span: DUMMY_SP, } } diff --git a/tests/ui/significant_drop_tightening.fixed b/tests/ui/significant_drop_tightening.fixed index 3d416056226c..559c1bb94570 100644 --- a/tests/ui/significant_drop_tightening.fixed +++ b/tests/ui/significant_drop_tightening.fixed @@ -146,3 +146,39 @@ pub fn unnecessary_contention_with_single_owned_results() { pub fn do_heavy_computation_that_takes_time(_: T) {} fn main() {} + +fn issue15574() { + use std::io::{BufRead, Read, stdin}; + use std::process; + + println!("Hello, what's your name?"); + + let mut stdin = stdin().lock().take(40); + //~^ significant_drop_tightening + let mut buffer = String::with_capacity(10); + + + //~^ significant_drop_tightening + if stdin.read_line(&mut buffer).is_err() { + eprintln!("An error has occured while reading."); + return; + } + drop(stdin); + println!("Our string has a capacity of {}", buffer.capacity()); + println!("Hello {}!", buffer); +} + +fn issue16343() { + fn get_items(x: &()) -> Vec<()> { + vec![*x] + } + + let storage = Mutex::new(()); + let lock = storage.lock().unwrap(); + //~^ significant_drop_tightening + let items = get_items(&lock); + drop(lock); + for item in items { + println!("item {:?}", item); + } +} diff --git a/tests/ui/significant_drop_tightening.rs b/tests/ui/significant_drop_tightening.rs index d9c4ad543593..dff36baf383d 100644 --- a/tests/ui/significant_drop_tightening.rs +++ b/tests/ui/significant_drop_tightening.rs @@ -142,3 +142,36 @@ pub fn unnecessary_contention_with_single_owned_results() { pub fn do_heavy_computation_that_takes_time(_: T) {} fn main() {} + +fn issue15574() { + use std::io::{BufRead, Read, stdin}; + use std::process; + + println!("Hello, what's your name?"); + let stdin = stdin().lock(); + //~^ significant_drop_tightening + let mut buffer = String::with_capacity(10); + + let mut stdin = stdin.take(40); + //~^ significant_drop_tightening + if stdin.read_line(&mut buffer).is_err() { + eprintln!("An error has occured while reading."); + return; + } + println!("Our string has a capacity of {}", buffer.capacity()); + println!("Hello {}!", buffer); +} + +fn issue16343() { + fn get_items(x: &()) -> Vec<()> { + vec![*x] + } + + let storage = Mutex::new(()); + let lock = storage.lock().unwrap(); + //~^ significant_drop_tightening + let items = get_items(&lock); + for item in items { + println!("item {:?}", item); + } +} diff --git a/tests/ui/significant_drop_tightening.stderr b/tests/ui/significant_drop_tightening.stderr index 25cd9da73a10..785bee4970ec 100644 --- a/tests/ui/significant_drop_tightening.stderr +++ b/tests/ui/significant_drop_tightening.stderr @@ -83,5 +83,74 @@ LL | LL ~ | -error: aborting due to 4 previous errors +error: temporary with significant `Drop` can be early dropped + --> tests/ui/significant_drop_tightening.rs:151:9 + | +LL | fn issue15574() { + | _________________- +LL | | use std::io::{BufRead, Read, stdin}; +LL | | use std::process; +... | +LL | | let stdin = stdin().lock(); + | | ^^^^^ +... | +LL | | println!("Hello {}!", buffer); +LL | | } + | |_- temporary `stdin` is currently being dropped at the end of its contained scope + | + = note: this might lead to unnecessary resource contention +help: merge the temporary construction with its single usage + | +LL ~ +LL + let mut stdin = stdin().lock().take(40); +LL | +LL | let mut buffer = String::with_capacity(10); +LL | +LL ~ + | + +error: temporary with significant `Drop` can be early dropped + --> tests/ui/significant_drop_tightening.rs:155:13 + | +LL | fn issue15574() { + | _________________- +LL | | use std::io::{BufRead, Read, stdin}; +LL | | use std::process; +... | +LL | | let mut stdin = stdin.take(40); + | | ^^^^^ +... | +LL | | println!("Hello {}!", buffer); +LL | | } + | |_- temporary `stdin` is currently being dropped at the end of its contained scope + | + = note: this might lead to unnecessary resource contention +help: drop the temporary after the end of its last usage + | +LL ~ } +LL + drop(stdin); + | + +error: temporary with significant `Drop` can be early dropped + --> tests/ui/significant_drop_tightening.rs:171:9 + | +LL | fn issue16343() { + | _________________- +LL | | fn get_items(x: &()) -> Vec<()> { +LL | | vec![*x] +... | +LL | | let lock = storage.lock().unwrap(); + | | ^^^^ +... | +LL | | } + | |_- temporary `lock` is currently being dropped at the end of its contained scope + | + = note: this might lead to unnecessary resource contention +help: drop the temporary after the end of its last usage + | +LL ~ let items = get_items(&lock); +LL + drop(lock); + | + +error: aborting due to 7 previous errors From e2b507233c3224187dd5ef53a9044a430e4f9129 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Tue, 2 Dec 2025 13:05:47 +1100 Subject: [PATCH 0104/1843] Add ProjectJsonTargetSpec.project_root Needed to support flychecking in a later diff --- src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs | 1 + src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs index 9beab3c0e45c..81d60179cc94 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs @@ -833,6 +833,7 @@ impl GlobalStateSnapshot { label: build.label, target_kind: build.target_kind, shell_runnables: project.runnables().to_owned(), + project_root: project.project_root().to_owned(), })); } ProjectWorkspaceKind::DetachedFile { .. } => {} diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs index e0f95a7830ea..8452b6493e87 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs @@ -68,6 +68,7 @@ pub(crate) struct ProjectJsonTargetSpec { pub(crate) label: String, pub(crate) target_kind: TargetKind, pub(crate) shell_runnables: Vec, + pub(crate) project_root: AbsPathBuf, } impl ProjectJsonTargetSpec { From ff94498ef2a831c47d9237262f612a3bb530bd69 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Tue, 2 Dec 2025 14:02:07 +1100 Subject: [PATCH 0105/1843] project-model: Helpers for traversing dep graph in ProjectJson Needed for all_workspace_dependencies_for_package implementation. --- .../crates/project-model/src/project_json.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs index b3478d2cfe03..adc9b1a49fd4 100644 --- a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs +++ b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs @@ -78,6 +78,13 @@ pub struct ProjectJson { runnables: Vec, } +impl std::ops::Index for ProjectJson { + type Output = Crate; + fn index(&self, index: CrateArrayIdx) -> &Self::Output { + &self.crates[index.0] + } +} + impl ProjectJson { /// Create a new ProjectJson instance. /// @@ -218,6 +225,14 @@ impl ProjectJson { .find(|build| build.build_file.as_std_path() == path) } + pub fn crate_by_label(&self, label: &str) -> Option<&Crate> { + // this is fast enough for now, but it's unfortunate that this is O(crates). + self.crates + .iter() + .filter(|krate| krate.is_workspace_member) + .find(|krate| krate.build.as_ref().is_some_and(|build| build.label == label)) + } + /// Returns the path to the project's manifest or root folder, if no manifest exists. pub fn manifest_or_root(&self) -> &AbsPath { self.manifest.as_ref().map_or(&self.project_root, |manifest| manifest.as_ref()) @@ -258,6 +273,12 @@ pub struct Crate { pub build: Option, } +impl Crate { + pub fn iter_deps(&self) -> impl ExactSizeIterator { + self.deps.iter().map(|dep| dep.krate) + } +} + /// Additional, build-specific data about a crate. #[derive(Clone, Debug, Eq, PartialEq)] pub struct Build { From 3083bde73d2e7a1e2227ef77f1f248598005ba08 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Tue, 2 Dec 2025 14:02:25 +1100 Subject: [PATCH 0106/1843] project-model: Don't do O(n) clones as well as O(n) search --- .../rust-analyzer/crates/project-model/src/project_json.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs index adc9b1a49fd4..8fe7885983a7 100644 --- a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs +++ b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs @@ -221,8 +221,9 @@ impl ProjectJson { self.crates .iter() .filter(|krate| krate.is_workspace_member) - .filter_map(|krate| krate.build.clone()) + .filter_map(|krate| krate.build.as_ref()) .find(|build| build.build_file.as_std_path() == path) + .cloned() } pub fn crate_by_label(&self, label: &str) -> Option<&Crate> { From ac641771a83d23f965b3dcb180bb854de21002fb Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0107/1843] project-model: Return crate by reference --- .../rust-analyzer/crates/project-model/src/project_json.rs | 3 +-- .../rust-analyzer/crates/rust-analyzer/src/global_state.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs index 8fe7885983a7..a7fba6936244 100644 --- a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs +++ b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs @@ -202,12 +202,11 @@ impl ProjectJson { &self.project_root } - pub fn crate_by_root(&self, root: &AbsPath) -> Option { + pub fn crate_by_root(&self, root: &AbsPath) -> Option<&Crate> { self.crates .iter() .filter(|krate| krate.is_workspace_member) .find(|krate| krate.root_module == root) - .cloned() } /// Returns the path to the project's manifest, if it exists. diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs index 81d60179cc94..99dc8bce062e 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs @@ -825,7 +825,7 @@ impl GlobalStateSnapshot { let Some(krate) = project.crate_by_root(path) else { continue; }; - let Some(build) = krate.build else { + let Some(build) = krate.build.clone() else { continue; }; From 3b97d38702700a274b22dcd9cc21d836d4dc1ce0 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Tue, 2 Dec 2025 14:11:52 +1100 Subject: [PATCH 0108/1843] Fix misuse of ? This exited the whole loop instead of having continue semantics and continuing to find workspaces. So wrap in find_map. --- .../crates/rust-analyzer/src/global_state.rs | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs index 99dc8bce062e..68d65cdee6f3 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs @@ -847,21 +847,18 @@ impl GlobalStateSnapshot { &self, package: &Arc, ) -> Option>> { - for workspace in self.workspaces.iter() { - match &workspace.kind { - ProjectWorkspaceKind::Cargo { cargo, .. } - | ProjectWorkspaceKind::DetachedFile { cargo: Some((cargo, _, _)), .. } => { - let package = cargo.packages().find(|p| cargo[*p].id == *package)?; + self.workspaces.iter().find_map(|workspace| match &workspace.kind { + ProjectWorkspaceKind::Cargo { cargo, .. } + | ProjectWorkspaceKind::DetachedFile { cargo: Some((cargo, _, _)), .. } => { + let package = cargo.packages().find(|p| cargo[*p].id == *package)?; - return cargo[package] - .all_member_deps - .as_ref() - .map(|deps| deps.iter().map(|dep| cargo[*dep].id.clone()).collect()); - } - _ => {} + return cargo[package] + .all_member_deps + .as_ref() + .map(|deps| deps.iter().map(|dep| cargo[*dep].id.clone()).collect()); } - } - None + _ => None, + }) } pub(crate) fn file_exists(&self, file_id: FileId) -> bool { From 327ea186f07053f70b4bf6af394e6f8ec80f303e Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0109/1843] flycheck: Make the flycheckable unit a flycheck::PackageSpecifier enum You should be able to flycheck a ProjectJson crate based on its build label. This paves the way for that. Context: I don't think this has been working for some time. It used to be that we would use cargo to build ProjectJson crates. Support for ProjectJson seems to have been somewhat steamrolled in PR 18845 (e4bf6e1bc36e4cbc8a36d7911788176eb9fac76e). --- .../crates/rust-analyzer/src/diagnostics.rs | 16 +++-- .../crates/rust-analyzer/src/flycheck.rs | 69 +++++++++++++++---- .../crates/rust-analyzer/src/global_state.rs | 50 ++++++++++---- .../src/handlers/notification.rs | 42 +++++++---- 4 files changed, 130 insertions(+), 47 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs index 4a247800af9d..712960f13d7e 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs @@ -3,7 +3,6 @@ pub(crate) mod flycheck_to_proto; use std::mem; -use cargo_metadata::PackageId; use ide::FileId; use ide_db::{FxHashMap, base_db::DbPanicContext}; use itertools::Itertools; @@ -12,10 +11,13 @@ use smallvec::SmallVec; use stdx::iter_eq_by; use triomphe::Arc; -use crate::{global_state::GlobalStateSnapshot, lsp, lsp_ext, main_loop::DiagnosticsTaskKind}; +use crate::{ + flycheck::PackageSpecifier, global_state::GlobalStateSnapshot, lsp, lsp_ext, + main_loop::DiagnosticsTaskKind, +}; pub(crate) type CheckFixes = - Arc>, FxHashMap>>>>; + Arc, FxHashMap>>>>; #[derive(Debug, Default, Clone)] pub struct DiagnosticsMapConfig { @@ -29,7 +31,7 @@ pub(crate) type DiagnosticsGeneration = usize; #[derive(Debug, Clone, Default)] pub(crate) struct WorkspaceFlycheckDiagnostic { - pub(crate) per_package: FxHashMap>, PackageFlycheckDiagnostic>, + pub(crate) per_package: FxHashMap, PackageFlycheckDiagnostic>, } #[derive(Debug, Clone)] @@ -85,7 +87,7 @@ impl DiagnosticCollection { pub(crate) fn clear_check_for_package( &mut self, flycheck_id: usize, - package_id: Arc, + package_id: PackageSpecifier, ) { let Some(check) = self.check.get_mut(flycheck_id) else { return; @@ -124,7 +126,7 @@ impl DiagnosticCollection { pub(crate) fn clear_check_older_than_for_package( &mut self, flycheck_id: usize, - package_id: Arc, + package_id: PackageSpecifier, generation: DiagnosticsGeneration, ) { let Some(check) = self.check.get_mut(flycheck_id) else { @@ -154,7 +156,7 @@ impl DiagnosticCollection { &mut self, flycheck_id: usize, generation: DiagnosticsGeneration, - package_id: &Option>, + package_id: &Option, file_id: FileId, diagnostic: lsp_types::Diagnostic, fix: Option>, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index b06264169188..2819ae98daaf 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -195,9 +195,9 @@ impl FlycheckHandle { /// Schedule a re-start of the cargo check worker to do a package wide check. pub(crate) fn restart_for_package( &self, - package: Arc, + package: PackageSpecifier, target: Option, - workspace_deps: Option>>, + workspace_deps: Option>, ) { let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; self.sender @@ -233,7 +233,7 @@ pub(crate) enum ClearDiagnosticsKind { #[derive(Debug)] pub(crate) enum ClearScope { Workspace, - Package(Arc), + Package(PackageSpecifier), } pub(crate) enum FlycheckMessage { @@ -243,7 +243,7 @@ pub(crate) enum FlycheckMessage { generation: DiagnosticsGeneration, workspace_root: Arc, diagnostic: Diagnostic, - package_id: Option>, + package_id: Option, }, /// Request clearing all outdated diagnostics. @@ -295,7 +295,32 @@ pub(crate) enum Progress { enum FlycheckScope { Workspace, - Package { package: Arc, workspace_deps: Option>> }, + Package { + // Either a cargo package or a $label in rust-project.check.overrideCommand + package: PackageSpecifier, + workspace_deps: Option>, + }, +} + +#[derive(Debug, Hash, PartialEq, Eq, Clone)] +pub(crate) enum PackageSpecifier { + Cargo { + /// The one in Cargo.toml, assumed to work with `cargo check -p {}` etc + package_id: Arc, + }, + BuildInfo { + /// If a `build` field is present in rust-project.json, its label field + label: String, + }, +} + +impl PackageSpecifier { + pub(crate) fn as_str(&self) -> &str { + match self { + Self::Cargo { package_id } => &package_id.repr, + Self::BuildInfo { label } => label, + } + } } enum StateChange { @@ -331,7 +356,7 @@ struct FlycheckActor { command_handle: Option>, /// The receiver side of the channel mentioned above. command_receiver: Option>, - diagnostics_cleared_for: FxHashSet>, + diagnostics_cleared_for: FxHashSet, diagnostics_received: DiagnosticsReceived, } @@ -564,7 +589,10 @@ impl FlycheckActor { msg.target.kind.iter().format_with(", ", |kind, f| f(&kind)), ))); let package_id = Arc::new(msg.package_id); - if self.diagnostics_cleared_for.insert(package_id.clone()) { + if self + .diagnostics_cleared_for + .insert(PackageSpecifier::Cargo { package_id: package_id.clone() }) + { tracing::trace!( flycheck_id = self.id, package_id = package_id.repr, @@ -572,7 +600,9 @@ impl FlycheckActor { ); self.send(FlycheckMessage::ClearDiagnostics { id: self.id, - kind: ClearDiagnosticsKind::All(ClearScope::Package(package_id)), + kind: ClearDiagnosticsKind::All(ClearScope::Package( + PackageSpecifier::Cargo { package_id }, + )), }); } } @@ -580,7 +610,7 @@ impl FlycheckActor { tracing::trace!( flycheck_id = self.id, message = diagnostic.message, - package_id = package_id.as_ref().map(|it| &it.repr), + package_id = package_id.as_ref().map(|it| it.as_str()), "diagnostic received" ); if self.diagnostics_received == DiagnosticsReceived::No { @@ -590,7 +620,7 @@ impl FlycheckActor { if self.diagnostics_cleared_for.insert(package_id.clone()) { tracing::trace!( flycheck_id = self.id, - package_id = package_id.repr, + package_id = package_id.as_str(), "clearing diagnostics" ); self.send(FlycheckMessage::ClearDiagnostics { @@ -666,7 +696,18 @@ impl FlycheckActor { match scope { FlycheckScope::Workspace => cmd.arg("--workspace"), - FlycheckScope::Package { package, .. } => cmd.arg("-p").arg(&package.repr), + FlycheckScope::Package { + package: PackageSpecifier::Cargo { package_id }, + .. + } => cmd.arg("-p").arg(&package_id.repr), + FlycheckScope::Package { + package: PackageSpecifier::BuildInfo { .. }, .. + } => { + // No way to flycheck this single package. All we have is a build label. + // There's no way to really say whether this build label happens to be + // a cargo canonical name, so we won't try. + return None; + } }; if let Some(tgt) = target { @@ -748,7 +789,7 @@ impl FlycheckActor { #[allow(clippy::large_enum_variant)] enum CargoCheckMessage { CompilerArtifact(cargo_metadata::Artifact), - Diagnostic { diagnostic: Diagnostic, package_id: Option> }, + Diagnostic { diagnostic: Diagnostic, package_id: Option }, } struct CargoCheckParser; @@ -767,7 +808,9 @@ impl JsonLinesParser for CargoCheckParser { cargo_metadata::Message::CompilerMessage(msg) => { Some(CargoCheckMessage::Diagnostic { diagnostic: msg.message, - package_id: Some(Arc::new(msg.package_id)), + package_id: Some(PackageSpecifier::Cargo { + package_id: Arc::new(msg.package_id), + }), }) } _ => None, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs index 68d65cdee6f3..0cfd0a141bae 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs @@ -9,7 +9,6 @@ use std::{ time::{Duration, Instant}, }; -use cargo_metadata::PackageId; use crossbeam_channel::{Receiver, Sender, unbounded}; use hir::ChangeWithProcMacros; use ide::{Analysis, AnalysisHost, Cancellable, FileId, SourceRootId}; @@ -36,7 +35,7 @@ use crate::{ config::{Config, ConfigChange, ConfigErrors, RatomlFileKind}, diagnostics::{CheckFixes, DiagnosticCollection}, discover, - flycheck::{FlycheckHandle, FlycheckMessage}, + flycheck::{FlycheckHandle, FlycheckMessage, PackageSpecifier}, line_index::{LineEndings, LineIndex}, lsp::{from_proto, to_proto::url_from_abs_path}, lsp_ext, @@ -845,20 +844,43 @@ impl GlobalStateSnapshot { pub(crate) fn all_workspace_dependencies_for_package( &self, - package: &Arc, - ) -> Option>> { - self.workspaces.iter().find_map(|workspace| match &workspace.kind { - ProjectWorkspaceKind::Cargo { cargo, .. } - | ProjectWorkspaceKind::DetachedFile { cargo: Some((cargo, _, _)), .. } => { - let package = cargo.packages().find(|p| cargo[*p].id == *package)?; + package: &PackageSpecifier, + ) -> Option> { + match package { + PackageSpecifier::Cargo { package_id } => { + self.workspaces.iter().find_map(|workspace| match &workspace.kind { + ProjectWorkspaceKind::Cargo { cargo, .. } + | ProjectWorkspaceKind::DetachedFile { cargo: Some((cargo, _, _)), .. } => { + let package = cargo.packages().find(|p| cargo[*p].id == *package_id)?; - return cargo[package] - .all_member_deps - .as_ref() - .map(|deps| deps.iter().map(|dep| cargo[*dep].id.clone()).collect()); + cargo[package].all_member_deps.as_ref().map(|deps| { + deps.iter() + .map(|dep| cargo[*dep].id.clone()) + .map(|p| PackageSpecifier::Cargo { package_id: p }) + .collect() + }) + } + _ => None, + }) } - _ => None, - }) + PackageSpecifier::BuildInfo { label } => { + self.workspaces.iter().find_map(|workspace| match &workspace.kind { + ProjectWorkspaceKind::Json(p) => { + let krate = p.crate_by_label(label)?; + Some( + krate + .iter_deps() + .filter_map(|dep| p[dep].build.as_ref()) + .map(|build| PackageSpecifier::BuildInfo { + label: build.label.clone(), + }) + .collect(), + ) + } + _ => None, + }) + } + } } pub(crate) fn file_exists(&self, file_id: FileId) -> bool { diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs index 4a6544508ff4..57adbbfe72a7 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs @@ -18,7 +18,7 @@ use vfs::{AbsPathBuf, ChangeKind, VfsPath}; use crate::{ config::{Config, ConfigChange}, - flycheck::{InvocationStrategy, Target}, + flycheck::{InvocationStrategy, PackageSpecifier, Target}, global_state::{FetchWorkspaceRequest, GlobalState}, lsp::{from_proto, utils::apply_document_changes}, lsp_ext::{self, RunFlycheckParams}, @@ -328,22 +328,32 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool { } InvocationStrategy::PerWorkspace => { Box::new(move || { - let target = TargetSpec::for_file(&world, file_id)?.and_then(|it| { + let target = TargetSpec::for_file(&world, file_id)?.map(|it| { let tgt_kind = it.target_kind(); let (tgt_name, root, package) = match it { - TargetSpec::Cargo(c) => (c.target, c.workspace_root, c.package_id), - _ => return None, + TargetSpec::Cargo(c) => ( + Some(c.target), + c.workspace_root, + PackageSpecifier::Cargo { package_id: c.package_id }, + ), + TargetSpec::ProjectJson(p) => ( + None, + p.project_root, + PackageSpecifier::BuildInfo { label: p.label.clone() }, + ), }; - let tgt = match tgt_kind { - project_model::TargetKind::Bin => Target::Bin(tgt_name), - project_model::TargetKind::Example => Target::Example(tgt_name), - project_model::TargetKind::Test => Target::Test(tgt_name), - project_model::TargetKind::Bench => Target::Benchmark(tgt_name), - _ => return Some((None, root, package)), - }; + let tgt = tgt_name.and_then(|tgt_name| { + Some(match tgt_kind { + project_model::TargetKind::Bin => Target::Bin(tgt_name), + project_model::TargetKind::Example => Target::Example(tgt_name), + project_model::TargetKind::Test => Target::Test(tgt_name), + project_model::TargetKind::Bench => Target::Benchmark(tgt_name), + _ => return None, + }) + }); - Some((Some(tgt), root, package)) + (tgt, root, package) }); tracing::debug!(?target, "flycheck target"); // we have a specific non-library target, attempt to only check that target, nothing @@ -365,7 +375,13 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool { cargo: Some((cargo, _, _)), .. } => *cargo.workspace_root() == root, - _ => false, + project_model::ProjectWorkspaceKind::Json(p) => { + *p.project_root() == root + } + project_model::ProjectWorkspaceKind::DetachedFile { + cargo: None, + .. + } => false, }); if let Some(idx) = package_workspace_idx { let workspace_deps = From 95a07dbfa06ad7c30d4e9fa2f85de2b991d610f3 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0110/1843] project-model: Introduce RunnableKind::Flycheck We need to distinguish from RunnableKind::Check, which is human-readable. --- .../crates/project-model/src/project_json.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs index a7fba6936244..536f170e1192 100644 --- a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs +++ b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs @@ -246,6 +246,10 @@ impl ProjectJson { pub fn runnables(&self) -> &[Runnable] { &self.runnables } + + pub fn runnable_template(&self, kind: RunnableKind) -> Option<&Runnable> { + self.runnables().iter().find(|r| r.kind == kind) + } } /// A crate points to the root module of a crate and lists the dependencies of the crate. This is @@ -349,6 +353,7 @@ pub struct Runnable { /// The kind of runnable. #[derive(Debug, Clone, PartialEq, Eq)] pub enum RunnableKind { + /// `cargo check`, basically, with human-readable output. Check, /// Can run a binary. @@ -356,6 +361,10 @@ pub enum RunnableKind { /// Run a single test. TestOne, + + /// Template for checking a target, emitting rustc JSON diagnostics. + /// May include {label} which will get the label from the `build` section of a crate. + Flycheck, } #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] @@ -462,6 +471,7 @@ pub struct RunnableData { #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub enum RunnableKindData { + Flycheck, Check, Run, TestOne, @@ -532,6 +542,7 @@ impl From for RunnableKind { RunnableKindData::Check => RunnableKind::Check, RunnableKindData::Run => RunnableKind::Run, RunnableKindData::TestOne => RunnableKind::TestOne, + RunnableKindData::Flycheck => RunnableKind::Flycheck, } } } From 2a899bb119bd9e33eadce3b7179cb1b8bc49fd78 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0111/1843] flycheck: Use RunnableKind::Flycheck from ProjectJson to flycheck This adds a substitution helper to get the right behaviour re {label} and $saved_file. --- .../crates/rust-analyzer/src/flycheck.rs | 226 +++++++++++++++++- .../crates/rust-analyzer/src/reload.rs | 19 +- 2 files changed, 238 insertions(+), 7 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index 2819ae98daaf..1b1e3344e25f 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -14,6 +14,7 @@ use ide_db::FxHashSet; use itertools::Itertools; use paths::{AbsPath, AbsPathBuf, Utf8Path, Utf8PathBuf}; use project_model::TargetDirectoryConfig; +use project_model::project_json; use rustc_hash::FxHashMap; use serde::Deserialize as _; use serde_derive::Deserialize; @@ -89,6 +90,24 @@ impl CargoOptions { } } +/// The flycheck config from a rust-project.json file or discoverConfig JSON output. +#[derive(Debug, Default)] +pub(crate) struct FlycheckConfigJson { + /// The template with [project_json::RunnableKind::Flycheck] + pub single_template: Option, +} + +impl FlycheckConfigJson { + pub(crate) fn any_configured(&self) -> bool { + // self.workspace_template.is_some() || + self.single_template.is_some() + } +} + +/// The flycheck config from rust-analyzer's own configuration. +/// +/// We rely on this when rust-project.json does not specify a flycheck runnable +/// #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum FlycheckConfig { CargoCommand { @@ -128,7 +147,7 @@ impl fmt::Display for FlycheckConfig { // in the IDE (e.g. in the VS Code status bar). let display_args = args .iter() - .map(|arg| if arg == SAVED_FILE_PLACEHOLDER { "..." } else { arg }) + .map(|arg| if arg == SAVED_FILE_PLACEHOLDER_DOLLAR { "..." } else { arg }) .collect::>(); write!(f, "{command} {}", display_args.join(" ")) @@ -156,6 +175,7 @@ impl FlycheckHandle { generation: Arc, sender: Sender, config: FlycheckConfig, + config_json: FlycheckConfigJson, sysroot_root: Option, workspace_root: AbsPathBuf, manifest_path: Option, @@ -166,6 +186,7 @@ impl FlycheckHandle { generation.load(Ordering::Relaxed), sender, config, + config_json, sysroot_root, workspace_root, manifest_path, @@ -341,6 +362,8 @@ struct FlycheckActor { generation: DiagnosticsGeneration, sender: Sender, config: FlycheckConfig, + config_json: FlycheckConfigJson, + manifest_path: Option, ws_target_dir: Option, /// Either the workspace root of the workspace we are flychecking, @@ -373,7 +396,66 @@ enum Event { CheckEvent(Option), } -pub(crate) const SAVED_FILE_PLACEHOLDER: &str = "$saved_file"; +/// This is stable behaviour. Don't change. +const SAVED_FILE_PLACEHOLDER_DOLLAR: &str = "$saved_file"; +const LABEL_INLINE: &str = "{label}"; +const SAVED_FILE_INLINE: &str = "{saved_file}"; + +struct Substitutions<'a> { + label: Option<&'a str>, + saved_file: Option<&'a str>, +} + +impl<'a> Substitutions<'a> { + /// If you have a runnable, and it has {label} in it somewhere, treat it as a template that + /// may be unsatisfied if you do not provide a label to substitute into it. Returns None in + /// that situation. Otherwise performs the requested substitutions. + /// + /// Same for {saved_file}. + /// + #[allow(clippy::disallowed_types)] /* generic parameter allows for FxHashMap */ + fn substitute( + self, + template: &project_json::Runnable, + extra_env: &std::collections::HashMap, H>, + ) -> Option { + let mut cmd = toolchain::command(&template.program, &template.cwd, extra_env); + for arg in &template.args { + if let Some(ix) = arg.find(LABEL_INLINE) { + if let Some(label) = self.label { + let mut arg = arg.to_string(); + arg.replace_range(ix..ix + LABEL_INLINE.len(), label); + cmd.arg(arg); + continue; + } else { + return None; + } + } + if let Some(ix) = arg.find(SAVED_FILE_INLINE) { + if let Some(saved_file) = self.saved_file { + let mut arg = arg.to_string(); + arg.replace_range(ix..ix + SAVED_FILE_INLINE.len(), saved_file); + cmd.arg(arg); + continue; + } else { + return None; + } + } + // Legacy syntax: full argument match + if arg == SAVED_FILE_PLACEHOLDER_DOLLAR { + if let Some(saved_file) = self.saved_file { + cmd.arg(saved_file); + continue; + } else { + return None; + } + } + cmd.arg(arg); + } + cmd.current_dir(&template.cwd); + Some(cmd) + } +} impl FlycheckActor { fn new( @@ -381,6 +463,7 @@ impl FlycheckActor { generation: DiagnosticsGeneration, sender: Sender, config: FlycheckConfig, + config_json: FlycheckConfigJson, sysroot_root: Option, workspace_root: AbsPathBuf, manifest_path: Option, @@ -392,6 +475,7 @@ impl FlycheckActor { generation, sender, config, + config_json, sysroot_root, root: Arc::new(workspace_root), scope: FlycheckScope::Workspace, @@ -672,6 +756,29 @@ impl FlycheckActor { self.diagnostics_received = DiagnosticsReceived::No; } + fn explicit_check_command( + &self, + scope: &FlycheckScope, + saved_file: Option<&AbsPath>, + ) -> Option { + let label = match scope { + // We could add a runnable like "RunnableKind::FlycheckWorkspace". But generally + // if you're not running cargo, it's because your workspace is too big to check + // all at once. You can always use `check_overrideCommand` with no {label}. + FlycheckScope::Workspace => return None, + FlycheckScope::Package { package: PackageSpecifier::BuildInfo { label }, .. } => { + label.as_str() + } + FlycheckScope::Package { + package: PackageSpecifier::Cargo { package_id: label }, + .. + } => &label.repr, + }; + let template = self.config_json.single_template.as_ref()?; + let subs = Substitutions { label: Some(label), saved_file: saved_file.map(|x| x.as_str()) }; + subs.substitute(template, &FxHashMap::default()) + } + /// Construct a `Command` object for checking the user's code. If the user /// has specified a custom command with placeholders that we cannot fill, /// return None. @@ -683,6 +790,20 @@ impl FlycheckActor { ) -> Option { match &self.config { FlycheckConfig::CargoCommand { command, options, ansi_color_output } => { + // Only use the rust-project.json's flycheck config when no check_overrideCommand + // is configured. In the FlycheckConcig::CustomCommand branch we will still do + // label substitution, but on the overrideCommand instead. + // + // There needs to be SOME way to override what your discoverConfig tool says, + // because to change the flycheck runnable there you may have to literally + // recompile the tool. + if self.config_json.any_configured() { + // Completely handle according to rust-project.json. + // We don't consider this to be "using cargo" so we will not apply any of the + // CargoOptions to the command. + return self.explicit_check_command(scope, saved_file); + } + let mut cmd = toolchain::command(Tool::Cargo.path(), &*self.root, &options.extra_env); if let Some(sysroot_root) = &self.sysroot_root @@ -757,7 +878,7 @@ impl FlycheckActor { // we're saving a file, replace the placeholder in the arguments. if let Some(saved_file) = saved_file { for arg in args { - if arg == SAVED_FILE_PLACEHOLDER { + if arg == SAVED_FILE_PLACEHOLDER_DOLLAR { cmd.arg(saved_file); } else { cmd.arg(arg); @@ -765,7 +886,7 @@ impl FlycheckActor { } } else { for arg in args { - if arg == SAVED_FILE_PLACEHOLDER { + if arg == SAVED_FILE_PLACEHOLDER_DOLLAR { // The custom command has a $saved_file placeholder, // but we had an IDE event that wasn't a file save. Do nothing. return None; @@ -837,3 +958,100 @@ enum JsonMessage { Cargo(cargo_metadata::Message), Rustc(Diagnostic), } + +#[cfg(test)] +mod tests { + use ide_db::FxHashMap; + use itertools::Itertools; + use paths::Utf8Path; + use project_model::project_json; + + use crate::flycheck::Substitutions; + + #[test] + fn test_substitutions() { + let label = ":label"; + let saved_file = "file.rs"; + + // Runnable says it needs both; you need both. + assert_eq!(test_substitute(None, None, "{label} {saved_file}").as_deref(), None); + assert_eq!(test_substitute(Some(label), None, "{label} {saved_file}").as_deref(), None); + assert_eq!( + test_substitute(None, Some(saved_file), "{label} {saved_file}").as_deref(), + None + ); + assert_eq!( + test_substitute(Some(label), Some(saved_file), "{label} {saved_file}").as_deref(), + Some("build :label file.rs") + ); + + // Only need label? only need label. + assert_eq!(test_substitute(None, None, "{label}").as_deref(), None); + assert_eq!(test_substitute(Some(label), None, "{label}").as_deref(), Some("build :label"),); + assert_eq!(test_substitute(None, Some(saved_file), "{label}").as_deref(), None,); + assert_eq!( + test_substitute(Some(label), Some(saved_file), "{label}").as_deref(), + Some("build :label"), + ); + + // Only need saved_file + assert_eq!(test_substitute(None, None, "{saved_file}").as_deref(), None); + assert_eq!(test_substitute(Some(label), None, "{saved_file}").as_deref(), None); + assert_eq!( + test_substitute(None, Some(saved_file), "{saved_file}").as_deref(), + Some("build file.rs") + ); + assert_eq!( + test_substitute(Some(label), Some(saved_file), "{saved_file}").as_deref(), + Some("build file.rs") + ); + + // Need neither + assert_eq!(test_substitute(None, None, "xxx").as_deref(), Some("build xxx")); + assert_eq!(test_substitute(Some(label), None, "xxx").as_deref(), Some("build xxx")); + assert_eq!(test_substitute(None, Some(saved_file), "xxx").as_deref(), Some("build xxx")); + assert_eq!( + test_substitute(Some(label), Some(saved_file), "xxx").as_deref(), + Some("build xxx") + ); + + // {label} mid-argument substitution + assert_eq!( + test_substitute(Some(label), None, "--label={label}").as_deref(), + Some("build --label=:label") + ); + + // {saved_file} mid-argument substitution + assert_eq!( + test_substitute(None, Some(saved_file), "--saved={saved_file}").as_deref(), + Some("build --saved=file.rs") + ); + + // $saved_file legacy support (no mid-argument substitution, we never supported that) + assert_eq!( + test_substitute(None, Some(saved_file), "$saved_file").as_deref(), + Some("build file.rs") + ); + + fn test_substitute( + label: Option<&str>, + saved_file: Option<&str>, + args: &str, + ) -> Option { + Substitutions { label, saved_file } + .substitute( + &project_json::Runnable { + program: "build".to_owned(), + args: Vec::from_iter(args.split_whitespace().map(ToOwned::to_owned)), + cwd: Utf8Path::new("/path").to_owned(), + kind: project_json::RunnableKind::Flycheck, + }, + &FxHashMap::default(), + ) + .map(|command| { + command.get_args().map(|x| x.to_string_lossy()).collect_vec().join(" ") + }) + .map(|args| format!("build {}", args)) + } + } +} diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs index e3a5ee221973..0a16b7a5614c 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs @@ -25,7 +25,9 @@ use load_cargo::{ProjectFolders, load_proc_macro}; use lsp_types::FileSystemWatcher; use paths::Utf8Path; use proc_macro_api::ProcMacroClient; -use project_model::{ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, WorkspaceBuildScripts}; +use project_model::{ + ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, WorkspaceBuildScripts, project_json, +}; use stdx::{format_to, thread::ThreadIntent}; use triomphe::Arc; use vfs::{AbsPath, AbsPathBuf, ChangeKind}; @@ -875,6 +877,7 @@ impl GlobalState { generation.clone(), sender.clone(), config, + crate::flycheck::FlycheckConfigJson::default(), None, self.config.root_path().clone(), None, @@ -894,16 +897,25 @@ impl GlobalState { cargo: Some((cargo, _, _)), .. } => ( + crate::flycheck::FlycheckConfigJson::default(), cargo.workspace_root(), Some(cargo.manifest_path()), Some(cargo.target_directory()), ), ProjectWorkspaceKind::Json(project) => { + let config_json = crate::flycheck::FlycheckConfigJson { + single_template: project + .runnable_template(project_json::RunnableKind::Flycheck) + .cloned(), + }; // Enable flychecks for json projects if a custom flycheck command was supplied // in the workspace configuration. match config { + _ if config_json.any_configured() => { + (config_json, project.path(), None, None) + } FlycheckConfig::CustomCommand { .. } => { - (project.path(), None, None) + (config_json, project.path(), None, None) } _ => return None, } @@ -913,12 +925,13 @@ impl GlobalState { ws.sysroot.root().map(ToOwned::to_owned), )) }) - .map(|(id, (root, manifest_path, target_dir), sysroot_root)| { + .map(|(id, (config_json, root, manifest_path, target_dir), sysroot_root)| { FlycheckHandle::spawn( id, generation.clone(), sender.clone(), config.clone(), + config_json, sysroot_root, root.to_path_buf(), manifest_path.map(|it| it.to_path_buf()), From 7f608da06af36c0c21b6119959e47d9480ccb18a Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0112/1843] flycheck: Support {label} in check_overrideCommand as well as $saved_file --- .../crates/rust-analyzer/src/flycheck.rs | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index 1b1e3344e25f..cf4ab29b8649 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -872,29 +872,23 @@ impl FlycheckActor { &*self.root } }; - let mut cmd = toolchain::command(command, root, extra_env); + let runnable = project_json::Runnable { + program: command.clone(), + cwd: Utf8Path::to_owned(root.as_ref()), + args: args.clone(), + kind: project_json::RunnableKind::Flycheck, + }; - // If the custom command has a $saved_file placeholder, and - // we're saving a file, replace the placeholder in the arguments. - if let Some(saved_file) = saved_file { - for arg in args { - if arg == SAVED_FILE_PLACEHOLDER_DOLLAR { - cmd.arg(saved_file); - } else { - cmd.arg(arg); - } - } - } else { - for arg in args { - if arg == SAVED_FILE_PLACEHOLDER_DOLLAR { - // The custom command has a $saved_file placeholder, - // but we had an IDE event that wasn't a file save. Do nothing. - return None; - } + let label = match scope { + FlycheckScope::Workspace => None, + // We support substituting both build labels (e.g. buck, bazel) and cargo package ids. + // With cargo package ids, you get `cargo check -p path+file:///path/to/rust-analyzer/crates/hir#0.0.0`. + // That does work! + FlycheckScope::Package { package, .. } => Some(package.as_str()), + }; - cmd.arg(arg); - } - } + let subs = Substitutions { label, saved_file: saved_file.map(|x| x.as_str()) }; + let cmd = subs.substitute(&runnable, extra_env)?; Some(cmd) } From 4e61c6052124c2e71403973cc3adb34c4ee5454d Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0113/1843] flycheck: Always flycheck single crate if there is a build label from rust-project.json This requires us to add $saved_file / {saved_file} interpolation back to restart_for_package. Otherwise we break existing users of $saved_file. No grand reason why we can't delete saved_file later, although I would just leave it because sometimes a build system might really know better which target(s) to build, including multiple targets. --- .../rust-analyzer/crates/rust-analyzer/src/flycheck.rs | 3 ++- .../crates/rust-analyzer/src/handlers/notification.rs | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index cf4ab29b8649..57ad774b1850 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -219,13 +219,14 @@ impl FlycheckHandle { package: PackageSpecifier, target: Option, workspace_deps: Option>, + saved_file: Option, ) { let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; self.sender .send(StateChange::Restart { generation, scope: FlycheckScope::Package { package, workspace_deps }, - saved_file: None, + saved_file, target, }) .unwrap(); diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs index 57adbbfe72a7..d95601043330 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs @@ -328,6 +328,7 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool { } InvocationStrategy::PerWorkspace => { Box::new(move || { + let saved_file = vfs_path.as_path().map(ToOwned::to_owned); let target = TargetSpec::for_file(&world, file_id)?.map(|it| { let tgt_kind = it.target_kind(); let (tgt_name, root, package) = match it { @@ -362,8 +363,10 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool { if let Some((target, root, package)) = target { // trigger a package check if we have a non-library target as that can't affect // anything else in the workspace OR if we're not allowed to check the workspace as - // the user opted into package checks then - let package_check_allowed = target.is_some() || !may_flycheck_workspace; + // the user opted into package checks then OR if this is not cargo. + let package_check_allowed = target.is_some() + || !may_flycheck_workspace + || matches!(package, PackageSpecifier::BuildInfo { .. }); if package_check_allowed { package_workspace_idx = world.workspaces.iter().position(|ws| match &ws.kind { @@ -390,6 +393,7 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool { package, target, workspace_deps, + saved_file.clone(), ); } } @@ -460,7 +464,6 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool { ws_contains_file && !is_pkg_ws }); - let saved_file = vfs_path.as_path().map(ToOwned::to_owned); let mut workspace_check_triggered = false; // Find and trigger corresponding flychecks 'flychecks: for flycheck in world.flycheck.iter() { From 778de45547f9a584894fad295c86539e7c57aa9d Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0114/1843] flycheck: Add display_command to pretty-print flycheck command being run in a notification --- .../crates/rust-analyzer/src/flycheck.rs | 82 +++++++++++++++++++ .../rust-analyzer/crates/toolchain/src/lib.rs | 3 + 2 files changed, 85 insertions(+) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index 57ad774b1850..7f814121e909 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -22,6 +22,7 @@ use serde_derive::Deserialize; pub(crate) use cargo_metadata::diagnostic::{ Applicability, Diagnostic, DiagnosticCode, DiagnosticLevel, DiagnosticSpan, }; +use toolchain::DISPLAY_COMMAND_IGNORE_ENVS; use toolchain::Tool; use triomphe::Arc; @@ -954,6 +955,54 @@ enum JsonMessage { Rustc(Diagnostic), } +/// Not good enough to execute in a shell, but good enough to show the user without all the noisy +/// quotes +/// +/// Pass implicit_cwd if there is one regarded as the obvious by the user, so we can skip showing it. +/// Compactness is the aim of the game, the output typically gets truncated quite a lot. +fn display_command(c: &Command, implicit_cwd: Option<&std::path::Path>) -> String { + let mut o = String::new(); + use std::fmt::Write; + let lossy = std::ffi::OsStr::to_string_lossy; + if let Some(dir) = c.get_current_dir() { + if Some(dir) == implicit_cwd.map(std::path::Path::new) { + // pass + } else if dir.to_string_lossy().contains(" ") { + write!(o, "cd {:?} && ", dir).unwrap(); + } else { + write!(o, "cd {} && ", dir.display()).unwrap(); + } + } + for (env, val) in c.get_envs() { + let (env, val) = (lossy(env), val.map(lossy).unwrap_or(std::borrow::Cow::Borrowed(""))); + if DISPLAY_COMMAND_IGNORE_ENVS.contains(&env.as_ref()) { + continue; + } + if env.contains(" ") { + write!(o, "\"{}={}\" ", env, val).unwrap(); + } else if val.contains(" ") { + write!(o, "{}=\"{}\" ", env, val).unwrap(); + } else { + write!(o, "{}={} ", env, val).unwrap(); + } + } + let prog = lossy(c.get_program()); + if prog.contains(" ") { + write!(o, "{:?}", prog).unwrap(); + } else { + write!(o, "{}", prog).unwrap(); + } + for arg in c.get_args() { + let arg = lossy(arg); + if arg.contains(" ") { + write!(o, " \"{}\"", arg).unwrap(); + } else { + write!(o, " {}", arg).unwrap(); + } + } + o +} + #[cfg(test)] mod tests { use ide_db::FxHashMap; @@ -962,6 +1011,7 @@ mod tests { use project_model::project_json; use crate::flycheck::Substitutions; + use crate::flycheck::display_command; #[test] fn test_substitutions() { @@ -1049,4 +1099,36 @@ mod tests { .map(|args| format!("build {}", args)) } } + + #[test] + fn test_display_command() { + use std::path::Path; + let workdir = Path::new("workdir"); + let mut cmd = toolchain::command("command", workdir, &FxHashMap::default()); + assert_eq!(display_command(cmd.arg("--arg"), Some(workdir)), "command --arg"); + assert_eq!( + display_command(cmd.arg("spaced arg"), Some(workdir)), + "command --arg \"spaced arg\"" + ); + assert_eq!( + display_command(cmd.env("ENVIRON", "yeah"), Some(workdir)), + "ENVIRON=yeah command --arg \"spaced arg\"" + ); + assert_eq!( + display_command(cmd.env("OTHER", "spaced env"), Some(workdir)), + "ENVIRON=yeah OTHER=\"spaced env\" command --arg \"spaced arg\"" + ); + assert_eq!( + display_command(cmd.current_dir("/tmp"), Some(workdir)), + "cd /tmp && ENVIRON=yeah OTHER=\"spaced env\" command --arg \"spaced arg\"" + ); + assert_eq!( + display_command(cmd.current_dir("/tmp and/thing"), Some(workdir)), + "cd \"/tmp and/thing\" && ENVIRON=yeah OTHER=\"spaced env\" command --arg \"spaced arg\"" + ); + assert_eq!( + display_command(cmd.current_dir("/tmp and/thing"), Some(Path::new("/tmp and/thing"))), + "ENVIRON=yeah OTHER=\"spaced env\" command --arg \"spaced arg\"" + ); + } } diff --git a/src/tools/rust-analyzer/crates/toolchain/src/lib.rs b/src/tools/rust-analyzer/crates/toolchain/src/lib.rs index 39319886cfe4..1a1726983870 100644 --- a/src/tools/rust-analyzer/crates/toolchain/src/lib.rs +++ b/src/tools/rust-analyzer/crates/toolchain/src/lib.rs @@ -74,6 +74,9 @@ impl Tool { // Prevent rustup from automatically installing toolchains, see https://github.com/rust-lang/rust-analyzer/issues/20719. pub const NO_RUSTUP_AUTO_INSTALL_ENV: (&str, &str) = ("RUSTUP_AUTO_INSTALL", "0"); +// These get ignored when displaying what command is running in LSP status messages. +pub const DISPLAY_COMMAND_IGNORE_ENVS: &[&str] = &[NO_RUSTUP_AUTO_INSTALL_ENV.0]; + #[allow(clippy::disallowed_types)] /* generic parameter allows for FxHashMap */ pub fn command( cmd: impl AsRef, From 53a371c505a00be220aca18dfba091ee2b2d8f31 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 4 Sep 2024 13:52:59 +1000 Subject: [PATCH 0115/1843] flycheck: notifications show full command when configured in a rust-project.json runnable For JSON / override users, pretty-print the custom flycheck command with fewer quote characters Better debug logging in flycheck --- .../crates/rust-analyzer/src/flycheck.rs | 52 ++++++++++++++----- .../crates/rust-analyzer/src/global_state.rs | 2 + .../crates/rust-analyzer/src/main_loop.rs | 27 +++++++--- .../crates/rust-analyzer/src/reload.rs | 1 + 4 files changed, 62 insertions(+), 20 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index 7f814121e909..6dcae76c9354 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -309,13 +309,18 @@ impl fmt::Debug for FlycheckMessage { #[derive(Debug)] pub(crate) enum Progress { - DidStart, + DidStart { + /// The user sees this in VSCode, etc. May be a shortened version of the command we actually + /// executed, otherwise it is way too long. + user_facing_command: String, + }, DidCheckCrate(String), DidFinish(io::Result<()>), DidCancel, DidFailToRestart(String), } +#[derive(Debug, Clone)] enum FlycheckScope { Workspace, Package { @@ -346,6 +351,16 @@ impl PackageSpecifier { } } +#[derive(Debug)] +enum FlycheckCommandOrigin { + /// Regular cargo invocation + Cargo, + /// Configured via check_overrideCommand + CheckOverrideCommand, + /// From a runnable with [project_json::RunnableKind::Flycheck] + ProjectJsonRunnable, +} + enum StateChange { Restart { generation: DiagnosticsGeneration, @@ -529,16 +544,28 @@ impl FlycheckActor { } let command = self.check_command(&scope, saved_file.as_deref(), target); - self.scope = scope; + self.scope = scope.clone(); self.generation = generation; - let Some(command) = command else { + let Some((command, origin)) = command else { + tracing::debug!(?scope, "failed to build flycheck command"); continue; }; - let formatted_command = format!("{command:?}"); + let debug_command = format!("{command:?}"); + let user_facing_command = match origin { + // Don't show all the --format=json-with-blah-blah args, just the simple + // version + FlycheckCommandOrigin::Cargo => self.config.to_string(), + // show them the full command but pretty printed. advanced user + FlycheckCommandOrigin::ProjectJsonRunnable + | FlycheckCommandOrigin::CheckOverrideCommand => display_command( + &command, + Some(std::path::Path::new(self.root.as_path())), + ), + }; - tracing::debug!(?command, "will restart flycheck"); + tracing::debug!(?origin, ?command, "will restart flycheck"); let (sender, receiver) = unbounded(); match CommandHandle::spawn( command, @@ -575,14 +602,14 @@ impl FlycheckActor { }, ) { Ok(command_handle) => { - tracing::debug!(command = formatted_command, "did restart flycheck"); + tracing::debug!(?origin, command = %debug_command, "did restart flycheck"); self.command_handle = Some(command_handle); self.command_receiver = Some(receiver); - self.report_progress(Progress::DidStart); + self.report_progress(Progress::DidStart { user_facing_command }); } Err(error) => { self.report_progress(Progress::DidFailToRestart(format!( - "Failed to run the following command: {formatted_command} error={error}" + "Failed to run the following command: {debug_command} origin={origin:?} error={error}" ))); } } @@ -789,7 +816,7 @@ impl FlycheckActor { scope: &FlycheckScope, saved_file: Option<&AbsPath>, target: Option, - ) -> Option { + ) -> Option<(Command, FlycheckCommandOrigin)> { match &self.config { FlycheckConfig::CargoCommand { command, options, ansi_color_output } => { // Only use the rust-project.json's flycheck config when no check_overrideCommand @@ -803,7 +830,8 @@ impl FlycheckActor { // Completely handle according to rust-project.json. // We don't consider this to be "using cargo" so we will not apply any of the // CargoOptions to the command. - return self.explicit_check_command(scope, saved_file); + let cmd = self.explicit_check_command(scope, saved_file)?; + return Some((cmd, FlycheckCommandOrigin::ProjectJsonRunnable)); } let mut cmd = @@ -864,7 +892,7 @@ impl FlycheckActor { self.ws_target_dir.as_ref().map(Utf8PathBuf::as_path), ); cmd.args(&options.extra_args); - Some(cmd) + Some((cmd, FlycheckCommandOrigin::Cargo)) } FlycheckConfig::CustomCommand { command, args, extra_env, invocation_strategy } => { let root = match invocation_strategy { @@ -892,7 +920,7 @@ impl FlycheckActor { let subs = Substitutions { label, saved_file: saved_file.map(|x| x.as_str()) }; let cmd = subs.substitute(&runnable, extra_env)?; - Some(cmd) + Some((cmd, FlycheckCommandOrigin::CheckOverrideCommand)) } } } diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs index 0cfd0a141bae..39b4aaa64738 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs @@ -112,6 +112,7 @@ pub(crate) struct GlobalState { pub(crate) flycheck_sender: Sender, pub(crate) flycheck_receiver: Receiver, pub(crate) last_flycheck_error: Option, + pub(crate) flycheck_formatted_commands: Vec, // Test explorer pub(crate) test_run_session: Option>, @@ -288,6 +289,7 @@ impl GlobalState { flycheck_sender, flycheck_receiver, last_flycheck_error: None, + flycheck_formatted_commands: vec![], test_run_session: None, test_run_sender, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs index dd0813c14454..62a3b3a17bdf 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs @@ -1179,8 +1179,24 @@ impl GlobalState { kind: ClearDiagnosticsKind::OlderThan(generation, ClearScope::Package(package_id)), } => self.diagnostics.clear_check_older_than_for_package(id, package_id, generation), FlycheckMessage::Progress { id, progress } => { + let format_with_id = |user_facing_command: String| { + if self.flycheck.len() == 1 { + user_facing_command + } else { + format!("{user_facing_command} (#{})", id + 1) + } + }; + + self.flycheck_formatted_commands + .resize_with(self.flycheck.len().max(id + 1), || { + format_with_id(self.config.flycheck(None).to_string()) + }); + let (state, message) = match progress { - flycheck::Progress::DidStart => (Progress::Begin, None), + flycheck::Progress::DidStart { user_facing_command } => { + self.flycheck_formatted_commands[id] = format_with_id(user_facing_command); + (Progress::Begin, None) + } flycheck::Progress::DidCheckCrate(target) => (Progress::Report, Some(target)), flycheck::Progress::DidCancel => { self.last_flycheck_error = None; @@ -1200,13 +1216,8 @@ impl GlobalState { } }; - // When we're running multiple flychecks, we have to include a disambiguator in - // the title, or the editor complains. Note that this is a user-facing string. - let title = if self.flycheck.len() == 1 { - format!("{}", self.config.flycheck(None)) - } else { - format!("{} (#{})", self.config.flycheck(None), id + 1) - }; + // Clone because we &mut self for report_progress + let title = self.flycheck_formatted_commands[id].clone(); self.report_progress( &title, state, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs index 0a16b7a5614c..ccafbd7b30b9 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs @@ -942,6 +942,7 @@ impl GlobalState { } } .into(); + self.flycheck_formatted_commands = vec![]; } } From 3fdb78cba695824cba5be894a032779c9b4a4ac3 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 3 Dec 2025 11:34:58 +1100 Subject: [PATCH 0116/1843] flycheck: Rename FlycheckConfig::CargoCommand to Automatic Because (1) it is what we use when there is no relevant config (2) we automatically use either rust-project.json's flycheck, or cargo This also puts check_command config into CargoOptions. It's a cargo option, after all. --- .../crates/rust-analyzer/src/config.rs | 20 ++++++----- .../crates/rust-analyzer/src/flycheck.rs | 36 ++++++++++++------- .../crates/rust-analyzer/src/test_runner.rs | 2 +- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs index e39569e108de..c2f7ada8c8ca 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs @@ -2431,6 +2431,8 @@ impl Config { pub(crate) fn cargo_test_options(&self, source_root: Option) -> CargoOptions { CargoOptions { + // Might be nice to allow users to specify test_command = "nextest" + subcommand: "test".into(), target_tuples: self.cargo_target(source_root).clone().into_iter().collect(), all_targets: false, no_default_features: *self.cargo_noDefaultFeatures(source_root), @@ -2464,9 +2466,9 @@ impl Config { }, } } - Some(_) | None => FlycheckConfig::CargoCommand { - command: self.check_command(source_root).clone(), - options: CargoOptions { + Some(_) | None => FlycheckConfig::Automatic { + cargo_options: CargoOptions { + subcommand: self.check_command(source_root).clone(), target_tuples: self .check_targets(source_root) .clone() @@ -4171,8 +4173,8 @@ mod tests { assert_eq!(config.cargo_targetDir(None), &None); assert!(matches!( config.flycheck(None), - FlycheckConfig::CargoCommand { - options: CargoOptions { target_dir_config: TargetDirectoryConfig::None, .. }, + FlycheckConfig::Automatic { + cargo_options: CargoOptions { target_dir_config: TargetDirectoryConfig::None, .. }, .. } )); @@ -4195,8 +4197,8 @@ mod tests { Utf8PathBuf::from(std::env::var("CARGO_TARGET_DIR").unwrap_or("target".to_owned())); assert!(matches!( config.flycheck(None), - FlycheckConfig::CargoCommand { - options: CargoOptions { target_dir_config, .. }, + FlycheckConfig::Automatic { + cargo_options: CargoOptions { target_dir_config, .. }, .. } if target_dir_config.target_dir(Some(&ws_target_dir)).map(Cow::into_owned) == Some(ws_target_dir.join("rust-analyzer")) @@ -4221,8 +4223,8 @@ mod tests { ); assert!(matches!( config.flycheck(None), - FlycheckConfig::CargoCommand { - options: CargoOptions { target_dir_config, .. }, + FlycheckConfig::Automatic { + cargo_options: CargoOptions { target_dir_config, .. }, .. } if target_dir_config.target_dir(None).map(Cow::into_owned) == Some(Utf8PathBuf::from("other_folder")) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs index 6dcae76c9354..512c231990cb 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs @@ -38,8 +38,11 @@ pub(crate) enum InvocationStrategy { PerWorkspace, } +/// Data needed to construct a `cargo` command invocation, e.g. for flycheck or running a test. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct CargoOptions { + /// The cargo subcommand to run, e.g. "check" or "clippy" + pub(crate) subcommand: String, pub(crate) target_tuples: Vec, pub(crate) all_targets: bool, pub(crate) set_test: bool, @@ -111,11 +114,16 @@ impl FlycheckConfigJson { /// #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum FlycheckConfig { - CargoCommand { - command: String, - options: CargoOptions, + /// Automatically use rust-project.json's flycheck runnable or just use cargo (the common case) + /// + /// We can't have a variant for ProjectJson because that is configured on the fly during + /// discoverConfig. We only know what we can read at config time. + Automatic { + /// If we do use cargo, how to build the check command + cargo_options: CargoOptions, ansi_color_output: bool, }, + /// check_overrideCommand. This overrides both cargo and rust-project.json's flycheck runnable. CustomCommand { command: String, args: Vec, @@ -127,7 +135,7 @@ pub(crate) enum FlycheckConfig { impl FlycheckConfig { pub(crate) fn invocation_strategy(&self) -> InvocationStrategy { match self { - FlycheckConfig::CargoCommand { .. } => InvocationStrategy::PerWorkspace, + FlycheckConfig::Automatic { .. } => InvocationStrategy::PerWorkspace, FlycheckConfig::CustomCommand { invocation_strategy, .. } => { invocation_strategy.clone() } @@ -138,7 +146,9 @@ impl FlycheckConfig { impl fmt::Display for FlycheckConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - FlycheckConfig::CargoCommand { command, .. } => write!(f, "cargo {command}"), + FlycheckConfig::Automatic { cargo_options, .. } => { + write!(f, "cargo {}", cargo_options.subcommand) + } FlycheckConfig::CustomCommand { command, args, .. } => { // Don't show `my_custom_check --foo $saved_file` literally to the user, as it // looks like we've forgotten to substitute $saved_file. @@ -572,11 +582,11 @@ impl FlycheckActor { CargoCheckParser, sender, match &self.config { - FlycheckConfig::CargoCommand { options, .. } => { + FlycheckConfig::Automatic { cargo_options, .. } => { let ws_target_dir = self.ws_target_dir.as_ref().map(Utf8PathBuf::as_path); let target_dir = - options.target_dir_config.target_dir(ws_target_dir); + cargo_options.target_dir_config.target_dir(ws_target_dir); // If `"rust-analyzer.cargo.targetDir": null`, we should use // workspace's target dir instead of hard-coded fallback. @@ -818,7 +828,7 @@ impl FlycheckActor { target: Option, ) -> Option<(Command, FlycheckCommandOrigin)> { match &self.config { - FlycheckConfig::CargoCommand { command, options, ansi_color_output } => { + FlycheckConfig::Automatic { cargo_options, ansi_color_output } => { // Only use the rust-project.json's flycheck config when no check_overrideCommand // is configured. In the FlycheckConcig::CustomCommand branch we will still do // label substitution, but on the overrideCommand instead. @@ -835,15 +845,15 @@ impl FlycheckActor { } let mut cmd = - toolchain::command(Tool::Cargo.path(), &*self.root, &options.extra_env); + toolchain::command(Tool::Cargo.path(), &*self.root, &cargo_options.extra_env); if let Some(sysroot_root) = &self.sysroot_root - && !options.extra_env.contains_key("RUSTUP_TOOLCHAIN") + && !cargo_options.extra_env.contains_key("RUSTUP_TOOLCHAIN") && std::env::var_os("RUSTUP_TOOLCHAIN").is_none() { cmd.env("RUSTUP_TOOLCHAIN", AsRef::::as_ref(sysroot_root)); } cmd.env("CARGO_LOG", "cargo::core::compiler::fingerprint=info"); - cmd.arg(command); + cmd.arg(&cargo_options.subcommand); match scope { FlycheckScope::Workspace => cmd.arg("--workspace"), @@ -887,11 +897,11 @@ impl FlycheckActor { cmd.arg("--keep-going"); - options.apply_on_command( + cargo_options.apply_on_command( &mut cmd, self.ws_target_dir.as_ref().map(Utf8PathBuf::as_path), ); - cmd.args(&options.extra_args); + cmd.args(&cargo_options.extra_args); Some((cmd, FlycheckCommandOrigin::Cargo)) } FlycheckConfig::CustomCommand { command, args, extra_env, invocation_strategy } => { diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs index 7111a15d0246..f0020f9088e3 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs @@ -105,7 +105,7 @@ impl CargoTestHandle { let mut cmd = toolchain::command(Tool::Cargo.path(), root, &options.extra_env); cmd.env("RUSTC_BOOTSTRAP", "1"); cmd.arg("--color=always"); - cmd.arg("test"); + cmd.arg(&options.subcommand); // test, usually cmd.arg("--package"); cmd.arg(&test_target.package); From 2d581773fed793f9d62b190e56374065d37291d7 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 3 Dec 2025 12:03:05 +1100 Subject: [PATCH 0117/1843] Fix RunnableKind::Run label interpolation It was pretty useless without this. Previously: Parsing target pattern `{label}` Caused by: Invalid target name `{label}`. (...) Build ID: 6dab5942-d81c-4430-83b0-5ba523999050 Network: Up: 0B Down: 0B Command: run. Time elapsed: 0.3s BUILD FAILED * The terminal process "buck2 'run', '{label}'" terminated with exit code: 3. --- .../crates/rust-analyzer/src/target_spec.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs index 8452b6493e87..b8d9acc02a32 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs @@ -77,7 +77,16 @@ impl ProjectJsonTargetSpec { RunnableKind::Bin => { for runnable in &self.shell_runnables { if matches!(runnable.kind, project_model::project_json::RunnableKind::Run) { - return Some(runnable.clone()); + let mut runnable = runnable.clone(); + + let replaced_args: Vec<_> = runnable + .args + .iter() + .map(|arg| arg.replace("{label}", &self.label)) + .collect(); + runnable.args = replaced_args; + + return Some(runnable); } } From 71e2ded9fb196bc9b4a1736759f9465f5d4d9619 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 3 Dec 2025 13:32:03 +1100 Subject: [PATCH 0118/1843] doc: Update docs for runnables to include run/flycheck --- .../crates/project-model/src/project_json.rs | 3 +++ .../docs/book/src/non_cargo_based_projects.md | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs index 536f170e1192..6938010cbd70 100644 --- a/src/tools/rust-analyzer/crates/project-model/src/project_json.rs +++ b/src/tools/rust-analyzer/crates/project-model/src/project_json.rs @@ -357,9 +357,12 @@ pub enum RunnableKind { Check, /// Can run a binary. + /// May include {label} which will get the label from the `build` section of a crate. Run, /// Run a single test. + /// May include {label} which will get the label from the `build` section of a crate. + /// May include {test_id} which will get the test clicked on by the user. TestOne, /// Template for checking a target, emitting rustc JSON diagnostics. diff --git a/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md b/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md index e7df4a5d7668..d8be9a82d0c9 100644 --- a/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md +++ b/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md @@ -204,16 +204,25 @@ interface Runnable { args: string[]; /// The current working directory of the runnable. cwd: string; - /// Used to decide what code lens to offer. + /// Maps a runnable to a piece of rust-analyzer functionality. /// - /// `testOne`: This runnable will be used when the user clicks the 'Run Test' - /// CodeLens above a test. + /// - `testOne`: This runnable will be used when the user clicks the 'Run Test' + /// CodeLens above a test. + /// - `run`: This runnable will be used when the user clicks the 'Run' CodeLens + /// above a main function or triggers a run command. + /// - `flycheck`: This is run to provide check-on-save diagnostics when the user + /// saves a file. It must emit rustc JSON diagnostics that rust-analyzer can + /// parse. If this runnable is not specified, we may try to use `cargo check -p`. + /// This is only run for a single crate that the user saved a file in. The + /// {label} syntax is replaced with `BuildInfo::label`. + /// Alternatively, you may use `{saved_file}` and figure out which crate + /// to produce diagnostics for based on that. /// /// The args for testOne can contain two template strings: /// `{label}` and `{test_id}`. `{label}` will be replaced - /// with the `Build::label` and `{test_id}` will be replaced + /// with the `BuildInfo::label` and `{test_id}` will be replaced /// with the test name. - kind: 'testOne' | string; + kind: 'testOne' | 'run' | 'flycheck' | string; } ``` From 422597f76395060d968fcb5c9e7de311bf1dc9a4 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 3 Dec 2025 14:44:40 +1100 Subject: [PATCH 0119/1843] doc: make example for workspace.discoverConfig actually work rust-project requires {arg} these days. No good giving people bad information even if it's not crucial to documenting this. --- .../rust-analyzer/crates/rust-analyzer/src/config.rs | 9 +++++---- .../docs/book/src/configuration_generated.md | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs index c2f7ada8c8ca..2b7ade6c26ef 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs @@ -480,8 +480,8 @@ config_data! { /// Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`]. /// - /// [`DiscoverWorkspaceConfig`] also requires setting `progress_label` and `files_to_watch`. - /// `progress_label` is used for the title in progress indicators, whereas `files_to_watch` + /// [`DiscoverWorkspaceConfig`] also requires setting `progressLabel` and `filesToWatch`. + /// `progressLabel` is used for the title in progress indicators, whereas `filesToWatch` /// is used to determine which build system-specific files should be watched in order to /// reload rust-analyzer. /// @@ -490,9 +490,10 @@ config_data! { /// "rust-analyzer.workspace.discoverConfig": { /// "command": [ /// "rust-project", - /// "develop-json" + /// "develop-json", + /// "{arg}" /// ], - /// "progressLabel": "rust-analyzer", + /// "progressLabel": "buck2/rust-project", /// "filesToWatch": [ /// "BUCK" /// ] diff --git a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md index 58b636334527..a0738ca0e179 100644 --- a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md +++ b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md @@ -1623,9 +1623,10 @@ Below is an example of a valid configuration: "rust-analyzer.workspace.discoverConfig": { "command": [ "rust-project", - "develop-json" + "develop-json", + "{arg}" ], - "progressLabel": "rust-analyzer", + "progressLabel": "buck2/rust-project", "filesToWatch": [ "BUCK" ] From f06a6b9fdcb9db492a364b48a08f70afa98da182 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 3 Dec 2025 17:11:16 +1100 Subject: [PATCH 0120/1843] doc: overhaul non-cargo build system docs --- .../crates/rust-analyzer/src/config.rs | 18 ++- .../docs/book/src/configuration_generated.md | 22 ++-- .../docs/book/src/non_cargo_based_projects.md | 105 +++++++++++++++--- .../rust-analyzer/editors/code/package.json | 4 +- 4 files changed, 120 insertions(+), 29 deletions(-) diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs index 2b7ade6c26ef..28ac94e4deb6 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs @@ -500,7 +500,7 @@ config_data! { /// } /// ``` /// - /// ## On `DiscoverWorkspaceConfig::command` + /// ## Workspace Discovery Protocol /// /// **Warning**: This format is provisional and subject to change. /// @@ -871,10 +871,18 @@ config_data! { /// (i.e., the folder containing the `Cargo.toml`). This can be overwritten /// by changing `#rust-analyzer.check.invocationStrategy#`. /// - /// If `$saved_file` is part of the command, rust-analyzer will pass - /// the absolute path of the saved file to the provided command. This is - /// intended to be used with non-Cargo build systems. - /// Note that `$saved_file` is experimental and may be removed in the future. + /// It supports two interpolation syntaxes, both mainly intended to be used with + /// [non-Cargo build systems](./non_cargo_based_projects.md): + /// + /// - If `{saved_file}` is part of the command, rust-analyzer will pass + /// the absolute path of the saved file to the provided command. + /// (A previous version, `$saved_file`, also works.) + /// - If `{label}` is part of the command, rust-analyzer will pass the + /// Cargo package ID, which can be used with `cargo check -p`, or a build label from + /// `rust-project.json`. If `{label}` is included, rust-analyzer behaves much like + /// [`"rust-analyzer.check.workspace": false`](#check.workspace). + /// + /// /// /// An example command would be: /// diff --git a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md index a0738ca0e179..c4124aaae075 100644 --- a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md +++ b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md @@ -323,10 +323,18 @@ each of them, with the working directory being the workspace root (i.e., the folder containing the `Cargo.toml`). This can be overwritten by changing `#rust-analyzer.check.invocationStrategy#`. -If `$saved_file` is part of the command, rust-analyzer will pass -the absolute path of the saved file to the provided command. This is -intended to be used with non-Cargo build systems. -Note that `$saved_file` is experimental and may be removed in the future. +It supports two interpolation syntaxes, both mainly intended to be used with +[non-Cargo build systems](./non_cargo_based_projects.md): + +- If `{saved_file}` is part of the command, rust-analyzer will pass + the absolute path of the saved file to the provided command. + (A previous version, `$saved_file`, also works.) +- If `{label}` is part of the command, rust-analyzer will pass the + Cargo package ID, which can be used with `cargo check -p`, or a build label from + `rust-project.json`. If `{label}` is included, rust-analyzer behaves much like + [`"rust-analyzer.check.workspace": false`](#check.workspace). + + An example command would be: @@ -1613,8 +1621,8 @@ Default: `null` Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`]. -[`DiscoverWorkspaceConfig`] also requires setting `progress_label` and `files_to_watch`. -`progress_label` is used for the title in progress indicators, whereas `files_to_watch` +[`DiscoverWorkspaceConfig`] also requires setting `progressLabel` and `filesToWatch`. +`progressLabel` is used for the title in progress indicators, whereas `filesToWatch` is used to determine which build system-specific files should be watched in order to reload rust-analyzer. @@ -1633,7 +1641,7 @@ Below is an example of a valid configuration: } ``` -## On `DiscoverWorkspaceConfig::command` +## Workspace Discovery Protocol **Warning**: This format is provisional and subject to change. diff --git a/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md b/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md index d8be9a82d0c9..a48b025c7b3a 100644 --- a/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md +++ b/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md @@ -229,7 +229,15 @@ interface Runnable { This format is provisional and subject to change. Specifically, the `roots` setup will be different eventually. -There are three ways to feed `rust-project.json` to rust-analyzer: +### Providing a JSON project to rust-analyzer + +There are four ways to feed `rust-project.json` to rust-analyzer: + +- Use + [`"rust-analyzer.workspace.discoverConfig": … }`](./configuration.md#workspace.discoverConfig) + to specify a workspace discovery command to generate project descriptions + on-the-fly. Please note that the command output is message-oriented and must + follow [the discovery protocol](./configuration.md#workspace-discovery-protocol). - Place `rust-project.json` file at the root of the project, and rust-analyzer will discover it. @@ -249,19 +257,86 @@ location or (for inline JSON) relative to `rootUri`. You can set the `RA_LOG` environment variable to `rust_analyzer=info` to inspect how rust-analyzer handles config and project loading. -Note that calls to `cargo check` are disabled when using -`rust-project.json` by default, so compilation errors and warnings will -no longer be sent to your LSP client. To enable these compilation errors -you will need to specify explicitly what command rust-analyzer should -run to perform the checks using the -`rust-analyzer.check.overrideCommand` configuration. As an example, the -following configuration explicitly sets `cargo check` as the `check` -command. +### Flycheck support - { "rust-analyzer.check.overrideCommand": ["cargo", "check", "--message-format=json"] } +Rust-analyzer has functionality to run an actual build of a crate when the user saves a file, to +fill in diagnostics it does not implement natively. This is known as "flycheck". -`check.overrideCommand` requires the command specified to output json -error messages for rust-analyzer to consume. The `--message-format=json` -flag does this for `cargo check` so whichever command you use must also -output errors in this format. See the [Configuration](#_configuration) -section for more information. +**Flycheck is disabled when using `rust-project.json` unless explicitly configured**, so compilation +errors and warnings will no longer be sent to your LSP client by default. To enable these +compilation errors you will need to specify explicitly what command rust-analyzer should run to +perform the checks. There are two ways to do this: + +- `rust-project.json` may contain a `runnables` field. The `flycheck` runnable may be used to + configure a check command. See above for documentation. + +- Using the [`rust-analyzer.check.overrideCommand`](./configuration.md#check.overrideCommand) + configuration. This will also override anything in `rust-project.json`. As an example, the + following configuration explicitly sets `cargo check` as the `check` command. + + ```json + { "rust-analyzer.check.overrideCommand": ["cargo", "check", "--message-format=json"] } + ``` + + Note also that this works with cargo projects. + +Either option requires the command specified to output JSON error messages for rust-analyzer to +consume. The `--message-format=json` flag does this for `cargo check` so whichever command you use +must also output errors in this format. + +Either option also supports two syntaxes within each argument: + +- `{label}` will be replaced with the `BuildInfo::label` of the crate + containing a saved file, if `BuildInfo` is provided. In the case of `check.overrideCommand` being + used in a Cargo project, this will be the cargo package ID, which can be used with `cargo check -p`. +- `{saved_file}` will be replaced with an absolute path to the saved file. This can be queried against a + build system to find targets that include the file. + +For example: + +```json +{ "rust-analyzer.check.overrideCommand": ["custom_crate_checker", "{label}"] } +``` + +If you do use `{label}` or `{saved_file}`, the command will not be run unless the relevant value can +be substituted. + + +#### Flycheck considerations + +##### Diagnostic output on error + +A flycheck command using a complex build orchestrator like `"bazel", "build", "{label}"`, even with +a tweak to return JSON messages, is often insufficient. Such a command will typically succeed if +there are warnings, but if there are errors, it might "fail to compile" the diagnostics and not +produce any output. You must build a package in such a way that the build succeeds even if `rustc` +exits with an error, and prints the JSON build messages in every case. + +##### Diagnostics for upstream crates + +`cargo check -p` re-prints any errors and warnings in crates higher up in the dependency graph +than the one requested. We do clear all diagnostics when flychecking, so if you manage to +replicate this behaviour, diagnostics for crates other than the one being checked will show up in +the editor. If you do not, then users may be confused that diagnostics are "stuck" or disappear +entirely when there is a build error in an upstream crate. + +##### Compiler options + +`cargo check` invokes rustc differently from `cargo build`. It turns off codegen (with `rustc +--emit=metadata`), which results in lower latency to get to diagnostics. If your build system can +configure this, it is recommended. + +If your build tool can configure rustc for incremental compiles, this is also recommended. + +##### Locking and pre-emption + +In any good build system, including Cargo, build commands sometimes block each other. Running a +flycheck will (by default) frequently block you from running other build commands. Generally this is +undesirable. Users will have to (unintuitively) press save again in the editor to cancel a +flycheck, so that some other command may proceed. + +If your build system has the ability to isolate any rust-analyzer-driven flychecks and prevent lock +contention, for example a separate build output directory and/or daemon instance, this is +recommended. Alternatively, consider using a feature if available that can set the priority of +various build invocations and automatically cancel lower-priority ones when needed. Flychecks should +be set to a lower priority than general direct build invocations. diff --git a/src/tools/rust-analyzer/editors/code/package.json b/src/tools/rust-analyzer/editors/code/package.json index 2157cbd48653..0d91378706a4 100644 --- a/src/tools/rust-analyzer/editors/code/package.json +++ b/src/tools/rust-analyzer/editors/code/package.json @@ -1213,7 +1213,7 @@ "title": "Check", "properties": { "rust-analyzer.check.overrideCommand": { - "markdownDescription": "Override the command rust-analyzer uses instead of `cargo check` for\ndiagnostics on save. The command is required to output json and\nshould therefore include `--message-format=json` or a similar option\n(if your client supports the `colorDiagnosticOutput` experimental\ncapability, you can use `--message-format=json-diagnostic-rendered-ansi`).\n\nIf you're changing this because you're using some tool wrapping\nCargo, you might also want to change\n`#rust-analyzer.cargo.buildScripts.overrideCommand#`.\n\nIf there are multiple linked projects/workspaces, this command is invoked for\neach of them, with the working directory being the workspace root\n(i.e., the folder containing the `Cargo.toml`). This can be overwritten\nby changing `#rust-analyzer.check.invocationStrategy#`.\n\nIf `$saved_file` is part of the command, rust-analyzer will pass\nthe absolute path of the saved file to the provided command. This is\nintended to be used with non-Cargo build systems.\nNote that `$saved_file` is experimental and may be removed in the future.\n\nAn example command would be:\n\n```bash\ncargo check --workspace --message-format=json --all-targets\n```\n\nNote: The option must be specified as an array of command line arguments, with\nthe first argument being the name of the command to run.", + "markdownDescription": "Override the command rust-analyzer uses instead of `cargo check` for\ndiagnostics on save. The command is required to output json and\nshould therefore include `--message-format=json` or a similar option\n(if your client supports the `colorDiagnosticOutput` experimental\ncapability, you can use `--message-format=json-diagnostic-rendered-ansi`).\n\nIf you're changing this because you're using some tool wrapping\nCargo, you might also want to change\n`#rust-analyzer.cargo.buildScripts.overrideCommand#`.\n\nIf there are multiple linked projects/workspaces, this command is invoked for\neach of them, with the working directory being the workspace root\n(i.e., the folder containing the `Cargo.toml`). This can be overwritten\nby changing `#rust-analyzer.check.invocationStrategy#`.\n\nIt supports two interpolation syntaxes, both mainly intended to be used with\n[non-Cargo build systems](./non_cargo_based_projects.md):\n\n- If `{saved_file}` is part of the command, rust-analyzer will pass\n the absolute path of the saved file to the provided command.\n (A previous version, `$saved_file`, also works.)\n- If `{label}` is part of the command, rust-analyzer will pass the\n Cargo package ID, which can be used with `cargo check -p`, or a build label from\n `rust-project.json`. If `{label}` is included, rust-analyzer behaves much like\n [`\"rust-analyzer.check.workspace\": false`](#check.workspace).\n\n\n\nAn example command would be:\n\n```bash\ncargo check --workspace --message-format=json --all-targets\n```\n\nNote: The option must be specified as an array of command line arguments, with\nthe first argument being the name of the command to run.", "default": null, "type": [ "null", @@ -3135,7 +3135,7 @@ "title": "Workspace", "properties": { "rust-analyzer.workspace.discoverConfig": { - "markdownDescription": "Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].\n\n[`DiscoverWorkspaceConfig`] also requires setting `progress_label` and `files_to_watch`.\n`progress_label` is used for the title in progress indicators, whereas `files_to_watch`\nis used to determine which build system-specific files should be watched in order to\nreload rust-analyzer.\n\nBelow is an example of a valid configuration:\n```json\n\"rust-analyzer.workspace.discoverConfig\": {\n \"command\": [\n \"rust-project\",\n \"develop-json\"\n ],\n \"progressLabel\": \"rust-analyzer\",\n \"filesToWatch\": [\n \"BUCK\"\n ]\n}\n```\n\n## On `DiscoverWorkspaceConfig::command`\n\n**Warning**: This format is provisional and subject to change.\n\n[`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to\n`DiscoverProjectData::Finished`:\n\n```norun\n#[derive(Debug, Clone, Deserialize, Serialize)]\n#[serde(tag = \"kind\")]\n#[serde(rename_all = \"snake_case\")]\nenum DiscoverProjectData {\n Finished { buildfile: Utf8PathBuf, project: ProjectJsonData },\n Error { error: String, source: Option },\n Progress { message: String },\n}\n```\n\nAs JSON, `DiscoverProjectData::Finished` is:\n\n```json\n{\n // the internally-tagged representation of the enum.\n \"kind\": \"finished\",\n // the file used by a non-Cargo build system to define\n // a package or target.\n \"buildfile\": \"rust-analyzer/BUILD\",\n // the contents of a rust-project.json, elided for brevity\n \"project\": {\n \"sysroot\": \"foo\",\n \"crates\": []\n }\n}\n```\n\nIt is encouraged, but not required, to use the other variants on `DiscoverProjectData`\nto provide a more polished end-user experience.\n\n`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be\nsubstituted with the JSON-serialized form of the following enum:\n\n```norun\n#[derive(PartialEq, Clone, Debug, Serialize)]\n#[serde(rename_all = \"camelCase\")]\npub enum DiscoverArgument {\n Path(AbsPathBuf),\n Buildfile(AbsPathBuf),\n}\n```\n\nThe JSON representation of `DiscoverArgument::Path` is:\n\n```json\n{\n \"path\": \"src/main.rs\"\n}\n```\n\nSimilarly, the JSON representation of `DiscoverArgument::Buildfile` is:\n\n```json\n{\n \"buildfile\": \"BUILD\"\n}\n```\n\n`DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and\ntherefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an\nexisting workspace. As a reference for implementors, buck2's `rust-project` will likely\nbe useful: .", + "markdownDescription": "Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].\n\n[`DiscoverWorkspaceConfig`] also requires setting `progressLabel` and `filesToWatch`.\n`progressLabel` is used for the title in progress indicators, whereas `filesToWatch`\nis used to determine which build system-specific files should be watched in order to\nreload rust-analyzer.\n\nBelow is an example of a valid configuration:\n```json\n\"rust-analyzer.workspace.discoverConfig\": {\n \"command\": [\n \"rust-project\",\n \"develop-json\",\n \"{arg}\"\n ],\n \"progressLabel\": \"buck2/rust-project\",\n \"filesToWatch\": [\n \"BUCK\"\n ]\n}\n```\n\n## Workspace Discovery Protocol\n\n**Warning**: This format is provisional and subject to change.\n\n[`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to\n`DiscoverProjectData::Finished`:\n\n```norun\n#[derive(Debug, Clone, Deserialize, Serialize)]\n#[serde(tag = \"kind\")]\n#[serde(rename_all = \"snake_case\")]\nenum DiscoverProjectData {\n Finished { buildfile: Utf8PathBuf, project: ProjectJsonData },\n Error { error: String, source: Option },\n Progress { message: String },\n}\n```\n\nAs JSON, `DiscoverProjectData::Finished` is:\n\n```json\n{\n // the internally-tagged representation of the enum.\n \"kind\": \"finished\",\n // the file used by a non-Cargo build system to define\n // a package or target.\n \"buildfile\": \"rust-analyzer/BUILD\",\n // the contents of a rust-project.json, elided for brevity\n \"project\": {\n \"sysroot\": \"foo\",\n \"crates\": []\n }\n}\n```\n\nIt is encouraged, but not required, to use the other variants on `DiscoverProjectData`\nto provide a more polished end-user experience.\n\n`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be\nsubstituted with the JSON-serialized form of the following enum:\n\n```norun\n#[derive(PartialEq, Clone, Debug, Serialize)]\n#[serde(rename_all = \"camelCase\")]\npub enum DiscoverArgument {\n Path(AbsPathBuf),\n Buildfile(AbsPathBuf),\n}\n```\n\nThe JSON representation of `DiscoverArgument::Path` is:\n\n```json\n{\n \"path\": \"src/main.rs\"\n}\n```\n\nSimilarly, the JSON representation of `DiscoverArgument::Buildfile` is:\n\n```json\n{\n \"buildfile\": \"BUILD\"\n}\n```\n\n`DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and\ntherefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an\nexisting workspace. As a reference for implementors, buck2's `rust-project` will likely\nbe useful: .", "default": null, "anyOf": [ { From b02e9756f2d0b0367cdedcba16016fc5e867999c Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Thu, 8 Jan 2026 09:53:00 +1100 Subject: [PATCH 0121/1843] Fix hir-ty clippy issue I am not familiar with this code at allso just doing what I can to unblock. --- .../crates/hir-ty/src/next_solver/infer/traits.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/traits.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/traits.rs index 14df42dc2aeb..dde623483642 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/traits.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/traits.rs @@ -55,6 +55,13 @@ impl ObligationCause { } } +impl Default for ObligationCause { + #[inline] + fn default() -> Self { + Self::new() + } +} + /// An `Obligation` represents some trait reference (e.g., `i32: Eq`) for /// which the "impl_source" must be found. The process of finding an "impl_source" is /// called "resolving" the `Obligation`. This process consists of From fb9b6bd5351c3eb4f62da83c35ca1636a3523a60 Mon Sep 17 00:00:00 2001 From: hulxv Date: Wed, 17 Dec 2025 22:57:32 +0200 Subject: [PATCH 0122/1843] Refactor socketpair tests to use utility functions for reading and writing inline byte slices for data writes Refactor socketpair tests to use utility functions for reading and writing --- .../tests/pass-dep/libc/libc-socketpair.rs | 189 ++++++------------ 1 file changed, 61 insertions(+), 128 deletions(-) diff --git a/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs b/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs index ce3927ce48ca..2031149aaf4f 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs @@ -9,6 +9,7 @@ use std::thread; #[path = "../../utils/libc.rs"] mod libc_utils; +use libc_utils::*; fn main() { test_socketpair(); @@ -21,139 +22,92 @@ fn main() { fn test_socketpair() { let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Read size == data available in buffer. - let data = "abcde".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); - let mut buf: [u8; 5] = [0; 5]; - let res = - unsafe { libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) }; - assert_eq!(res, 5); - assert_eq!(buf, "abcde".as_bytes()); + let data = b"abcde"; + write_all_from_slice(fds[0], data).unwrap(); + let buf = read_all_into_array::<5>(fds[1]).unwrap(); + assert_eq!(&buf, data); // Read size > data available in buffer. - let data = "abc".as_bytes(); - let res = unsafe { libc_utils::write_all(fds[0], data.as_ptr() as *const libc::c_void, 3) }; - assert_eq!(res, 3); + let data = b"abc"; + write_all_from_slice(fds[0], data).unwrap(); let mut buf2: [u8; 5] = [0; 5]; - let res = unsafe { libc::read(fds[1], buf2.as_mut_ptr().cast(), buf2.len() as libc::size_t) }; - assert!(res > 0 && res <= 3); - let res = res as usize; - assert_eq!(buf2[..res], data[..res]); - if res < 3 { - // Drain the rest from the read end. - let res = unsafe { libc_utils::read_all(fds[1], buf2[res..].as_mut_ptr().cast(), 3 - res) }; - assert!(res > 0); - } + let (read, rest) = read_into_slice(fds[1], &mut buf2).unwrap(); + assert_eq!(read[..], data[..read.len()]); + // Write 2 more bytes so we can exactly fill the `rest`. + write_all_from_slice(fds[0], b"12").unwrap(); + read_all_into_slice(fds[1], rest).unwrap(); // Test read and write from another direction. // Read size == data available in buffer. - let data = "12345".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); - let mut buf3: [u8; 5] = [0; 5]; - let res = unsafe { - libc_utils::read_all(fds[0], buf3.as_mut_ptr().cast(), buf3.len() as libc::size_t) - }; - assert_eq!(res, 5); - assert_eq!(buf3, "12345".as_bytes()); + let data = b"12345"; + write_all_from_slice(fds[1], data).unwrap(); + let buf3 = read_all_into_array::<5>(fds[0]).unwrap(); + assert_eq!(&buf3, data); // Read size > data available in buffer. - let data = "123".as_bytes(); - let res = unsafe { libc_utils::write_all(fds[1], data.as_ptr() as *const libc::c_void, 3) }; - assert_eq!(res, 3); + let data = b"123"; + write_all_from_slice(fds[1], data).unwrap(); let mut buf4: [u8; 5] = [0; 5]; - let res = unsafe { libc::read(fds[0], buf4.as_mut_ptr().cast(), buf4.len() as libc::size_t) }; - assert!(res > 0 && res <= 3); - let res = res as usize; - assert_eq!(buf4[..res], data[..res]); - if res < 3 { - // Drain the rest from the read end. - let res = unsafe { libc_utils::read_all(fds[0], buf4[res..].as_mut_ptr().cast(), 3 - res) }; - assert!(res > 0); - } + let (read, rest) = read_into_slice(fds[0], &mut buf4).unwrap(); + assert_eq!(read[..], data[..read.len()]); + // Write 2 more bytes so we can exactly fill the `rest`. + write_all_from_slice(fds[1], b"12").unwrap(); + read_all_into_slice(fds[0], rest).unwrap(); // Test when happens when we close one end, with some data in the buffer. - let res = unsafe { libc_utils::write_all(fds[0], data.as_ptr() as *const libc::c_void, 3) }; - assert_eq!(res, 3); - unsafe { libc::close(fds[0]) }; + write_all_from_slice(fds[0], data).unwrap(); + errno_check(unsafe { libc::close(fds[0]) }); // Reading the other end should return that data, then EOF. let mut buf: [u8; 5] = [0; 5]; - let res = - unsafe { libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) }; - assert_eq!(res, 3); - assert_eq!(&buf[0..3], "123".as_bytes()); - let res = - unsafe { libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) }; + read_all_into_slice(fds[1], &mut buf[0..3]).unwrap(); + assert_eq!(&buf[0..3], data); + let res = read_into_slice(fds[1], &mut buf[3..5]).unwrap().0.len(); assert_eq!(res, 0); // 0-sized read: EOF. // Writing the other end should emit EPIPE. - let res = unsafe { libc_utils::write_all(fds[1], data.as_ptr() as *const libc::c_void, 1) }; - assert_eq!(res, -1); + let res = write_all_from_slice(fds[1], &mut buf); + assert_eq!(res, Err(-1)); assert_eq!(std::io::Error::last_os_error().raw_os_error(), Some(libc::EPIPE)); } fn test_socketpair_threaded() { let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); let thread1 = thread::spawn(move || { - let mut buf: [u8; 5] = [0; 5]; - let res: i64 = unsafe { - libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) - .try_into() - .unwrap() - }; - assert_eq!(res, 5); - assert_eq!(buf, "abcde".as_bytes()); + let buf = read_all_into_array::<5>(fds[1]).unwrap(); + assert_eq!(&buf, b"abcde"); }); thread::yield_now(); - let data = "abcde".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); + write_all_from_slice(fds[0], b"abcde").unwrap(); thread1.join().unwrap(); // Read and write from different direction let thread2 = thread::spawn(move || { thread::yield_now(); - let data = "12345".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); + write_all_from_slice(fds[1], b"12345").unwrap(); }); - let mut buf: [u8; 5] = [0; 5]; - let res = - unsafe { libc_utils::read_all(fds[0], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) }; - assert_eq!(res, 5); - assert_eq!(buf, "12345".as_bytes()); + let buf = read_all_into_array::<5>(fds[0]).unwrap(); + assert_eq!(&buf, b"12345"); thread2.join().unwrap(); } fn test_race() { static mut VAL: u8 = 0; let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); let thread1 = thread::spawn(move || { - let mut buf: [u8; 1] = [0; 1]; // write() from the main thread will occur before the read() here // because preemption is disabled and the main thread yields after write(). - let res: i32 = unsafe { - libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) - .try_into() - .unwrap() - }; - assert_eq!(res, 1); - assert_eq!(buf, "a".as_bytes()); + let buf = read_all_into_array::<1>(fds[1]).unwrap(); + assert_eq!(&buf, b"a"); // The read above establishes a happens-before so it is now safe to access this global variable. unsafe { assert_eq!(VAL, 1) }; }); unsafe { VAL = 1 }; - let data = "a".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 1) }; - assert_eq!(res, 1); + write_all_from_slice(fds[0], b"a").unwrap(); thread::yield_now(); thread1.join().unwrap(); } @@ -161,22 +115,15 @@ fn test_race() { // Test the behaviour of a socketpair getting blocked on read and subsequently unblocked. fn test_blocking_read() { let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); let thread1 = thread::spawn(move || { // Let this thread block on read. - let mut buf: [u8; 3] = [0; 3]; - let res = unsafe { - libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) - }; - assert_eq!(res, 3); - assert_eq!(&buf, "abc".as_bytes()); + let buf = read_all_into_array::<3>(fds[1]).unwrap(); + assert_eq!(&buf, b"abc"); }); let thread2 = thread::spawn(move || { // Unblock thread1 by doing writing something. - let data = "abc".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 3) }; - assert_eq!(res, 3); + write_all_from_slice(fds[0], b"abc").unwrap(); }); thread1.join().unwrap(); thread2.join().unwrap(); @@ -185,26 +132,17 @@ fn test_blocking_read() { // Test the behaviour of a socketpair getting blocked on write and subsequently unblocked. fn test_blocking_write() { let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); let arr1: [u8; 0x34000] = [1; 0x34000]; // Exhaust the space in the buffer so the subsequent write will block. - let res = - unsafe { libc_utils::write_all(fds[0], arr1.as_ptr() as *const libc::c_void, arr1.len()) }; - assert_eq!(res, 0x34000); + write_all_from_slice(fds[0], &arr1).unwrap(); let thread1 = thread::spawn(move || { - let data = "abc".as_bytes().as_ptr(); // The write below will be blocked because the buffer is already full. - let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 3) }; - assert_eq!(res, 3); + write_all_from_slice(fds[0], b"abc").unwrap(); }); let thread2 = thread::spawn(move || { // Unblock thread1 by freeing up some space. - let mut buf: [u8; 3] = [0; 3]; - let res = unsafe { - libc_utils::read_all(fds[1], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) - }; - assert_eq!(res, 3); + let buf = read_all_into_array::<3>(fds[1]).unwrap(); assert_eq!(buf, [1, 1, 1]); }); thread1.join().unwrap(); @@ -215,30 +153,25 @@ fn test_blocking_write() { fn test_socketpair_setfl_getfl() { // Initialise socketpair fds. let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Test if both sides have O_RDWR. - let res = unsafe { libc::fcntl(fds[0], libc::F_GETFL) }; - assert_eq!(res, libc::O_RDWR); - let res = unsafe { libc::fcntl(fds[1], libc::F_GETFL) }; - assert_eq!(res, libc::O_RDWR); + assert_eq!(errno_result(unsafe { libc::fcntl(fds[0], libc::F_GETFL) }).unwrap(), libc::O_RDWR); + assert_eq!(errno_result(unsafe { libc::fcntl(fds[1], libc::F_GETFL) }).unwrap(), libc::O_RDWR); // Add the O_NONBLOCK flag with F_SETFL. - let res = unsafe { libc::fcntl(fds[0], libc::F_SETFL, libc::O_NONBLOCK) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::fcntl(fds[0], libc::F_SETFL, libc::O_NONBLOCK) }); // Test if the O_NONBLOCK flag is successfully added. - let res = unsafe { libc::fcntl(fds[0], libc::F_GETFL) }; - assert_eq!(res, libc::O_RDWR | libc::O_NONBLOCK); + assert_eq!( + errno_result(unsafe { libc::fcntl(fds[0], libc::F_GETFL) }).unwrap(), + libc::O_RDWR | libc::O_NONBLOCK + ); // The other side remains unchanged. - let res = unsafe { libc::fcntl(fds[1], libc::F_GETFL) }; - assert_eq!(res, libc::O_RDWR); + assert_eq!(errno_result(unsafe { libc::fcntl(fds[1], libc::F_GETFL) }).unwrap(), libc::O_RDWR); // Test if O_NONBLOCK flag can be unset. - let res = unsafe { libc::fcntl(fds[0], libc::F_SETFL, 0) }; - assert_eq!(res, 0); - let res = unsafe { libc::fcntl(fds[0], libc::F_GETFL) }; - assert_eq!(res, libc::O_RDWR); + errno_check(unsafe { libc::fcntl(fds[0], libc::F_SETFL, 0) }); + assert_eq!(errno_result(unsafe { libc::fcntl(fds[0], libc::F_GETFL) }).unwrap(), libc::O_RDWR); } From ffaf76a7cfbf29968e6a33387fc96aa7b17a88fc Mon Sep 17 00:00:00 2001 From: hulxv Date: Wed, 17 Dec 2025 23:05:49 +0200 Subject: [PATCH 0123/1843] Refactor epoll tests to use errno_result and improve notification checks --- .../pass-dep/libc/libc-epoll-blocking.rs | 162 ++++++------------ .../pass-dep/libc/libc-epoll-no-blocking.rs | 20 +-- src/tools/miri/tests/utils/libc.rs | 11 +- 3 files changed, 75 insertions(+), 118 deletions(-) diff --git a/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs b/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs index c67386b4f84c..d5a59796de02 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs @@ -7,6 +7,8 @@ use std::thread; #[path = "../../utils/libc.rs"] mod libc_utils; +use libc_utils::epoll::*; +use libc_utils::*; // This is a set of testcases for blocking epoll. @@ -20,47 +22,22 @@ fn main() { } // Using `as` cast since `EPOLLET` wraps around -const EPOLL_IN_OUT_ET: u32 = (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _; - -#[track_caller] -fn check_epoll_wait( - epfd: i32, - expected_notifications: &[(u32, u64)], - timeout: i32, -) { - let epoll_event = libc::epoll_event { events: 0, u64: 0 }; - let mut array: [libc::epoll_event; N] = [epoll_event; N]; - let maxsize = N; - let array_ptr = array.as_mut_ptr(); - let res = unsafe { libc::epoll_wait(epfd, array_ptr, maxsize.try_into().unwrap(), timeout) }; - if res < 0 { - panic!("epoll_wait failed: {}", std::io::Error::last_os_error()); - } - let got_notifications = - unsafe { std::slice::from_raw_parts(array_ptr, res.try_into().unwrap()) }; - let got_notifications = got_notifications.iter().map(|e| (e.events, e.u64)).collect::>(); - assert_eq!(got_notifications, expected_notifications, "got wrong notifications"); -} +const EPOLL_IN_OUT_ET: i32 = (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _; // This test allows epoll_wait to block, then unblock without notification. fn test_epoll_block_without_notification() { // Create an epoll instance. - let epfd = unsafe { libc::epoll_create1(0) }; - assert_ne!(epfd, -1); + let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap(); // Create an eventfd instances. let flags = libc::EFD_NONBLOCK | libc::EFD_CLOEXEC; - let fd = unsafe { libc::eventfd(0, flags) }; + let fd = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap(); // Register eventfd with epoll. - let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd as u64 }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) }; - assert_eq!(res, 0); + epoll_ctl_add(epfd, fd, EPOLL_IN_OUT_ET).unwrap(); // epoll_wait to clear notification. - let expected_event = u32::try_from(libc::EPOLLOUT).unwrap(); - let expected_value = fd as u64; - check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 0); + check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT as _, data: fd }], 0); // This epoll wait blocks, and timeout without notification. check_epoll_wait::<1>(epfd, &[], 5); @@ -69,102 +46,86 @@ fn test_epoll_block_without_notification() { // This test triggers notification and unblocks the epoll_wait before timeout. fn test_epoll_block_then_unblock() { // Create an epoll instance. - let epfd = unsafe { libc::epoll_create1(0) }; - assert_ne!(epfd, -1); + let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap(); // Create a socketpair instance. let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Register one side of the socketpair with epoll. - let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) }; - assert_eq!(res, 0); + epoll_ctl_add(epfd, fds[0], EPOLL_IN_OUT_ET).unwrap(); // epoll_wait to clear notification. - let expected_event = u32::try_from(libc::EPOLLOUT).unwrap(); - let expected_value = fds[0] as u64; - check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 0); + check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT as _, data: fds[0] }], 0); // epoll_wait before triggering notification so it will block then get unblocked before timeout. - let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap(); - let expected_value = fds[0] as u64; let thread1 = thread::spawn(move || { thread::yield_now(); - let data = "abcde".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); + write_all_from_slice(fds[1], b"abcde").unwrap(); }); - check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 10); + check_epoll_wait::<1>( + epfd, + &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fds[0] }], + 10, + ); thread1.join().unwrap(); } // This test triggers a notification after epoll_wait times out. fn test_notification_after_timeout() { // Create an epoll instance. - let epfd = unsafe { libc::epoll_create1(0) }; - assert_ne!(epfd, -1); + let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap(); // Create a socketpair instance. let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Register one side of the socketpair with epoll. - let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) }; - assert_eq!(res, 0); + epoll_ctl_add(epfd, fds[0], EPOLL_IN_OUT_ET).unwrap(); // epoll_wait to clear notification. - let expected_event = u32::try_from(libc::EPOLLOUT).unwrap(); - let expected_value = fds[0] as u64; - check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 0); + check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT as _, data: fds[0] }], 0); // epoll_wait timeouts without notification. check_epoll_wait::<1>(epfd, &[], 10); // Trigger epoll notification after timeout. - let data = "abcde".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); + write_all_from_slice(fds[1], b"abcde").unwrap(); // Check the result of the notification. - let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap(); - let expected_value = fds[0] as u64; - check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 10); + check_epoll_wait::<1>( + epfd, + &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fds[0] }], + 10, + ); } // This test shows a data_race before epoll had vector clocks added. fn test_epoll_race() { // Create an epoll instance. - let epfd = unsafe { libc::epoll_create1(0) }; - assert_ne!(epfd, -1); + let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap(); // Create an eventfd instance. let flags = libc::EFD_NONBLOCK | libc::EFD_CLOEXEC; - let fd = unsafe { libc::eventfd(0, flags) }; + let fd = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap(); // Register eventfd with the epoll instance. - let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd as u64 }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) }; - assert_eq!(res, 0); + epoll_ctl_add(epfd, fd, EPOLL_IN_OUT_ET).unwrap(); static mut VAL: u8 = 0; let thread1 = thread::spawn(move || { // Write to the static mut variable. unsafe { VAL = 1 }; // Write to the eventfd instance. - let sized_8_data: [u8; 8] = 1_u64.to_ne_bytes(); - let res = unsafe { libc::write(fd, sized_8_data.as_ptr() as *const libc::c_void, 8) }; - // write returns number of bytes written, which is always 8. - assert_eq!(res, 8); + write_all_from_slice(fd, &1_u64.to_ne_bytes()).unwrap(); }); thread::yield_now(); // epoll_wait for the event to happen. - let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap(); - let expected_value = u64::try_from(fd).unwrap(); - check_epoll_wait::<8>(epfd, &[(expected_event, expected_value)], -1); + check_epoll_wait::<8>( + epfd, + &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fd }], + -1, + ); // Read from the static mut variable. #[allow(static_mut_refs)] unsafe { @@ -177,35 +138,28 @@ fn test_epoll_race() { /// epoll it is blocked on. fn wakeup_on_new_interest() { // Create an epoll instance. - let epfd = unsafe { libc::epoll_create1(0) }; - assert_ne!(epfd, -1); + let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap(); // Create a socketpair instance. let mut fds = [-1, -1]; - let res = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; - assert_eq!(res, 0); + errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Write to fd[0] - let data = "abcde".as_bytes().as_ptr(); - let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; - assert_eq!(res, 5); + write_all_from_slice(fds[0], b"abcde").unwrap(); // Block a thread on the epoll instance. let t = std::thread::spawn(move || { - let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap(); - let expected_value = u64::try_from(fds[1]).unwrap(); - check_epoll_wait::<8>(epfd, &[(expected_event, expected_value)], -1); + check_epoll_wait::<8>( + epfd, + &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fds[1] }], + -1, + ); }); // Ensure the thread is blocked. std::thread::yield_now(); // Register fd[1] with EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP - let mut ev = libc::epoll_event { - events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET | libc::EPOLLRDHUP) as _, - u64: u64::try_from(fds[1]).unwrap(), - }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[1], &mut ev) }; - assert_eq!(res, 0); + epoll_ctl_add(epfd, fds[1], EPOLL_IN_OUT_ET | libc::EPOLLRDHUP as i32).unwrap(); // This should wake up the thread. t.join().unwrap(); @@ -215,25 +169,23 @@ fn wakeup_on_new_interest() { /// to consume them all. fn multiple_events_wake_multiple_threads() { // Create an epoll instance. - let epfd = unsafe { libc::epoll_create1(0) }; - assert_ne!(epfd, -1); + let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap(); // Create an eventfd instance. let flags = libc::EFD_NONBLOCK | libc::EFD_CLOEXEC; - let fd1 = unsafe { libc::eventfd(0, flags) }; + let fd1 = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap(); // Make a duplicate so that we have two file descriptors for the same file description. - let fd2 = unsafe { libc::dup(fd1) }; + let fd2 = errno_result(unsafe { libc::dup(fd1) }).unwrap(); // Register both with epoll. - let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd1 as u64 }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd1, &mut ev) }; - assert_eq!(res, 0); - let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd2 as u64 }; - let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd2, &mut ev) }; - assert_eq!(res, 0); + epoll_ctl_add(epfd, fd1, EPOLL_IN_OUT_ET).unwrap(); + epoll_ctl_add(epfd, fd2, EPOLL_IN_OUT_ET).unwrap(); // Consume the initial events. - let expected = [(libc::EPOLLOUT as u32, fd1 as u64), (libc::EPOLLOUT as u32, fd2 as u64)]; + let expected = [ + Ev { events: libc::EPOLLOUT as _, data: fd1 }, + Ev { events: libc::EPOLLOUT as _, data: fd2 }, + ]; check_epoll_wait::<8>(epfd, &expected, -1); // Block two threads on the epoll, both wanting to get just one event. @@ -241,19 +193,19 @@ fn multiple_events_wake_multiple_threads() { let mut e = libc::epoll_event { events: 0, u64: 0 }; let res = unsafe { libc::epoll_wait(epfd, &raw mut e, 1, -1) }; assert!(res == 1); - (e.events, e.u64) + Ev { events: e.events.cast_signed(), data: e.u64.try_into().unwrap() } }); let t2 = thread::spawn(move || { let mut e = libc::epoll_event { events: 0, u64: 0 }; let res = unsafe { libc::epoll_wait(epfd, &raw mut e, 1, -1) }; assert!(res == 1); - (e.events, e.u64) + Ev { events: e.events.cast_signed(), data: e.u64.try_into().unwrap() } }); // Yield so both threads are waiting now. thread::yield_now(); // Trigger the eventfd. This triggers two events at once! - libc_utils::write_all_from_slice(fd1, &0_u64.to_ne_bytes()).unwrap(); + write_all_from_slice(fd1, &0_u64.to_ne_bytes()).unwrap(); // Both threads should have been woken up so that both events can be consumed. let e1 = t1.join().unwrap(); diff --git a/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs b/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs index c2789eb2f6c6..490895c8541b 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs @@ -60,7 +60,7 @@ fn test_epoll_socketpair() { errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Write to fd[0] - write_all_from_slice(fds[0], "abcde".as_bytes()).unwrap(); + write_all_from_slice(fds[0], b"abcde").unwrap(); // Register fd[1] with EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP epoll_ctl_add(epfd, fds[1], EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP).unwrap(); @@ -72,7 +72,7 @@ fn test_epoll_socketpair() { check_epoll_wait_noblock::<8>(epfd, &[]); // Write some more to fd[0]. - write_all_from_slice(fds[0], "abcde".as_bytes()).unwrap(); + write_all_from_slice(fds[0], b"abcde").unwrap(); // This did not change the readiness of fd[1], so we should get no event. // However, Linux seems to always deliver spurious events to the peer on each write, @@ -140,7 +140,7 @@ fn test_epoll_ctl_del() { errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Write to fd[0] - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; assert_eq!(res, 5); @@ -168,7 +168,7 @@ fn test_two_epoll_instance() { errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Write to the socketpair. - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; assert_eq!(res, 5); @@ -208,7 +208,7 @@ fn test_two_same_fd_in_same_epoll_instance() { assert_eq!(res, 0); // Write to the socketpair. - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; assert_eq!(res, 5); @@ -288,7 +288,7 @@ fn test_epoll_socketpair_both_sides() { // Write to fds[1]. // (We do the write after the register here, unlike in `test_epoll_socketpair`, to ensure // we cover both orders in which this could be done.) - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5) }; assert_eq!(res, 5); @@ -307,7 +307,7 @@ fn test_epoll_socketpair_both_sides() { let res = unsafe { libc_utils::read_all(fds[0], buf.as_mut_ptr().cast(), buf.len() as libc::size_t) }; assert_eq!(res, 5); - assert_eq!(buf, "abcde".as_bytes()); + assert_eq!(buf, *b"abcde"); // The state of fds[1] does not change (was writable, is writable). // However, we force a spurious wakeup as the read buffer just got emptied. @@ -500,7 +500,7 @@ fn test_no_notification_for_unregister_flag() { assert_eq!(res, 0); // Write to fd[1]. - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res: i32 = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5).try_into().unwrap() }; @@ -534,7 +534,7 @@ fn test_socketpair_epollerr() { errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }); // Write to fd[0] - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res = unsafe { libc_utils::write_all(fds[0], data as *const libc::c_void, 5) }; assert_eq!(res, 5); @@ -747,7 +747,7 @@ fn test_issue_4374_reads() { assert_eq!(unsafe { libc::fcntl(fds[1], libc::F_SETFL, libc::O_NONBLOCK) }, 0); // Write to fds[1] so that fds[0] becomes readable. - let data = "abcde".as_bytes().as_ptr(); + let data = b"abcde".as_ptr(); let res: i32 = unsafe { libc_utils::write_all(fds[1], data as *const libc::c_void, 5).try_into().unwrap() }; diff --git a/src/tools/miri/tests/utils/libc.rs b/src/tools/miri/tests/utils/libc.rs index 0765bacb6bd8..4492f5080180 100644 --- a/src/tools/miri/tests/utils/libc.rs +++ b/src/tools/miri/tests/utils/libc.rs @@ -113,7 +113,7 @@ pub mod epoll { /// The libc epoll_event type doesn't fit to the EPOLLIN etc constants, so we have our /// own type. We also make the data field an int since we typically want to store FDs there. - #[derive(PartialEq, Debug)] + #[derive(PartialEq, Debug, Clone, Copy)] pub struct Ev { pub events: c_int, pub data: c_int, @@ -138,10 +138,10 @@ pub mod epoll { } #[track_caller] - pub fn check_epoll_wait_noblock(epfd: i32, expected: &[Ev]) { + pub fn check_epoll_wait(epfd: i32, expected: &[Ev], timeout: i32) { let mut array: [libc::epoll_event; N] = [libc::epoll_event { events: 0, u64: 0 }; N]; let num = errno_result(unsafe { - libc::epoll_wait(epfd, array.as_mut_ptr(), N.try_into().unwrap(), 0) + libc::epoll_wait(epfd, array.as_mut_ptr(), N.try_into().unwrap(), timeout) }) .expect("epoll_wait returned an error"); let got = &mut array[..num.try_into().unwrap()]; @@ -151,4 +151,9 @@ pub mod epoll { .collect::>(); assert_eq!(got, expected, "got wrong notifications"); } + + #[track_caller] + pub fn check_epoll_wait_noblock(epfd: i32, expected: &[Ev]) { + check_epoll_wait::(epfd, expected, 0); + } } From 6a9de224c463f54dc66dec6e3dbe3e244d2d7014 Mon Sep 17 00:00:00 2001 From: The rustc-josh-sync Cronjob Bot Date: Thu, 8 Jan 2026 04:20:55 +0000 Subject: [PATCH 0124/1843] Prepare for merging from rust-lang/rust This updates the rust-version file to 548e586795f6b6fe089d8329aa5edbf0f5202646. --- src/tools/rust-analyzer/rust-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/rust-version b/src/tools/rust-analyzer/rust-version index 5ffe95a0b54f..4b08b0884ca8 100644 --- a/src/tools/rust-analyzer/rust-version +++ b/src/tools/rust-analyzer/rust-version @@ -1 +1 @@ -e7d44143a12a526488e4f0c0d7ea8e62a4fe9354 +548e586795f6b6fe089d8329aa5edbf0f5202646 From 5d8a7daf2ab5d13330030880af021cf1cf418a7e Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Thu, 8 Jan 2026 09:25:28 +0200 Subject: [PATCH 0125/1843] Fixes for builtin derive expansions - Do not store the `MacroCallId` of the "real" expansion anywhere, so that the IDE layer could not expand it by mistake - Fix a stupid bug where we used the directive of the `derive` itself instead of of the macro, leading us to re-expand it again and again. --- .../crates/hir-def/src/builtin_derive.rs | 22 ++++- .../crates/hir-def/src/dyn_map.rs | 10 ++- .../crates/hir-def/src/item_scope.rs | 19 ++-- .../crates/hir-def/src/lang_item.rs | 54 ++++++++++-- .../crates/hir-def/src/nameres.rs | 10 ++- .../crates/hir-def/src/nameres/collector.rs | 88 ++++++++++++------- .../rust-analyzer/crates/hir/src/semantics.rs | 31 +++++-- .../crates/hir/src/semantics/source_to_def.rs | 17 ++-- .../crates/ide/src/expand_macro.rs | 60 +++++-------- .../crates/intern/src/symbol/symbols.rs | 1 + 10 files changed, 211 insertions(+), 101 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-def/src/builtin_derive.rs b/src/tools/rust-analyzer/crates/hir-def/src/builtin_derive.rs index 32385516ab58..946f08ec3682 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/builtin_derive.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/builtin_derive.rs @@ -8,7 +8,8 @@ use intern::{Symbol, sym}; use tt::TextRange; use crate::{ - AdtId, BuiltinDeriveImplId, BuiltinDeriveImplLoc, FunctionId, HasModule, db::DefDatabase, + AdtId, BuiltinDeriveImplId, BuiltinDeriveImplLoc, FunctionId, HasModule, MacroId, + db::DefDatabase, lang_item::LangItems, }; macro_rules! declare_enum { @@ -86,6 +87,25 @@ declare_enum!( DispatchFromDyn => [], ); +impl BuiltinDeriveImplTrait { + pub fn derive_macro(self, lang_items: &LangItems) -> Option { + match self { + BuiltinDeriveImplTrait::Copy => lang_items.CopyDerive, + BuiltinDeriveImplTrait::Clone => lang_items.CloneDerive, + BuiltinDeriveImplTrait::Default => lang_items.DefaultDerive, + BuiltinDeriveImplTrait::Debug => lang_items.DebugDerive, + BuiltinDeriveImplTrait::Hash => lang_items.HashDerive, + BuiltinDeriveImplTrait::Ord => lang_items.OrdDerive, + BuiltinDeriveImplTrait::PartialOrd => lang_items.PartialOrdDerive, + BuiltinDeriveImplTrait::Eq => lang_items.EqDerive, + BuiltinDeriveImplTrait::PartialEq => lang_items.PartialEqDerive, + BuiltinDeriveImplTrait::CoerceUnsized | BuiltinDeriveImplTrait::DispatchFromDyn => { + lang_items.CoercePointeeDerive + } + } + } +} + impl BuiltinDeriveImplMethod { pub fn trait_method( self, diff --git a/src/tools/rust-analyzer/crates/hir-def/src/dyn_map.rs b/src/tools/rust-analyzer/crates/hir-def/src/dyn_map.rs index 7d3a94b03833..4308d0ef1c29 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/dyn_map.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/dyn_map.rs @@ -27,14 +27,15 @@ pub mod keys { use std::marker::PhantomData; + use either::Either; use hir_expand::{MacroCallId, attrs::AttrId}; use rustc_hash::FxHashMap; use syntax::{AstNode, AstPtr, ast}; use crate::{ - BlockId, ConstId, EnumId, EnumVariantId, ExternBlockId, ExternCrateId, FieldId, FunctionId, - ImplId, LifetimeParamId, Macro2Id, MacroRulesId, ProcMacroId, StaticId, StructId, TraitId, - TypeAliasId, TypeOrConstParamId, UnionId, UseId, + BlockId, BuiltinDeriveImplId, ConstId, EnumId, EnumVariantId, ExternBlockId, ExternCrateId, + FieldId, FunctionId, ImplId, LifetimeParamId, Macro2Id, MacroRulesId, ProcMacroId, + StaticId, StructId, TraitId, TypeAliasId, TypeOrConstParamId, UnionId, UseId, dyn_map::{DynMap, Policy}, }; @@ -71,7 +72,8 @@ pub mod keys { ( AttrId, /* derive() */ MacroCallId, - /* actual derive macros */ Box<[Option]>, + /* actual derive macros */ + Box<[Option>]>, ), > = Key::new(); diff --git a/src/tools/rust-analyzer/crates/hir-def/src/item_scope.rs b/src/tools/rust-analyzer/crates/hir-def/src/item_scope.rs index a3278dd76c86..9e1efb977786 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/item_scope.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/item_scope.rs @@ -4,6 +4,7 @@ use std::{fmt, sync::LazyLock}; use base_db::Crate; +use either::Either; use hir_expand::{AstId, MacroCallId, attrs::AttrId, name::Name}; use indexmap::map::Entry; use itertools::Itertools; @@ -199,7 +200,7 @@ struct DeriveMacroInvocation { attr_id: AttrId, /// The `#[derive]` call attr_call_id: MacroCallId, - derive_call_ids: SmallVec<[Option; 4]>, + derive_call_ids: SmallVec<[Option>; 4]>, } pub(crate) static BUILTIN_SCOPE: LazyLock> = LazyLock::new(|| { @@ -345,7 +346,9 @@ impl ItemScope { pub fn all_macro_calls(&self) -> impl Iterator + '_ { self.macro_invocations.values().copied().chain(self.attr_macros.values().copied()).chain( self.derive_macros.values().flat_map(|it| { - it.iter().flat_map(|it| it.derive_call_ids.iter().copied().flatten()) + it.iter().flat_map(|it| { + it.derive_call_ids.iter().copied().flatten().flat_map(|it| it.left()) + }) }), ) } @@ -379,6 +382,10 @@ impl ItemScope { self.types.get(name).map(|item| (item.def, item.vis)) } + pub(crate) fn makro(&self, name: &Name) -> Option { + self.macros.get(name).map(|item| item.def) + } + /// XXX: this is O(N) rather than O(1), try to not introduce new usages. pub(crate) fn name_of(&self, item: ItemInNs) -> Option<(&Name, Visibility, /*declared*/ bool)> { match item { @@ -519,7 +526,7 @@ impl ItemScope { pub(crate) fn set_derive_macro_invoc( &mut self, adt: AstId, - call: MacroCallId, + call: Either, id: AttrId, idx: usize, ) { @@ -539,7 +546,7 @@ impl ItemScope { adt: AstId, attr_id: AttrId, attr_call_id: MacroCallId, - mut derive_call_ids: SmallVec<[Option; 4]>, + mut derive_call_ids: SmallVec<[Option>; 4]>, ) { derive_call_ids.shrink_to_fit(); self.derive_macros.entry(adt).or_default().push(DeriveMacroInvocation { @@ -554,7 +561,9 @@ impl ItemScope { ) -> impl Iterator< Item = ( AstId, - impl Iterator])>, + impl Iterator< + Item = (AttrId, MacroCallId, &[Option>]), + >, ), > + '_ { self.derive_macros.iter().map(|(k, v)| { diff --git a/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs b/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs index eba4d87ec9f8..092ff6e48671 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs @@ -7,8 +7,8 @@ use intern::{Symbol, sym}; use stdx::impl_from; use crate::{ - AdtId, AssocItemId, AttrDefId, Crate, EnumId, EnumVariantId, FunctionId, ImplId, ModuleDefId, - StaticId, StructId, TraitId, TypeAliasId, UnionId, + AdtId, AssocItemId, AttrDefId, Crate, EnumId, EnumVariantId, FunctionId, ImplId, MacroId, + ModuleDefId, StaticId, StructId, TraitId, TypeAliasId, UnionId, attrs::AttrFlags, db::DefDatabase, nameres::{DefMap, assoc::TraitItems, crate_def_map, crate_local_def_map}, @@ -99,7 +99,7 @@ pub fn crate_lang_items(db: &dyn DefDatabase, krate: Crate) -> Option Option { + let mut current = &core_def_map[core_def_map.root]; + for module in modules { + let Some((ModuleDefId::ModuleId(cur), _)) = + current.scope.type_(&Name::new_symbol_root(module.clone())) + else { + return None; + }; + if cur.krate(db) != core_def_map.krate() || cur.block(db) != core_def_map.block_id() { + return None; + } + current = &core_def_map[cur]; + } + current.scope.makro(&Name::new_symbol_root(name)) +} + #[salsa::tracked(returns(as_deref))] pub(crate) fn crate_notable_traits(db: &dyn DefDatabase, krate: Crate) -> Option> { let mut traits = Vec::new(); @@ -195,7 +216,11 @@ macro_rules! language_item_table { @non_lang_core_traits: - $( core::$($non_lang_module:ident)::*, $non_lang_trait:ident; )* + $( core::$($non_lang_trait_module:ident)::*, $non_lang_trait:ident; )* + + @non_lang_core_macros: + + $( core::$($non_lang_macro_module:ident)::*, $non_lang_macro:ident, $non_lang_macro_field:ident; )* ) => { #[allow(non_snake_case)] // FIXME: Should we remove this? #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] @@ -207,6 +232,9 @@ macro_rules! language_item_table { $( pub $non_lang_trait: Option, )* + $( + pub $non_lang_macro_field: Option, + )* } impl LangItems { @@ -218,6 +246,7 @@ macro_rules! language_item_table { fn merge_prefer_self(&mut self, other: &Self) { $( self.$lang_item = self.$lang_item.or(other.$lang_item); )* $( self.$non_lang_trait = self.$non_lang_trait.or(other.$non_lang_trait); )* + $( self.$non_lang_macro_field = self.$non_lang_macro_field.or(other.$non_lang_macro_field); )* } fn assign_lang_item(&mut self, name: Symbol, target: LangItemTarget) { @@ -233,8 +262,9 @@ macro_rules! language_item_table { } } - fn fill_non_lang_core_traits(&mut self, db: &dyn DefDatabase, core_def_map: &DefMap) { - $( self.$non_lang_trait = resolve_core_trait(db, core_def_map, &[ $(sym::$non_lang_module),* ], sym::$non_lang_trait); )* + fn fill_non_lang_core_items(&mut self, db: &dyn DefDatabase, core_def_map: &DefMap) { + $( self.$non_lang_trait = resolve_core_trait(db, core_def_map, &[ $(sym::$non_lang_trait_module),* ], sym::$non_lang_trait); )* + $( self.$non_lang_macro_field = resolve_core_macro(db, core_def_map, &[ $(sym::$non_lang_macro_module),* ], sym::$non_lang_macro); )* } } @@ -479,4 +509,16 @@ language_item_table! { LangItems => core::hash, Hash; core::cmp, Ord; core::cmp, Eq; + + @non_lang_core_macros: + core::default, Default, DefaultDerive; + core::fmt, Debug, DebugDerive; + core::hash, Hash, HashDerive; + core::cmp, PartialOrd, PartialOrdDerive; + core::cmp, Ord, OrdDerive; + core::cmp, PartialEq, PartialEqDerive; + core::cmp, Eq, EqDerive; + core::marker, CoercePointee, CoercePointeeDerive; + core::marker, Copy, CopyDerive; + core::clone, Clone, CloneDerive; } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs index 5f05cdb1e2ba..150372f1a0d9 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs @@ -61,6 +61,7 @@ mod tests; use std::ops::{Deref, DerefMut, Index, IndexMut}; use base_db::Crate; +use either::Either; use hir_expand::{ EditionedFileId, ErasedAstId, HirFileId, InFile, MacroCallId, mod_path::ModPath, name::Name, proc_macro::ProcMacroKind, @@ -75,8 +76,8 @@ use triomphe::Arc; use tt::TextRange; use crate::{ - AstId, BlockId, BlockLoc, ExternCrateId, FunctionId, FxIndexMap, Lookup, MacroCallStyles, - MacroExpander, MacroId, ModuleId, ModuleIdLt, ProcMacroId, UseId, + AstId, BlockId, BlockLoc, BuiltinDeriveImplId, ExternCrateId, FunctionId, FxIndexMap, Lookup, + MacroCallStyles, MacroExpander, MacroId, ModuleId, ModuleIdLt, ProcMacroId, UseId, db::DefDatabase, item_scope::{BuiltinShadowMode, ItemScope}, item_tree::TreeId, @@ -192,7 +193,8 @@ pub struct DefMap { /// Tracks which custom derives are in scope for an item, to allow resolution of derive helper /// attributes. // FIXME: Figure out a better way for the IDE layer to resolve these? - derive_helpers_in_scope: FxHashMap, Vec<(Name, MacroId, MacroCallId)>>, + derive_helpers_in_scope: + FxHashMap, Vec<(Name, MacroId, Either)>>, /// A mapping from [`hir_expand::MacroDefId`] to [`crate::MacroId`]. pub macro_def_to_macro_id: FxHashMap, @@ -540,7 +542,7 @@ impl DefMap { pub fn derive_helpers_in_scope( &self, id: AstId, - ) -> Option<&[(Name, MacroId, MacroCallId)]> { + ) -> Option<&[(Name, MacroId, Either)]> { self.derive_helpers_in_scope.get(&id.map(|it| it.upcast())).map(Deref::deref) } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs index 87ade0651762..323060f61d15 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs @@ -3,7 +3,7 @@ //! `DefCollector::collect` contains the fixed-point iteration loop which //! resolves imports and expands macros. -use std::{iter, mem}; +use std::{iter, mem, ops::Range}; use base_db::{BuiltDependency, Crate, CrateOrigin, LangCrateOrigin}; use cfg::{CfgAtom, CfgExpr, CfgOptions}; @@ -226,6 +226,7 @@ struct DeferredBuiltinDerive { container: ItemContainerId, derive_attr_id: AttrId, derive_index: u32, + helpers_range: Range, } /// Walks the tree of module recursively @@ -1354,7 +1355,7 @@ impl<'db> DefCollector<'db> { if let Ok((macro_id, def_id, call_id)) = id { self.def_map.modules[directive.module_id].scope.set_derive_macro_invoc( ast_id.ast_id, - call_id, + Either::Left(call_id), *derive_attr, *derive_pos, ); @@ -1369,7 +1370,7 @@ impl<'db> DefCollector<'db> { .extend(izip!( helpers.iter().cloned(), iter::repeat(macro_id), - iter::repeat(call_id), + iter::repeat(Either::Left(call_id)), )); } } @@ -1492,6 +1493,8 @@ impl<'db> DefCollector<'db> { Interned::new(path), ); + derive_call_ids.push(None); + // Try to resolve the derive immediately. If we succeed, we can also use the fast path // for builtin derives. If not, we cannot use it, as it can cause the ADT to become // interned while the derive is still unresolved, which will cause it to get forgotten. @@ -1506,23 +1509,42 @@ impl<'db> DefCollector<'db> { call_id, ); + let ast_id_without_path = ast_id.ast_id; + let directive = MacroDirective { + module_id: directive.module_id, + depth: directive.depth + 1, + kind: MacroDirectiveKind::Derive { + ast_id, + derive_attr: *attr_id, + derive_pos: idx, + ctxt: call_site.ctx, + derive_macro_id: call_id, + }, + container: directive.container, + }; + if let Ok((macro_id, def_id, call_id)) = id { - derive_call_ids.push(Some(call_id)); + let (mut helpers_start, mut helpers_end) = (0, 0); // Record its helper attributes. if def_id.krate != self.def_map.krate { let def_map = crate_def_map(self.db, def_id.krate); if let Some(helpers) = def_map.data.exported_derives.get(¯o_id) { - self.def_map + let derive_helpers = self + .def_map .derive_helpers_in_scope - .entry(ast_id.ast_id.map(|it| it.upcast())) - .or_default() - .extend(izip!( - helpers.iter().cloned(), - iter::repeat(macro_id), - iter::repeat(call_id), - )); + .entry( + ast_id_without_path.map(|it| it.upcast()), + ) + .or_default(); + helpers_start = derive_helpers.len(); + derive_helpers.extend(izip!( + helpers.iter().cloned(), + iter::repeat(macro_id), + iter::repeat(Either::Left(call_id)), + )); + helpers_end = derive_helpers.len(); } } @@ -1531,7 +1553,7 @@ impl<'db> DefCollector<'db> { def_id.kind { self.deferred_builtin_derives - .entry(ast_id.ast_id.upcast()) + .entry(ast_id_without_path.upcast()) .or_default() .push(DeferredBuiltinDerive { call_id, @@ -1541,24 +1563,15 @@ impl<'db> DefCollector<'db> { depth: directive.depth, derive_attr_id: *attr_id, derive_index: idx as u32, + helpers_range: helpers_start..helpers_end, }); } else { - push_resolved(&mut resolved, directive, call_id); + push_resolved(&mut resolved, &directive, call_id); + *derive_call_ids.last_mut().unwrap() = + Some(Either::Left(call_id)); } } else { - derive_call_ids.push(None); - self.unresolved_macros.push(MacroDirective { - module_id: directive.module_id, - depth: directive.depth + 1, - kind: MacroDirectiveKind::Derive { - ast_id, - derive_attr: *attr_id, - derive_pos: idx, - ctxt: call_site.ctx, - derive_macro_id: call_id, - }, - container: directive.container, - }); + self.unresolved_macros.push(directive); } } @@ -1858,9 +1871,8 @@ impl ModCollector<'_, '_> { ast_id: FileAstId, id: AdtId, def_map: &mut DefMap| { - let Some(deferred_derives) = - deferred_derives.remove(&InFile::new(file_id, ast_id.upcast())) - else { + let ast_id = InFile::new(file_id, ast_id.upcast()); + let Some(deferred_derives) = deferred_derives.remove(&ast_id.upcast()) else { return; }; let module = &mut def_map.modules[module_id]; @@ -1876,6 +1888,22 @@ impl ModCollector<'_, '_> { }, ); module.scope.define_builtin_derive_impl(impl_id); + module.scope.set_derive_macro_invoc( + ast_id, + Either::Right(impl_id), + deferred_derive.derive_attr_id, + deferred_derive.derive_index as usize, + ); + // Change its helper attributes to the new id. + if let Some(derive_helpers) = + def_map.derive_helpers_in_scope.get_mut(&ast_id.map(|it| it.upcast())) + { + for (_, _, call_id) in + &mut derive_helpers[deferred_derive.helpers_range.clone()] + { + *call_id = Either::Right(impl_id); + } + } }); } }; diff --git a/src/tools/rust-analyzer/crates/hir/src/semantics.rs b/src/tools/rust-analyzer/crates/hir/src/semantics.rs index f4c42537de93..e55b693ef018 100644 --- a/src/tools/rust-analyzer/crates/hir/src/semantics.rs +++ b/src/tools/rust-analyzer/crates/hir/src/semantics.rs @@ -13,7 +13,7 @@ use std::{ use base_db::FxIndexSet; use either::Either; use hir_def::{ - DefWithBodyId, MacroId, StructId, TraitId, VariantId, + BuiltinDeriveImplId, DefWithBodyId, HasModule, MacroId, StructId, TraitId, VariantId, attrs::parse_extra_crate_attrs, expr_store::{Body, ExprOrPatSource, HygieneId, path::Path}, hir::{BindingId, Expr, ExprId, ExprOrPatId, Pat}, @@ -622,7 +622,20 @@ impl<'db> SemanticsImpl<'db> { Some( calls .into_iter() - .map(|call| macro_call_to_macro_id(ctx, call?).map(|id| Macro { id })) + .map(|call| { + let call = call?; + match call { + Either::Left(call) => { + macro_call_to_macro_id(ctx, call).map(|id| Macro { id }) + } + Either::Right(call) => { + let call = call.loc(self.db); + let krate = call.krate(self.db); + let lang_items = hir_def::lang_item::lang_items(self.db, krate); + call.trait_.derive_macro(lang_items).map(|id| Macro { id }) + } + } + }) .collect(), ) }) @@ -633,7 +646,7 @@ impl<'db> SemanticsImpl<'db> { .derive_macro_calls(attr)? .into_iter() .flat_map(|call| { - let file_id = call?; + let file_id = call?.left()?; let ExpandResult { value, err } = self.db.parse_macro_expansion(file_id); let root_node = value.0.syntax_node(); self.cache(root_node.clone(), file_id.into()); @@ -643,7 +656,10 @@ impl<'db> SemanticsImpl<'db> { Some(res) } - fn derive_macro_calls(&self, attr: &ast::Attr) -> Option>> { + fn derive_macro_calls( + &self, + attr: &ast::Attr, + ) -> Option>>> { let adt = attr.syntax().parent().and_then(ast::Adt::cast)?; let file_id = self.find_file(adt.syntax()).file_id; let adt = InFile::new(file_id, &adt); @@ -690,8 +706,9 @@ impl<'db> SemanticsImpl<'db> { .derive_helpers_in_scope(InFile::new(sa.file_id, id))? .iter() .filter(|&(name, _, _)| *name == attr_name) - .map(|&(_, macro_, call)| (macro_.into(), call)) + .filter_map(|&(_, macro_, call)| Some((macro_.into(), call.left()?))) .collect(); + // FIXME: We filter our builtin derive "fake" expansions, is this correct? Should we still expose them somehow? res.is_empty().not().then_some(res) } @@ -1338,6 +1355,7 @@ impl<'db> SemanticsImpl<'db> { // FIXME: We need to call `f` for all of them as well though! process_expansion_for_token(ctx, &mut stack, derive_attr); for derive in derives.into_iter().flatten() { + let Either::Left(derive) = derive else { continue }; process_expansion_for_token(ctx, &mut stack, derive); } } @@ -1467,11 +1485,12 @@ impl<'db> SemanticsImpl<'db> { for (.., derive) in helpers.iter().filter(|(helper, ..)| *helper == attr_name) { + let Either::Left(derive) = *derive else { continue }; // as there may be multiple derives registering the same helper // name, we gotta make sure to call this for all of them! // FIXME: We need to call `f` for all of them as well though! res = res - .or(process_expansion_for_token(ctx, &mut stack, *derive)); + .or(process_expansion_for_token(ctx, &mut stack, derive)); } res }) diff --git a/src/tools/rust-analyzer/crates/hir/src/semantics/source_to_def.rs b/src/tools/rust-analyzer/crates/hir/src/semantics/source_to_def.rs index 257405992731..d222c3dc7ed1 100644 --- a/src/tools/rust-analyzer/crates/hir/src/semantics/source_to_def.rs +++ b/src/tools/rust-analyzer/crates/hir/src/semantics/source_to_def.rs @@ -87,10 +87,10 @@ use either::Either; use hir_def::{ - AdtId, BlockId, ConstId, ConstParamId, DefWithBodyId, EnumId, EnumVariantId, ExternBlockId, - ExternCrateId, FieldId, FunctionId, GenericDefId, GenericParamId, ImplId, LifetimeParamId, - Lookup, MacroId, ModuleId, StaticId, StructId, TraitId, TypeAliasId, TypeParamId, UnionId, - UseId, VariantId, + AdtId, BlockId, BuiltinDeriveImplId, ConstId, ConstParamId, DefWithBodyId, EnumId, + EnumVariantId, ExternBlockId, ExternCrateId, FieldId, FunctionId, GenericDefId, GenericParamId, + ImplId, LifetimeParamId, Lookup, MacroId, ModuleId, StaticId, StructId, TraitId, TypeAliasId, + TypeParamId, UnionId, UseId, VariantId, dyn_map::{ DynMap, keys::{self, Key}, @@ -394,7 +394,7 @@ impl SourceToDefCtx<'_, '_> { &mut self, item: InFile<&ast::Adt>, src: InFile, - ) -> Option<(AttrId, MacroCallId, &[Option])> { + ) -> Option<(AttrId, MacroCallId, &[Option>])> { let map = self.dyn_map(item)?; map[keys::DERIVE_MACRO_CALL] .get(&AstPtr::new(&src.value)) @@ -409,8 +409,11 @@ impl SourceToDefCtx<'_, '_> { pub(super) fn derive_macro_calls<'slf>( &'slf mut self, adt: InFile<&ast::Adt>, - ) -> Option])> + use<'slf>> - { + ) -> Option< + impl Iterator< + Item = (AttrId, MacroCallId, &'slf [Option>]), + > + use<'slf>, + > { self.dyn_map(adt).as_ref().map(|&map| { let dyn_map = &map[keys::DERIVE_MACRO_CALL]; adt.value diff --git a/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs b/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs index 7d02b8091890..ba8b3aa9cafe 100644 --- a/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs +++ b/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs @@ -583,26 +583,16 @@ fn main() { fn macro_expand_derive() { check( r#" -//- proc_macros: identity -//- minicore: clone, derive +//- proc_macros: identity, derive_identity +//- minicore: derive #[proc_macros::identity] -#[derive(C$0lone)] +#[derive(proc_macros::DeriveIde$0ntity)] struct Foo {} "#, expect![[r#" - Clone - impl <>core::clone::Clone for Foo< >where { - fn clone(&self) -> Self { - match self { - Foo{} - => Foo{} - , - - } - } - - }"#]], + proc_macros::DeriveIdentity + struct Foo{}"#]], ); } @@ -610,15 +600,17 @@ struct Foo {} fn macro_expand_derive2() { check( r#" -//- minicore: copy, clone, derive +//- proc_macros: derive_identity +//- minicore: derive -#[derive(Cop$0y)] -#[derive(Clone)] +#[derive(proc_macros::$0DeriveIdentity)] +#[derive(proc_macros::DeriveIdentity)] struct Foo {} "#, expect![[r#" - Copy - impl <>core::marker::Copy for Foo< >where{}"#]], + proc_macros::DeriveIdentity + #[derive(proc_macros::DeriveIdentity)] + struct Foo{}"#]], ); } @@ -626,35 +618,27 @@ struct Foo {} fn macro_expand_derive_multi() { check( r#" -//- minicore: copy, clone, derive +//- proc_macros: derive_identity +//- minicore: derive -#[derive(Cop$0y, Clone)] +#[derive(proc_macros::DeriveIdent$0ity, proc_macros::DeriveIdentity)] struct Foo {} "#, expect![[r#" - Copy - impl <>core::marker::Copy for Foo< >where{}"#]], + proc_macros::DeriveIdentity + struct Foo{}"#]], ); check( r#" -//- minicore: copy, clone, derive +//- proc_macros: derive_identity +//- minicore: derive -#[derive(Copy, Cl$0one)] +#[derive(proc_macros::DeriveIdentity, proc_macros::De$0riveIdentity)] struct Foo {} "#, expect![[r#" - Clone - impl <>core::clone::Clone for Foo< >where { - fn clone(&self) -> Self { - match self { - Foo{} - => Foo{} - , - - } - } - - }"#]], + proc_macros::DeriveIdentity + struct Foo{}"#]], ); } diff --git a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs index b6efc599f181..3fadca29d118 100644 --- a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs +++ b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs @@ -532,4 +532,5 @@ define_symbols! { CoerceUnsized, DispatchFromDyn, define_opaque, + marker, } From acc8c0bd65a057c7e61d3e70556e66b66d3f45b7 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 Dec 2025 13:05:09 +0000 Subject: [PATCH 0126/1843] Reduce usage of FnAbi in codegen_llvm_intrinsic_call --- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 88 ++++++++++++++++++- compiler/rustc_codegen_llvm/src/intrinsic.rs | 30 ++++++- 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 36ea76cbc51a..f138b87d3405 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -4,10 +4,10 @@ mod simd; #[cfg(feature = "master")] use std::iter; -#[cfg(feature = "master")] -use gccjit::Type; use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] +use gccjit::{FnAttribute, Type}; +#[cfg(feature = "master")] use rustc_abi::ExternAbi; use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::MemFlags; @@ -22,13 +22,18 @@ use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; +use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, Ty}; +#[cfg(feature = "master")] +use rustc_session::config; use rustc_span::{Span, Symbol, sym}; +#[cfg(feature = "master")] +use rustc_target::callconv::ArgAttributes; use rustc_target::callconv::{ArgAbi, PassMode}; -use crate::abi::{FnAbiGccExt, GccType}; +use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType}; use crate::builder::Builder; use crate::common::{SignType, TypeReflection}; use crate::context::CodegenCx; @@ -621,7 +626,82 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } else { self.linkage.set(FunctionType::Extern); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let fn_ty = fn_abi.gcc_type(self); + assert!(!fn_abi.ret.is_indirect()); + assert!(!fn_abi.c_variadic); + + let return_type = match fn_abi.ret.mode { + PassMode::Ignore => self.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => { + fn_abi.ret.layout.immediate_gcc_type(self) + } + PassMode::Cast { .. } | PassMode::Indirect { .. } => { + unreachable!() + } + }; + + #[cfg(feature = "master")] + let mut non_null_args = Vec::new(); + + #[cfg(feature = "master")] + let mut apply_attrs = + |mut ty: Type<'gcc>, attrs: &ArgAttributes, arg_index: usize| { + if self.sess().opts.optimize == config::OptLevel::No { + return ty; + } + if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NoAlias) { + ty = ty.make_restrict() + } + if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NonNull) { + non_null_args.push(arg_index as i32 + 1); + } + ty + }; + #[cfg(not(feature = "master"))] + let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| ty; + + let mut argument_tys = Vec::with_capacity(fn_abi.args.len()); + for arg in fn_abi.args.iter() { + match arg.mode { + PassMode::Ignore => {} + PassMode::Pair(a, b) => { + let arg_pos = argument_tys.len(); + argument_tys.push(apply_attrs( + arg.layout.scalar_pair_element_gcc_type(self, 0), + &a, + arg_pos, + )); + argument_tys.push(apply_attrs( + arg.layout.scalar_pair_element_gcc_type(self, 1), + &b, + arg_pos + 1, + )); + } + PassMode::Direct(attrs) => argument_tys.push(apply_attrs( + arg.layout.immediate_gcc_type(self), + &attrs, + argument_tys.len(), + )), + PassMode::Indirect { .. } | PassMode::Cast { .. } => { + unreachable!() + } + } + } + + #[cfg(feature = "master")] + let fn_attrs = if non_null_args.is_empty() { + Vec::new() + } else { + vec![FnAttribute::NonNull(non_null_args)] + }; + + let fn_ty = FnAbiGcc { + return_type, + arguments_type: argument_tys, + is_c_variadic: false, + on_stack_param_indices: FxHashSet::default(), + #[cfg(feature = "master")] + fn_attributes: fn_attrs, + }; let func = match sym { "llvm.fma.f16" => { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 481f75f337d6..7327137fbbdd 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -649,7 +649,32 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // FIXME remove usage of fn_abi let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); assert!(!fn_abi.ret.is_indirect()); - let fn_ty = fn_abi.llvm_type(self); + assert!(!fn_abi.c_variadic); + + let llreturn_ty = match &fn_abi.ret.mode { + PassMode::Ignore => self.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => fn_abi.ret.layout.immediate_llvm_type(self), + PassMode::Cast { .. } | PassMode::Indirect { .. } => { + unreachable!() + } + }; + + let mut llargument_tys = Vec::with_capacity(fn_abi.args.len()); + for arg in &fn_abi.args { + match &arg.mode { + PassMode::Ignore => {} + PassMode::Direct(_) => llargument_tys.push(arg.layout.immediate_llvm_type(self)), + PassMode::Pair(..) => { + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(self, 0, true)); + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(self, 1, true)); + } + PassMode::Indirect { .. } | PassMode::Cast { .. } => { + unreachable!() + } + }; + } + + let fn_ty = self.type_func(&llargument_tys, llreturn_ty); let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) { llfn @@ -665,12 +690,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let llfn = declare_raw_fn( self, sym, - fn_abi.llvm_cconv(self), + llvm::CCallConv, llvm::UnnamedAddr::Global, llvm::Visibility::Default, fn_ty, ); - fn_abi.apply_attrs_llfn(self, llfn, Some(instance)); llfn }; From f004b3eb19b988a9066ca9dc9b2a63919f504ada Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 Dec 2025 13:05:09 +0000 Subject: [PATCH 0127/1843] Reduce usage of FnAbi in codegen_llvm_intrinsic_call --- src/intrinsic/mod.rs | 88 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs index 36ea76cbc51a..f138b87d3405 100644 --- a/src/intrinsic/mod.rs +++ b/src/intrinsic/mod.rs @@ -4,10 +4,10 @@ mod simd; #[cfg(feature = "master")] use std::iter; -#[cfg(feature = "master")] -use gccjit::Type; use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] +use gccjit::{FnAttribute, Type}; +#[cfg(feature = "master")] use rustc_abi::ExternAbi; use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::MemFlags; @@ -22,13 +22,18 @@ use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; +use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, Ty}; +#[cfg(feature = "master")] +use rustc_session::config; use rustc_span::{Span, Symbol, sym}; +#[cfg(feature = "master")] +use rustc_target::callconv::ArgAttributes; use rustc_target::callconv::{ArgAbi, PassMode}; -use crate::abi::{FnAbiGccExt, GccType}; +use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType}; use crate::builder::Builder; use crate::common::{SignType, TypeReflection}; use crate::context::CodegenCx; @@ -621,7 +626,82 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } else { self.linkage.set(FunctionType::Extern); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let fn_ty = fn_abi.gcc_type(self); + assert!(!fn_abi.ret.is_indirect()); + assert!(!fn_abi.c_variadic); + + let return_type = match fn_abi.ret.mode { + PassMode::Ignore => self.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => { + fn_abi.ret.layout.immediate_gcc_type(self) + } + PassMode::Cast { .. } | PassMode::Indirect { .. } => { + unreachable!() + } + }; + + #[cfg(feature = "master")] + let mut non_null_args = Vec::new(); + + #[cfg(feature = "master")] + let mut apply_attrs = + |mut ty: Type<'gcc>, attrs: &ArgAttributes, arg_index: usize| { + if self.sess().opts.optimize == config::OptLevel::No { + return ty; + } + if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NoAlias) { + ty = ty.make_restrict() + } + if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NonNull) { + non_null_args.push(arg_index as i32 + 1); + } + ty + }; + #[cfg(not(feature = "master"))] + let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| ty; + + let mut argument_tys = Vec::with_capacity(fn_abi.args.len()); + for arg in fn_abi.args.iter() { + match arg.mode { + PassMode::Ignore => {} + PassMode::Pair(a, b) => { + let arg_pos = argument_tys.len(); + argument_tys.push(apply_attrs( + arg.layout.scalar_pair_element_gcc_type(self, 0), + &a, + arg_pos, + )); + argument_tys.push(apply_attrs( + arg.layout.scalar_pair_element_gcc_type(self, 1), + &b, + arg_pos + 1, + )); + } + PassMode::Direct(attrs) => argument_tys.push(apply_attrs( + arg.layout.immediate_gcc_type(self), + &attrs, + argument_tys.len(), + )), + PassMode::Indirect { .. } | PassMode::Cast { .. } => { + unreachable!() + } + } + } + + #[cfg(feature = "master")] + let fn_attrs = if non_null_args.is_empty() { + Vec::new() + } else { + vec![FnAttribute::NonNull(non_null_args)] + }; + + let fn_ty = FnAbiGcc { + return_type, + arguments_type: argument_tys, + is_c_variadic: false, + on_stack_param_indices: FxHashSet::default(), + #[cfg(feature = "master")] + fn_attributes: fn_attrs, + }; let func = match sym { "llvm.fma.f16" => { From f1ab0036585e64bffbdc8ac20e7b0a5fb9507e95 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:47:29 +0000 Subject: [PATCH 0128/1843] Don't compute FnAbi for LLVM intrinsics in backends --- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 101 ++---------------- compiler/rustc_codegen_llvm/src/intrinsic.rs | 48 +++++---- 2 files changed, 34 insertions(+), 115 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index f138b87d3405..de262994fa45 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -4,9 +4,9 @@ mod simd; #[cfg(feature = "master")] use std::iter; -use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] -use gccjit::{FnAttribute, Type}; +use gccjit::Type; +use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] use rustc_abi::ExternAbi; use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; @@ -24,16 +24,16 @@ use rustc_codegen_ssa::traits::{ }; use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; +#[cfg(feature = "master")] +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; -#[cfg(feature = "master")] -use rustc_session::config; use rustc_span::{Span, Symbol, sym}; -#[cfg(feature = "master")] -use rustc_target::callconv::ArgAttributes; use rustc_target::callconv::{ArgAbi, PassMode}; -use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType}; +#[cfg(feature = "master")] +use crate::abi::FnAbiGccExt; +use crate::abi::GccType; use crate::builder::Builder; use crate::common::{SignType, TypeReflection}; use crate::context::CodegenCx; @@ -625,83 +625,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc *func } else { self.linkage.set(FunctionType::Extern); - let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - assert!(!fn_abi.ret.is_indirect()); - assert!(!fn_abi.c_variadic); - - let return_type = match fn_abi.ret.mode { - PassMode::Ignore => self.type_void(), - PassMode::Direct(_) | PassMode::Pair(..) => { - fn_abi.ret.layout.immediate_gcc_type(self) - } - PassMode::Cast { .. } | PassMode::Indirect { .. } => { - unreachable!() - } - }; - - #[cfg(feature = "master")] - let mut non_null_args = Vec::new(); - - #[cfg(feature = "master")] - let mut apply_attrs = - |mut ty: Type<'gcc>, attrs: &ArgAttributes, arg_index: usize| { - if self.sess().opts.optimize == config::OptLevel::No { - return ty; - } - if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NoAlias) { - ty = ty.make_restrict() - } - if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NonNull) { - non_null_args.push(arg_index as i32 + 1); - } - ty - }; - #[cfg(not(feature = "master"))] - let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| ty; - - let mut argument_tys = Vec::with_capacity(fn_abi.args.len()); - for arg in fn_abi.args.iter() { - match arg.mode { - PassMode::Ignore => {} - PassMode::Pair(a, b) => { - let arg_pos = argument_tys.len(); - argument_tys.push(apply_attrs( - arg.layout.scalar_pair_element_gcc_type(self, 0), - &a, - arg_pos, - )); - argument_tys.push(apply_attrs( - arg.layout.scalar_pair_element_gcc_type(self, 1), - &b, - arg_pos + 1, - )); - } - PassMode::Direct(attrs) => argument_tys.push(apply_attrs( - arg.layout.immediate_gcc_type(self), - &attrs, - argument_tys.len(), - )), - PassMode::Indirect { .. } | PassMode::Cast { .. } => { - unreachable!() - } - } - } - - #[cfg(feature = "master")] - let fn_attrs = if non_null_args.is_empty() { - Vec::new() - } else { - vec![FnAttribute::NonNull(non_null_args)] - }; - - let fn_ty = FnAbiGcc { - return_type, - arguments_type: argument_tys, - is_c_variadic: false, - on_stack_param_indices: FxHashSet::default(), - #[cfg(feature = "master")] - fn_attributes: fn_attrs, - }; let func = match sym { "llvm.fma.f16" => { @@ -714,13 +637,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc self.intrinsics.borrow_mut().insert(sym.to_string(), func); - self.on_stack_function_params - .borrow_mut() - .insert(func, fn_ty.on_stack_param_indices); - #[cfg(feature = "master")] - for fn_attr in fn_ty.fn_attributes { - func.add_attribute(fn_attr); - } + self.on_stack_function_params.borrow_mut().insert(func, FxHashSet::default()); crate::attributes::from_fn_attrs(self, func, instance); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7327137fbbdd..13402edd13e2 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -646,32 +646,34 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ) -> Self::Value { let tcx = self.tcx(); - // FIXME remove usage of fn_abi - let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - assert!(!fn_abi.ret.is_indirect()); - assert!(!fn_abi.c_variadic); - - let llreturn_ty = match &fn_abi.ret.mode { - PassMode::Ignore => self.type_void(), - PassMode::Direct(_) | PassMode::Pair(..) => fn_abi.ret.layout.immediate_llvm_type(self), - PassMode::Cast { .. } | PassMode::Indirect { .. } => { - unreachable!() + let fn_ty = instance.ty(tcx, self.typing_env()); + let fn_sig = match *fn_ty.kind() { + ty::FnDef(def_id, args) => { + tcx.instantiate_bound_regions_with_erased(tcx.fn_sig(def_id).instantiate(tcx, args)) } + _ => unreachable!(), + }; + assert!(!fn_sig.c_variadic); + + let ret_layout = self.layout_of(fn_sig.output()); + let llreturn_ty = if ret_layout.is_zst() { + self.type_void() + } else { + ret_layout.immediate_llvm_type(self) }; - let mut llargument_tys = Vec::with_capacity(fn_abi.args.len()); - for arg in &fn_abi.args { - match &arg.mode { - PassMode::Ignore => {} - PassMode::Direct(_) => llargument_tys.push(arg.layout.immediate_llvm_type(self)), - PassMode::Pair(..) => { - llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(self, 0, true)); - llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(self, 1, true)); - } - PassMode::Indirect { .. } | PassMode::Cast { .. } => { - unreachable!() - } - }; + let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len()); + for &arg in fn_sig.inputs() { + let arg_layout = self.layout_of(arg); + if arg_layout.is_zst() { + continue; + } + if let BackendRepr::ScalarPair(_, _) = arg_layout.backend_repr { + llargument_tys.push(arg_layout.scalar_pair_element_llvm_type(self, 0, true)); + llargument_tys.push(arg_layout.scalar_pair_element_llvm_type(self, 1, true)); + continue; + } + llargument_tys.push(arg_layout.immediate_llvm_type(self)); } let fn_ty = self.type_func(&llargument_tys, llreturn_ty); From e90414f6224b95662c2f52fcfc508a316f54db09 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:47:29 +0000 Subject: [PATCH 0129/1843] Don't compute FnAbi for LLVM intrinsics in backends --- src/intrinsic/mod.rs | 101 ++++--------------------------------------- 1 file changed, 9 insertions(+), 92 deletions(-) diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs index f138b87d3405..de262994fa45 100644 --- a/src/intrinsic/mod.rs +++ b/src/intrinsic/mod.rs @@ -4,9 +4,9 @@ mod simd; #[cfg(feature = "master")] use std::iter; -use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] -use gccjit::{FnAttribute, Type}; +use gccjit::Type; +use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] use rustc_abi::ExternAbi; use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; @@ -24,16 +24,16 @@ use rustc_codegen_ssa::traits::{ }; use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; +#[cfg(feature = "master")] +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; -#[cfg(feature = "master")] -use rustc_session::config; use rustc_span::{Span, Symbol, sym}; -#[cfg(feature = "master")] -use rustc_target::callconv::ArgAttributes; use rustc_target::callconv::{ArgAbi, PassMode}; -use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType}; +#[cfg(feature = "master")] +use crate::abi::FnAbiGccExt; +use crate::abi::GccType; use crate::builder::Builder; use crate::common::{SignType, TypeReflection}; use crate::context::CodegenCx; @@ -625,83 +625,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc *func } else { self.linkage.set(FunctionType::Extern); - let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - assert!(!fn_abi.ret.is_indirect()); - assert!(!fn_abi.c_variadic); - - let return_type = match fn_abi.ret.mode { - PassMode::Ignore => self.type_void(), - PassMode::Direct(_) | PassMode::Pair(..) => { - fn_abi.ret.layout.immediate_gcc_type(self) - } - PassMode::Cast { .. } | PassMode::Indirect { .. } => { - unreachable!() - } - }; - - #[cfg(feature = "master")] - let mut non_null_args = Vec::new(); - - #[cfg(feature = "master")] - let mut apply_attrs = - |mut ty: Type<'gcc>, attrs: &ArgAttributes, arg_index: usize| { - if self.sess().opts.optimize == config::OptLevel::No { - return ty; - } - if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NoAlias) { - ty = ty.make_restrict() - } - if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NonNull) { - non_null_args.push(arg_index as i32 + 1); - } - ty - }; - #[cfg(not(feature = "master"))] - let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| ty; - - let mut argument_tys = Vec::with_capacity(fn_abi.args.len()); - for arg in fn_abi.args.iter() { - match arg.mode { - PassMode::Ignore => {} - PassMode::Pair(a, b) => { - let arg_pos = argument_tys.len(); - argument_tys.push(apply_attrs( - arg.layout.scalar_pair_element_gcc_type(self, 0), - &a, - arg_pos, - )); - argument_tys.push(apply_attrs( - arg.layout.scalar_pair_element_gcc_type(self, 1), - &b, - arg_pos + 1, - )); - } - PassMode::Direct(attrs) => argument_tys.push(apply_attrs( - arg.layout.immediate_gcc_type(self), - &attrs, - argument_tys.len(), - )), - PassMode::Indirect { .. } | PassMode::Cast { .. } => { - unreachable!() - } - } - } - - #[cfg(feature = "master")] - let fn_attrs = if non_null_args.is_empty() { - Vec::new() - } else { - vec![FnAttribute::NonNull(non_null_args)] - }; - - let fn_ty = FnAbiGcc { - return_type, - arguments_type: argument_tys, - is_c_variadic: false, - on_stack_param_indices: FxHashSet::default(), - #[cfg(feature = "master")] - fn_attributes: fn_attrs, - }; let func = match sym { "llvm.fma.f16" => { @@ -714,13 +637,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc self.intrinsics.borrow_mut().insert(sym.to_string(), func); - self.on_stack_function_params - .borrow_mut() - .insert(func, fn_ty.on_stack_param_indices); - #[cfg(feature = "master")] - for fn_attr in fn_ty.fn_attributes { - func.add_attribute(fn_attr); - } + self.on_stack_function_params.borrow_mut().insert(func, FxHashSet::default()); crate::attributes::from_fn_attrs(self, func, instance); From 72f1ac9fbdbff5b432519891ac3d826a0333cf34 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Wed, 12 Mar 2025 10:26:37 +0000 Subject: [PATCH 0130/1843] Compile-Time Reflection MVP: tuples --- clippy_utils/src/sym.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/clippy_utils/src/sym.rs b/clippy_utils/src/sym.rs index a0d2e8673fe6..74f89cfc6811 100644 --- a/clippy_utils/src/sym.rs +++ b/clippy_utils/src/sym.rs @@ -62,7 +62,6 @@ generate! { MsrvStack, Octal, OpenOptions, - Other, PathLookup, Regex, RegexBuilder, From 21f6afca40cb423b4920fbc571490f801aa95d16 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 8 Jan 2026 14:10:33 +0100 Subject: [PATCH 0131/1843] Update `literal-escaper` version to `0.0.7` --- clippy_dev/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clippy_dev/Cargo.toml b/clippy_dev/Cargo.toml index c2abbac37535..238465210ee2 100644 --- a/clippy_dev/Cargo.toml +++ b/clippy_dev/Cargo.toml @@ -10,7 +10,7 @@ clap = { version = "4.4", features = ["derive"] } indoc = "1.0" itertools = "0.12" opener = "0.7" -rustc-literal-escaper = "0.0.5" +rustc-literal-escaper = "0.0.7" walkdir = "2.3" [package.metadata.rust-analyzer] From 0051e31f6f8000c6bf2fab1549a55a26d4d455ff Mon Sep 17 00:00:00 2001 From: dianqk Date: Tue, 23 Dec 2025 22:25:11 +0800 Subject: [PATCH 0132/1843] New MIR Pass: SsaRangePropagation --- compiler/rustc_middle/src/mir/statement.rs | 6 + compiler/rustc_mir_transform/src/lib.rs | 4 + .../rustc_mir_transform/src/ssa_range_prop.rs | 203 ++++++++++++++++++ ...a_range.on_assert.SsaRangePropagation.diff | 69 ++++++ .../ssa_range.on_if.SsaRangePropagation.diff | 63 ++++++ ...ssa_range.on_if_2.SsaRangePropagation.diff | 20 ++ ...sa_range.on_match.SsaRangePropagation.diff | 33 +++ ..._range.on_match_2.SsaRangePropagation.diff | 26 +++ tests/mir-opt/range/ssa_range.rs | 70 ++++++ 9 files changed, 494 insertions(+) create mode 100644 compiler/rustc_mir_transform/src/ssa_range_prop.rs create mode 100644 tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff create mode 100644 tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff create mode 100644 tests/mir-opt/range/ssa_range.on_if_2.SsaRangePropagation.diff create mode 100644 tests/mir-opt/range/ssa_range.on_match.SsaRangePropagation.diff create mode 100644 tests/mir-opt/range/ssa_range.on_match_2.SsaRangePropagation.diff create mode 100644 tests/mir-opt/range/ssa_range.rs diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index 1ba1ae3e1531..2ee1d53cabd5 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -374,6 +374,12 @@ impl<'tcx> Place<'tcx> { self.projection.iter().any(|elem| elem.is_indirect()) } + /// Returns `true` if the `Place` always refers to the same memory region + /// whatever the state of the program. + pub fn is_stable_offset(&self) -> bool { + self.projection.iter().all(|elem| elem.is_stable_offset()) + } + /// Returns `true` if this `Place`'s first projection is `Deref`. /// /// This is useful because for MIR phases `AnalysisPhase::PostCleanup` and later, diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 701d7ff854a7..24ea4a5cd6df 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -198,6 +198,7 @@ declare_passes! { mod single_use_consts : SingleUseConsts; mod sroa : ScalarReplacementOfAggregates; mod strip_debuginfo : StripDebugInfo; + mod ssa_range_prop: SsaRangePropagation; mod unreachable_enum_branching : UnreachableEnumBranching; mod unreachable_prop : UnreachablePropagation; mod validate : Validator; @@ -741,6 +742,9 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &dead_store_elimination::DeadStoreElimination::Initial, &gvn::GVN, &simplify::SimplifyLocals::AfterGVN, + // This pass does attempt to track assignments. + // Keep it close to GVN which merges identical values into the same local. + &ssa_range_prop::SsaRangePropagation, &match_branches::MatchBranchSimplification, &dataflow_const_prop::DataflowConstProp, &single_use_consts::SingleUseConsts, diff --git a/compiler/rustc_mir_transform/src/ssa_range_prop.rs b/compiler/rustc_mir_transform/src/ssa_range_prop.rs new file mode 100644 index 000000000000..47e17de0560b --- /dev/null +++ b/compiler/rustc_mir_transform/src/ssa_range_prop.rs @@ -0,0 +1,203 @@ +//! A pass that propagates the known ranges of SSA locals. +//! We can know the ranges of SSA locals in certain locations for the following code: +//! ``` +//! fn foo(a: u32) { +//! let b = a < 9; // the integer representation of b is within the full range [0, 2). +//! if b { +//! let c = b; // c is true since b is within the range [1, 2). +//! let d = a < 8; // d is true since a is within the range [0, 9). +//! } +//! } +//! ``` +use rustc_abi::WrappingRange; +use rustc_const_eval::interpret::Scalar; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::graph::dominators::Dominators; +use rustc_index::bit_set::DenseBitSet; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::{BasicBlock, Body, Location, Operand, Place, TerminatorKind, *}; +use rustc_middle::ty::{TyCtxt, TypingEnv}; +use rustc_span::DUMMY_SP; + +use crate::ssa::SsaLocals; + +pub(super) struct SsaRangePropagation; + +impl<'tcx> crate::MirPass<'tcx> for SsaRangePropagation { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 1 + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let typing_env = body.typing_env(tcx); + let ssa = SsaLocals::new(tcx, body, typing_env); + // Clone dominators because we need them while mutating the body. + let dominators = body.basic_blocks.dominators().clone(); + let mut range_set = + RangeSet::new(tcx, typing_env, body, &ssa, &body.local_decls, dominators); + + let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec(); + for bb in reverse_postorder { + let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; + range_set.visit_basic_block_data(bb, data); + } + } + + fn is_required(&self) -> bool { + false + } +} + +struct RangeSet<'tcx, 'body, 'a> { + tcx: TyCtxt<'tcx>, + typing_env: TypingEnv<'tcx>, + ssa: &'a SsaLocals, + local_decls: &'body LocalDecls<'tcx>, + dominators: Dominators, + /// Known ranges at each locations. + ranges: FxHashMap, Vec<(Location, WrappingRange)>>, + /// Determines if the basic block has a single unique predecessor. + unique_predecessors: DenseBitSet, +} + +impl<'tcx, 'body, 'a> RangeSet<'tcx, 'body, 'a> { + fn new( + tcx: TyCtxt<'tcx>, + typing_env: TypingEnv<'tcx>, + body: &Body<'tcx>, + ssa: &'a SsaLocals, + local_decls: &'body LocalDecls<'tcx>, + dominators: Dominators, + ) -> Self { + let predecessors = body.basic_blocks.predecessors(); + let mut unique_predecessors = DenseBitSet::new_empty(body.basic_blocks.len()); + for bb in body.basic_blocks.indices() { + if predecessors[bb].len() == 1 { + unique_predecessors.insert(bb); + } + } + RangeSet { + tcx, + typing_env, + ssa, + local_decls, + dominators, + ranges: FxHashMap::default(), + unique_predecessors, + } + } + + /// Create a new known range at the location. + fn insert_range(&mut self, place: Place<'tcx>, location: Location, range: WrappingRange) { + assert!(self.is_ssa(place)); + self.ranges.entry(place).or_default().push((location, range)); + } + + /// Get the known range at the location. + fn get_range(&self, place: &Place<'tcx>, location: Location) -> Option { + let Some(ranges) = self.ranges.get(place) else { + return None; + }; + // FIXME: This should use the intersection of all valid ranges. + let (_, range) = + ranges.iter().find(|(range_loc, _)| range_loc.dominates(location, &self.dominators))?; + Some(*range) + } + + fn try_as_constant( + &mut self, + place: Place<'tcx>, + location: Location, + ) -> Option> { + if let Some(range) = self.get_range(&place, location) + && range.start == range.end + { + let ty = place.ty(self.local_decls, self.tcx).ty; + let layout = self.tcx.layout_of(self.typing_env.as_query_input(ty)).ok()?; + let value = ConstValue::Scalar(Scalar::from_uint(range.start, layout.size)); + let const_ = Const::Val(value, ty); + return Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_ }); + } + None + } + + fn is_ssa(&self, place: Place<'tcx>) -> bool { + self.ssa.is_ssa(place.local) && place.is_stable_offset() + } +} + +impl<'tcx> MutVisitor<'tcx> for RangeSet<'tcx, '_, '_> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { + // Attempts to simplify an operand to a constant value. + if let Some(place) = operand.place() + && let Some(const_) = self.try_as_constant(place, location) + { + *operand = Operand::Constant(Box::new(const_)); + }; + } + + fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { + self.super_terminator(terminator, location); + match &terminator.kind { + TerminatorKind::Assert { cond, expected, target, .. } => { + if let Some(place) = cond.place() + && self.is_ssa(place) + { + let successor = Location { block: *target, statement_index: 0 }; + if location.dominates(successor, &self.dominators) { + assert_ne!(location.block, successor.block); + let val = *expected as u128; + let range = WrappingRange { start: val, end: val }; + self.insert_range(place, successor, range); + } + } + } + TerminatorKind::SwitchInt { discr, targets } => { + if let Some(place) = discr.place() + && self.is_ssa(place) + // Reduce the potential compile-time overhead. + && targets.all_targets().len() < 16 + { + let mut distinct_targets: FxHashMap = FxHashMap::default(); + for (_, target) in targets.iter() { + let targets = distinct_targets.entry(target).or_default(); + *targets += 1; + } + for (val, target) in targets.iter() { + if distinct_targets[&target] != 1 { + // FIXME: For multiple targets, the range can be the union of their values. + continue; + } + let successor = Location { block: target, statement_index: 0 }; + if self.unique_predecessors.contains(successor.block) { + assert_ne!(location.block, successor.block); + let range = WrappingRange { start: val, end: val }; + self.insert_range(place, successor, range); + } + } + + // FIXME: The range for the otherwise target be extend to more types. + // For instance, `val` is within the range [4, 1) at the otherwise target of `matches!(val, 1 | 2 | 3)`. + let otherwise = Location { block: targets.otherwise(), statement_index: 0 }; + if place.ty(self.local_decls, self.tcx).ty.is_bool() + && let [val] = targets.all_values() + && self.unique_predecessors.contains(otherwise.block) + { + assert_ne!(location.block, otherwise.block); + let range = if val.get() == 0 { + WrappingRange { start: 1, end: 1 } + } else { + WrappingRange { start: 0, end: 0 } + }; + self.insert_range(place, otherwise, range); + } + } + } + _ => {} + } + } +} diff --git a/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff new file mode 100644 index 000000000000..ae3f49a8847b --- /dev/null +++ b/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff @@ -0,0 +1,69 @@ +- // MIR for `on_assert` before SsaRangePropagation ++ // MIR for `on_assert` after SsaRangePropagation + + fn on_assert(_1: usize, _2: &[u8]) -> u8 { + debug i => _1; + debug v => _2; + let mut _0: u8; + let _3: (); + let mut _4: bool; + let mut _5: usize; + let mut _6: usize; + let mut _7: &[u8]; + let mut _8: !; + let _9: usize; + let mut _10: usize; + let mut _11: bool; + scope 1 (inlined core::slice::::len) { + scope 2 (inlined std::ptr::metadata::<[u8]>) { + } + } + + bb0: { + StorageLive(_3); + nop; + StorageLive(_5); + _5 = copy _1; + nop; + StorageLive(_7); + _7 = &(*_2); + _6 = PtrMetadata(copy _2); + StorageDead(_7); + _4 = Lt(copy _1, copy _6); + switchInt(copy _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + nop; + StorageDead(_5); + _3 = const (); + nop; + StorageDead(_3); + StorageLive(_9); + _9 = copy _1; + _10 = copy _6; +- _11 = copy _4; +- assert(copy _4, "index out of bounds: the length is {} but the index is {}", copy _6, copy _1) -> [success: bb3, unwind unreachable]; ++ _11 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", copy _6, copy _1) -> [success: bb3, unwind unreachable]; + } + + bb2: { + nop; + StorageDead(_5); + StorageLive(_8); + _8 = panic(const "assertion failed: i < v.len()") -> unwind unreachable; + } + + bb3: { + _0 = copy (*_2)[_1]; + StorageDead(_9); + return; + } + } + + ALLOC0 (size: 29, align: 1) { + 0x00 │ 61 73 73 65 72 74 69 6f 6e 20 66 61 69 6c 65 64 │ assertion failed + 0x10 │ 3a 20 69 20 3c 20 76 2e 6c 65 6e 28 29 │ : i < v.len() + } + diff --git a/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff new file mode 100644 index 000000000000..2493e069edd4 --- /dev/null +++ b/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff @@ -0,0 +1,63 @@ +- // MIR for `on_if` before SsaRangePropagation ++ // MIR for `on_if` after SsaRangePropagation + + fn on_if(_1: usize, _2: &[u8]) -> u8 { + debug i => _1; + debug v => _2; + let mut _0: u8; + let mut _3: bool; + let mut _4: usize; + let mut _5: usize; + let mut _6: &[u8]; + let _7: usize; + let mut _8: usize; + let mut _9: bool; + scope 1 (inlined core::slice::::len) { + scope 2 (inlined std::ptr::metadata::<[u8]>) { + } + } + + bb0: { + nop; + StorageLive(_4); + _4 = copy _1; + nop; + StorageLive(_6); + _6 = &(*_2); + _5 = PtrMetadata(copy _2); + StorageDead(_6); + _3 = Lt(copy _1, copy _5); + switchInt(copy _3) -> [0: bb3, otherwise: bb1]; + } + + bb1: { + nop; + StorageDead(_4); + StorageLive(_7); + _7 = copy _1; + _8 = copy _5; +- _9 = copy _3; +- assert(copy _3, "index out of bounds: the length is {} but the index is {}", copy _5, copy _1) -> [success: bb2, unwind unreachable]; ++ _9 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", copy _5, copy _1) -> [success: bb2, unwind unreachable]; + } + + bb2: { + _0 = copy (*_2)[_1]; + StorageDead(_7); + goto -> bb4; + } + + bb3: { + nop; + StorageDead(_4); + _0 = const 0_u8; + goto -> bb4; + } + + bb4: { + nop; + return; + } + } + diff --git a/tests/mir-opt/range/ssa_range.on_if_2.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_if_2.SsaRangePropagation.diff new file mode 100644 index 000000000000..8a957238c845 --- /dev/null +++ b/tests/mir-opt/range/ssa_range.on_if_2.SsaRangePropagation.diff @@ -0,0 +1,20 @@ +- // MIR for `on_if_2` before SsaRangePropagation ++ // MIR for `on_if_2` after SsaRangePropagation + + fn on_if_2(_1: bool) -> bool { + let mut _0: bool; + + bb0: { + switchInt(copy _1) -> [1: bb2, otherwise: bb1]; + } + + bb1: { + goto -> bb2; + } + + bb2: { + _0 = copy _1; + return; + } + } + diff --git a/tests/mir-opt/range/ssa_range.on_match.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_match.SsaRangePropagation.diff new file mode 100644 index 000000000000..f91ac7090dde --- /dev/null +++ b/tests/mir-opt/range/ssa_range.on_match.SsaRangePropagation.diff @@ -0,0 +1,33 @@ +- // MIR for `on_match` before SsaRangePropagation ++ // MIR for `on_match` after SsaRangePropagation + + fn on_match(_1: u8) -> u8 { + debug i => _1; + let mut _0: u8; + + bb0: { + switchInt(copy _1) -> [1: bb3, 2: bb2, otherwise: bb1]; + } + + bb1: { + _0 = const 0_u8; + goto -> bb4; + } + + bb2: { +- _0 = copy _1; ++ _0 = const 2_u8; + goto -> bb4; + } + + bb3: { +- _0 = copy _1; ++ _0 = const 1_u8; + goto -> bb4; + } + + bb4: { + return; + } + } + diff --git a/tests/mir-opt/range/ssa_range.on_match_2.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_match_2.SsaRangePropagation.diff new file mode 100644 index 000000000000..53433d9fe4ba --- /dev/null +++ b/tests/mir-opt/range/ssa_range.on_match_2.SsaRangePropagation.diff @@ -0,0 +1,26 @@ +- // MIR for `on_match_2` before SsaRangePropagation ++ // MIR for `on_match_2` after SsaRangePropagation + + fn on_match_2(_1: u8) -> u8 { + debug i => _1; + let mut _0: u8; + + bb0: { + switchInt(copy _1) -> [1: bb2, 2: bb2, otherwise: bb1]; + } + + bb1: { + _0 = const 0_u8; + goto -> bb3; + } + + bb2: { + _0 = copy _1; + goto -> bb3; + } + + bb3: { + return; + } + } + diff --git a/tests/mir-opt/range/ssa_range.rs b/tests/mir-opt/range/ssa_range.rs new file mode 100644 index 000000000000..964d9b97cf92 --- /dev/null +++ b/tests/mir-opt/range/ssa_range.rs @@ -0,0 +1,70 @@ +//@ test-mir-pass: SsaRangePropagation +//@ compile-flags: -Zmir-enable-passes=+GVN,+Inline --crate-type=lib -Cpanic=abort + +#![feature(custom_mir, core_intrinsics)] + +use std::intrinsics::mir::*; + +// EMIT_MIR ssa_range.on_if.SsaRangePropagation.diff +pub fn on_if(i: usize, v: &[u8]) -> u8 { + // CHECK-LABEL: fn on_if( + // CHECK: assert(const true + if i < v.len() { v[i] } else { 0 } +} + +// EMIT_MIR ssa_range.on_assert.SsaRangePropagation.diff +pub fn on_assert(i: usize, v: &[u8]) -> u8 { + // CHECK-LABEL: fn on_assert( + // CHECK: assert(const true + assert!(i < v.len()); + v[i] +} + +// EMIT_MIR ssa_range.on_match.SsaRangePropagation.diff +pub fn on_match(i: u8) -> u8 { + // CHECK-LABEL: fn on_match( + // CHECK: switchInt(copy _1) -> [1: [[BB_V1:bb.*]], 2: [[BB_V2:bb.*]], + // CHECK: [[BB_V2]]: { + // CHECK-NEXT: _0 = const 2_u8; + // CHECK: [[BB_V1]]: { + // CHECK-NEXT: _0 = const 1_u8; + match i { + 1 => i, + 2 => i, + _ => 0, + } +} + +// EMIT_MIR ssa_range.on_match_2.SsaRangePropagation.diff +pub fn on_match_2(i: u8) -> u8 { + // CHECK-LABEL: fn on_match_2( + // CHECK: switchInt(copy _1) -> [1: [[BB:bb.*]], 2: [[BB]], + // CHECK: [[BB]]: { + // CHECK-NEXT: _0 = copy _1; + match i { + 1 | 2 => i, + _ => 0, + } +} + +// EMIT_MIR ssa_range.on_if_2.SsaRangePropagation.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn on_if_2(a: bool) -> bool { + // CHECK-LABEL: fn on_if_2( + // CHECK: _0 = copy _1; + mir! { + { + match a { + true => bb2, + _ => bb1 + } + } + bb1 = { + Goto(bb2) + } + bb2 = { + RET = a; + Return() + } + } +} From e9a67c74720911c531290c8ebc330e0fb5dc34d9 Mon Sep 17 00:00:00 2001 From: dianqk Date: Fri, 2 Jan 2026 22:15:40 +0800 Subject: [PATCH 0133/1843] Propagates assume --- .../rustc_mir_transform/src/ssa_range_prop.rs | 16 ++++++ ...a_range.on_assume.SsaRangePropagation.diff | 56 +++++++++++++++++++ tests/mir-opt/range/ssa_range.rs | 10 ++++ 3 files changed, 82 insertions(+) create mode 100644 tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff diff --git a/compiler/rustc_mir_transform/src/ssa_range_prop.rs b/compiler/rustc_mir_transform/src/ssa_range_prop.rs index 47e17de0560b..7a8be8efdfd1 100644 --- a/compiler/rustc_mir_transform/src/ssa_range_prop.rs +++ b/compiler/rustc_mir_transform/src/ssa_range_prop.rs @@ -140,6 +140,22 @@ impl<'tcx> MutVisitor<'tcx> for RangeSet<'tcx, '_, '_> { }; } + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + self.super_statement(statement, location); + match &statement.kind { + StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(operand)) => { + if let Some(place) = operand.place() + && self.is_ssa(place) + { + let successor = location.successor_within_block(); + let range = WrappingRange { start: 1, end: 1 }; + self.insert_range(place, successor, range); + } + } + _ => {} + } + } + fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { self.super_terminator(terminator, location); match &terminator.kind { diff --git a/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff new file mode 100644 index 000000000000..3be6f083604d --- /dev/null +++ b/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff @@ -0,0 +1,56 @@ +- // MIR for `on_assume` before SsaRangePropagation ++ // MIR for `on_assume` after SsaRangePropagation + + fn on_assume(_1: usize, _2: &[u8]) -> u8 { + debug i => _1; + debug v => _2; + let mut _0: u8; + let _3: (); + let _4: (); + let mut _5: bool; + let mut _6: usize; + let mut _7: usize; + let mut _8: &[u8]; + let _9: usize; + let mut _10: usize; + let mut _11: bool; + scope 1 (inlined core::slice::::len) { + scope 2 (inlined std::ptr::metadata::<[u8]>) { + } + } + + bb0: { + StorageLive(_3); + StorageLive(_4); + nop; + StorageLive(_6); + _6 = copy _1; + nop; + StorageLive(_8); + _8 = &(*_2); + _7 = PtrMetadata(copy _2); + StorageDead(_8); + _5 = Lt(copy _1, copy _7); + nop; + StorageDead(_6); + assume(copy _5); + nop; + StorageDead(_4); + _3 = const (); + StorageDead(_3); + StorageLive(_9); + _9 = copy _1; + _10 = copy _7; +- _11 = copy _5; +- assert(copy _5, "index out of bounds: the length is {} but the index is {}", copy _7, copy _1) -> [success: bb1, unwind unreachable]; ++ _11 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", copy _7, copy _1) -> [success: bb1, unwind unreachable]; + } + + bb1: { + _0 = copy (*_2)[_1]; + StorageDead(_9); + return; + } + } + diff --git a/tests/mir-opt/range/ssa_range.rs b/tests/mir-opt/range/ssa_range.rs index 964d9b97cf92..c8440a10a408 100644 --- a/tests/mir-opt/range/ssa_range.rs +++ b/tests/mir-opt/range/ssa_range.rs @@ -20,6 +20,16 @@ pub fn on_assert(i: usize, v: &[u8]) -> u8 { v[i] } +// EMIT_MIR ssa_range.on_assume.SsaRangePropagation.diff +pub fn on_assume(i: usize, v: &[u8]) -> u8 { + // CHECK-LABEL: fn on_assume( + // CHECK: assert(const true + unsafe { + std::intrinsics::assume(i < v.len()); + } + v[i] +} + // EMIT_MIR ssa_range.on_match.SsaRangePropagation.diff pub fn on_match(i: u8) -> u8 { // CHECK-LABEL: fn on_match( From fb5c85071b47a7d1c1c5db9381c0e8722507d350 Mon Sep 17 00:00:00 2001 From: human9000 Date: Mon, 5 Jan 2026 18:14:12 +0500 Subject: [PATCH 0134/1843] MGCA: literals support --- clippy_lints/src/utils/author.rs | 1 + clippy_utils/src/consts.rs | 2 +- clippy_utils/src/hir_utils.rs | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clippy_lints/src/utils/author.rs b/clippy_lints/src/utils/author.rs index 455f76edc904..7a54ba7a8fe1 100644 --- a/clippy_lints/src/utils/author.rs +++ b/clippy_lints/src/utils/author.rs @@ -324,6 +324,7 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { ConstArgKind::Tup(..) => chain!(self, "let ConstArgKind::Tup(..) = {const_arg}.kind"), ConstArgKind::Infer(..) => chain!(self, "let ConstArgKind::Infer(..) = {const_arg}.kind"), ConstArgKind::Error(..) => chain!(self, "let ConstArgKind::Error(..) = {const_arg}.kind"), + ConstArgKind::Literal(..) => chain!(self, "let ConstArgKind::Literal(..) = {const_arg}.kind") } } diff --git a/clippy_utils/src/consts.rs b/clippy_utils/src/consts.rs index 46b87fd5df96..5f4b87590dc1 100644 --- a/clippy_utils/src/consts.rs +++ b/clippy_utils/src/consts.rs @@ -1140,7 +1140,7 @@ pub fn const_item_rhs_to_expr<'tcx>(tcx: TyCtxt<'tcx>, ct_rhs: ConstItemRhs<'tcx ConstItemRhs::Body(body_id) => Some(tcx.hir_body(body_id).value), ConstItemRhs::TypeConst(const_arg) => match const_arg.kind { ConstArgKind::Anon(anon) => Some(tcx.hir_body(anon.body).value), - ConstArgKind::Struct(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Tup(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) | ConstArgKind::Infer(..) => { + ConstArgKind::Struct(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Tup(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) | ConstArgKind::Infer(..) | ConstArgKind::Literal(..) => { None }, }, diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs index 57c896c97172..b4e483ea8072 100644 --- a/clippy_utils/src/hir_utils.rs +++ b/clippy_utils/src/hir_utils.rs @@ -689,6 +689,9 @@ impl HirEqInterExpr<'_, '_, '_> { .zip(*args_b) .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) } + (ConstArgKind::Literal(kind_l), ConstArgKind::Literal(kind_r)) => { + kind_l == kind_r + }, // Use explicit match for now since ConstArg is undergoing flux. ( ConstArgKind::Path(..) @@ -697,6 +700,7 @@ impl HirEqInterExpr<'_, '_, '_> { | ConstArgKind::Tup(..) | ConstArgKind::Infer(..) | ConstArgKind::Struct(..) + | ConstArgKind::Literal(..) | ConstArgKind::Error(..), _, ) => false, @@ -1577,6 +1581,7 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> { } }, ConstArgKind::Infer(..) | ConstArgKind::Error(..) => {}, + ConstArgKind::Literal(lit) => lit.hash(&mut self.s) } } From 331f75f4a773fadba135664190905457946ea98e Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Thu, 8 Jan 2026 16:37:55 +0100 Subject: [PATCH 0135/1843] Do not warn on arithmetic side effect for `String`+`String` The previous fix only handled `String`+`str`. --- .../src/operators/arithmetic_side_effects.rs | 5 +- tests/ui/arithmetic_side_effects.rs | 1 - tests/ui/arithmetic_side_effects.stderr | 266 +++++++++--------- 3 files changed, 134 insertions(+), 138 deletions(-) diff --git a/clippy_lints/src/operators/arithmetic_side_effects.rs b/clippy_lints/src/operators/arithmetic_side_effects.rs index 91a069559f7b..106286d16d18 100644 --- a/clippy_lints/src/operators/arithmetic_side_effects.rs +++ b/clippy_lints/src/operators/arithmetic_side_effects.rs @@ -33,7 +33,10 @@ impl ArithmeticSideEffects { allowed_binary.extend([ ("f32", FxHashSet::from_iter(["f32"])), ("f64", FxHashSet::from_iter(["f64"])), - ("std::string::String", FxHashSet::from_iter(["str"])), + ( + "std::string::String", + FxHashSet::from_iter(["str", "std::string::String"]), + ), ]); for (lhs, rhs) in &conf.arithmetic_side_effects_allowed_binary { allowed_binary.entry(lhs).or_default().insert(rhs); diff --git a/tests/ui/arithmetic_side_effects.rs b/tests/ui/arithmetic_side_effects.rs index b7ed596d811e..87397a549bf4 100644 --- a/tests/ui/arithmetic_side_effects.rs +++ b/tests/ui/arithmetic_side_effects.rs @@ -174,7 +174,6 @@ pub fn hard_coded_allowed() { let _ = Saturating(0u32) + Saturating(0u32); let _ = String::new() + ""; let _ = String::new() + &String::new(); - //~^ arithmetic_side_effects let _ = Wrapping(0u32) + Wrapping(0u32); let saturating: Saturating = Saturating(0u32); diff --git a/tests/ui/arithmetic_side_effects.stderr b/tests/ui/arithmetic_side_effects.stderr index 22742a82601a..2767a051786e 100644 --- a/tests/ui/arithmetic_side_effects.stderr +++ b/tests/ui/arithmetic_side_effects.stderr @@ -14,784 +14,778 @@ LL | let _ = 1f128 + 1f128; | ^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:176:13 - | -LL | let _ = String::new() + &String::new(); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:312:5 + --> tests/ui/arithmetic_side_effects.rs:311:5 | LL | _n += 1; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:314:5 + --> tests/ui/arithmetic_side_effects.rs:313:5 | LL | _n += &1; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:316:5 + --> tests/ui/arithmetic_side_effects.rs:315:5 | LL | _n -= 1; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:318:5 + --> tests/ui/arithmetic_side_effects.rs:317:5 | LL | _n -= &1; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:320:5 + --> tests/ui/arithmetic_side_effects.rs:319:5 | LL | _n /= 0; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:322:5 + --> tests/ui/arithmetic_side_effects.rs:321:5 | LL | _n /= &0; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:324:5 + --> tests/ui/arithmetic_side_effects.rs:323:5 | LL | _n %= 0; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:326:5 + --> tests/ui/arithmetic_side_effects.rs:325:5 | LL | _n %= &0; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:328:5 + --> tests/ui/arithmetic_side_effects.rs:327:5 | LL | _n *= 2; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:330:5 + --> tests/ui/arithmetic_side_effects.rs:329:5 | LL | _n *= &2; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:332:5 + --> tests/ui/arithmetic_side_effects.rs:331:5 | LL | _n += -1; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:334:5 + --> tests/ui/arithmetic_side_effects.rs:333:5 | LL | _n += &-1; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:336:5 + --> tests/ui/arithmetic_side_effects.rs:335:5 | LL | _n -= -1; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:338:5 + --> tests/ui/arithmetic_side_effects.rs:337:5 | LL | _n -= &-1; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:340:5 + --> tests/ui/arithmetic_side_effects.rs:339:5 | LL | _n /= -0; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:342:5 + --> tests/ui/arithmetic_side_effects.rs:341:5 | LL | _n /= &-0; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:344:5 + --> tests/ui/arithmetic_side_effects.rs:343:5 | LL | _n %= -0; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:346:5 + --> tests/ui/arithmetic_side_effects.rs:345:5 | LL | _n %= &-0; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:348:5 + --> tests/ui/arithmetic_side_effects.rs:347:5 | LL | _n *= -2; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:350:5 + --> tests/ui/arithmetic_side_effects.rs:349:5 | LL | _n *= &-2; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:352:5 + --> tests/ui/arithmetic_side_effects.rs:351:5 | LL | _custom += Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:354:5 + --> tests/ui/arithmetic_side_effects.rs:353:5 | LL | _custom += &Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:356:5 + --> tests/ui/arithmetic_side_effects.rs:355:5 | LL | _custom -= Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:358:5 + --> tests/ui/arithmetic_side_effects.rs:357:5 | LL | _custom -= &Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:360:5 + --> tests/ui/arithmetic_side_effects.rs:359:5 | LL | _custom /= Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:362:5 + --> tests/ui/arithmetic_side_effects.rs:361:5 | LL | _custom /= &Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:364:5 + --> tests/ui/arithmetic_side_effects.rs:363:5 | LL | _custom %= Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:366:5 + --> tests/ui/arithmetic_side_effects.rs:365:5 | LL | _custom %= &Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:368:5 + --> tests/ui/arithmetic_side_effects.rs:367:5 | LL | _custom *= Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:370:5 + --> tests/ui/arithmetic_side_effects.rs:369:5 | LL | _custom *= &Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:372:5 + --> tests/ui/arithmetic_side_effects.rs:371:5 | LL | _custom >>= Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:374:5 + --> tests/ui/arithmetic_side_effects.rs:373:5 | LL | _custom >>= &Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:376:5 + --> tests/ui/arithmetic_side_effects.rs:375:5 | LL | _custom <<= Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:378:5 + --> tests/ui/arithmetic_side_effects.rs:377:5 | LL | _custom <<= &Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:380:5 + --> tests/ui/arithmetic_side_effects.rs:379:5 | LL | _custom += -Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:382:5 + --> tests/ui/arithmetic_side_effects.rs:381:5 | LL | _custom += &-Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:384:5 + --> tests/ui/arithmetic_side_effects.rs:383:5 | LL | _custom -= -Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:386:5 + --> tests/ui/arithmetic_side_effects.rs:385:5 | LL | _custom -= &-Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:388:5 + --> tests/ui/arithmetic_side_effects.rs:387:5 | LL | _custom /= -Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:390:5 + --> tests/ui/arithmetic_side_effects.rs:389:5 | LL | _custom /= &-Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:392:5 + --> tests/ui/arithmetic_side_effects.rs:391:5 | LL | _custom %= -Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:394:5 + --> tests/ui/arithmetic_side_effects.rs:393:5 | LL | _custom %= &-Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:396:5 + --> tests/ui/arithmetic_side_effects.rs:395:5 | LL | _custom *= -Custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:398:5 + --> tests/ui/arithmetic_side_effects.rs:397:5 | LL | _custom *= &-Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:400:5 + --> tests/ui/arithmetic_side_effects.rs:399:5 | LL | _custom >>= -Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:402:5 + --> tests/ui/arithmetic_side_effects.rs:401:5 | LL | _custom >>= &-Custom; | ^^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:404:5 + --> tests/ui/arithmetic_side_effects.rs:403:5 | LL | _custom <<= -Custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:406:5 + --> tests/ui/arithmetic_side_effects.rs:405:5 | LL | _custom <<= &-Custom; | ^^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:410:10 + --> tests/ui/arithmetic_side_effects.rs:409:10 | LL | _n = _n + 1; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:412:10 + --> tests/ui/arithmetic_side_effects.rs:411:10 | LL | _n = _n + &1; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:414:10 + --> tests/ui/arithmetic_side_effects.rs:413:10 | LL | _n = 1 + _n; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:416:10 + --> tests/ui/arithmetic_side_effects.rs:415:10 | LL | _n = &1 + _n; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:418:10 + --> tests/ui/arithmetic_side_effects.rs:417:10 | LL | _n = _n - 1; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:420:10 + --> tests/ui/arithmetic_side_effects.rs:419:10 | LL | _n = _n - &1; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:422:10 + --> tests/ui/arithmetic_side_effects.rs:421:10 | LL | _n = 1 - _n; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:424:10 + --> tests/ui/arithmetic_side_effects.rs:423:10 | LL | _n = &1 - _n; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:426:10 + --> tests/ui/arithmetic_side_effects.rs:425:10 | LL | _n = _n / 0; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:428:10 + --> tests/ui/arithmetic_side_effects.rs:427:10 | LL | _n = _n / &0; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:430:10 + --> tests/ui/arithmetic_side_effects.rs:429:10 | LL | _n = _n % 0; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:432:10 + --> tests/ui/arithmetic_side_effects.rs:431:10 | LL | _n = _n % &0; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:434:10 + --> tests/ui/arithmetic_side_effects.rs:433:10 | LL | _n = _n * 2; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:436:10 + --> tests/ui/arithmetic_side_effects.rs:435:10 | LL | _n = _n * &2; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:438:10 + --> tests/ui/arithmetic_side_effects.rs:437:10 | LL | _n = 2 * _n; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:440:10 + --> tests/ui/arithmetic_side_effects.rs:439:10 | LL | _n = &2 * _n; | ^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:442:10 + --> tests/ui/arithmetic_side_effects.rs:441:10 | LL | _n = 23 + &85; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:444:10 + --> tests/ui/arithmetic_side_effects.rs:443:10 | LL | _n = &23 + 85; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:446:10 + --> tests/ui/arithmetic_side_effects.rs:445:10 | LL | _n = &23 + &85; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:448:15 + --> tests/ui/arithmetic_side_effects.rs:447:15 | LL | _custom = _custom + _custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:450:15 + --> tests/ui/arithmetic_side_effects.rs:449:15 | LL | _custom = _custom + &_custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:452:15 + --> tests/ui/arithmetic_side_effects.rs:451:15 | LL | _custom = Custom + _custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:454:15 + --> tests/ui/arithmetic_side_effects.rs:453:15 | LL | _custom = &Custom + _custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:456:15 + --> tests/ui/arithmetic_side_effects.rs:455:15 | LL | _custom = _custom - Custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:458:15 + --> tests/ui/arithmetic_side_effects.rs:457:15 | LL | _custom = _custom - &Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:460:15 + --> tests/ui/arithmetic_side_effects.rs:459:15 | LL | _custom = Custom - _custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:462:15 + --> tests/ui/arithmetic_side_effects.rs:461:15 | LL | _custom = &Custom - _custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:464:15 + --> tests/ui/arithmetic_side_effects.rs:463:15 | LL | _custom = _custom / Custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:466:15 + --> tests/ui/arithmetic_side_effects.rs:465:15 | LL | _custom = _custom / &Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:468:15 + --> tests/ui/arithmetic_side_effects.rs:467:15 | LL | _custom = _custom % Custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:470:15 + --> tests/ui/arithmetic_side_effects.rs:469:15 | LL | _custom = _custom % &Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:472:15 + --> tests/ui/arithmetic_side_effects.rs:471:15 | LL | _custom = _custom * Custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:474:15 + --> tests/ui/arithmetic_side_effects.rs:473:15 | LL | _custom = _custom * &Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:476:15 + --> tests/ui/arithmetic_side_effects.rs:475:15 | LL | _custom = Custom * _custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:478:15 + --> tests/ui/arithmetic_side_effects.rs:477:15 | LL | _custom = &Custom * _custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:480:15 + --> tests/ui/arithmetic_side_effects.rs:479:15 | LL | _custom = Custom + &Custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:482:15 + --> tests/ui/arithmetic_side_effects.rs:481:15 | LL | _custom = &Custom + Custom; | ^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:484:15 + --> tests/ui/arithmetic_side_effects.rs:483:15 | LL | _custom = &Custom + &Custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:486:15 + --> tests/ui/arithmetic_side_effects.rs:485:15 | LL | _custom = _custom >> _custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:488:15 + --> tests/ui/arithmetic_side_effects.rs:487:15 | LL | _custom = _custom >> &_custom; | ^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:490:15 + --> tests/ui/arithmetic_side_effects.rs:489:15 | LL | _custom = Custom << _custom; | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:492:15 + --> tests/ui/arithmetic_side_effects.rs:491:15 | LL | _custom = &Custom << _custom; | ^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:496:23 + --> tests/ui/arithmetic_side_effects.rs:495:23 | LL | _n.saturating_div(0); | ^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:498:21 + --> tests/ui/arithmetic_side_effects.rs:497:21 | LL | _n.wrapping_div(0); | ^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:500:21 + --> tests/ui/arithmetic_side_effects.rs:499:21 | LL | _n.wrapping_rem(0); | ^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:502:28 + --> tests/ui/arithmetic_side_effects.rs:501:28 | LL | _n.wrapping_rem_euclid(0); | ^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:505:23 + --> tests/ui/arithmetic_side_effects.rs:504:23 | LL | _n.saturating_div(_n); | ^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:507:21 + --> tests/ui/arithmetic_side_effects.rs:506:21 | LL | _n.wrapping_div(_n); | ^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:509:21 + --> tests/ui/arithmetic_side_effects.rs:508:21 | LL | _n.wrapping_rem(_n); | ^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:511:28 + --> tests/ui/arithmetic_side_effects.rs:510:28 | LL | _n.wrapping_rem_euclid(_n); | ^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:514:23 + --> tests/ui/arithmetic_side_effects.rs:513:23 | LL | _n.saturating_div(*Box::new(_n)); | ^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:518:10 + --> tests/ui/arithmetic_side_effects.rs:517:10 | LL | _n = -_n; | ^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:520:10 + --> tests/ui/arithmetic_side_effects.rs:519:10 | LL | _n = -&_n; | ^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:522:15 + --> tests/ui/arithmetic_side_effects.rs:521:15 | LL | _custom = -_custom; | ^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:524:15 + --> tests/ui/arithmetic_side_effects.rs:523:15 | LL | _custom = -&_custom; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:526:9 + --> tests/ui/arithmetic_side_effects.rs:525:9 | LL | _ = -*Box::new(_n); | ^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:536:5 + --> tests/ui/arithmetic_side_effects.rs:535:5 | LL | 1 + i; | ^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:538:5 + --> tests/ui/arithmetic_side_effects.rs:537:5 | LL | i * 2; | ^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:540:5 + --> tests/ui/arithmetic_side_effects.rs:539:5 | LL | 1 % i / 2; | ^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:542:5 + --> tests/ui/arithmetic_side_effects.rs:541:5 | LL | i - 2 + 2 - i; | ^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:544:5 + --> tests/ui/arithmetic_side_effects.rs:543:5 | LL | -i; | ^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:556:5 + --> tests/ui/arithmetic_side_effects.rs:555:5 | LL | i += 1; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:558:5 + --> tests/ui/arithmetic_side_effects.rs:557:5 | LL | i -= 1; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:560:5 + --> tests/ui/arithmetic_side_effects.rs:559:5 | LL | i *= 2; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:563:5 + --> tests/ui/arithmetic_side_effects.rs:562:5 | LL | i /= 0; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:566:5 + --> tests/ui/arithmetic_side_effects.rs:565:5 | LL | i /= var1; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:568:5 + --> tests/ui/arithmetic_side_effects.rs:567:5 | LL | i /= var2; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:571:5 + --> tests/ui/arithmetic_side_effects.rs:570:5 | LL | i %= 0; | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:574:5 + --> tests/ui/arithmetic_side_effects.rs:573:5 | LL | i %= var1; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:576:5 + --> tests/ui/arithmetic_side_effects.rs:575:5 | LL | i %= var2; | ^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:587:5 + --> tests/ui/arithmetic_side_effects.rs:586:5 | LL | 10 / a | ^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:642:9 + --> tests/ui/arithmetic_side_effects.rs:641:9 | LL | x / maybe_zero | ^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:647:9 + --> tests/ui/arithmetic_side_effects.rs:646:9 | LL | x % maybe_zero | ^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:659:5 + --> tests/ui/arithmetic_side_effects.rs:658:5 | LL | one.add_assign(1); | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:664:5 + --> tests/ui/arithmetic_side_effects.rs:663:5 | LL | one.sub_assign(1); | ^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:685:5 + --> tests/ui/arithmetic_side_effects.rs:684:5 | LL | one.add(&one); | ^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:687:5 + --> tests/ui/arithmetic_side_effects.rs:686:5 | LL | Box::new(one).add(one); | ^^^^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:696:13 + --> tests/ui/arithmetic_side_effects.rs:695:13 | LL | let _ = u128::MAX + u128::from(1u8); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:713:13 + --> tests/ui/arithmetic_side_effects.rs:712:13 | LL | let _ = u128::MAX * u128::from(1u8); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:736:33 + --> tests/ui/arithmetic_side_effects.rs:735:33 | LL | let _ = Duration::from_secs(86400 * Foo::from(1)); | ^^^^^^^^^^^^^^^^^^^^ error: arithmetic operation that can potentially result in unexpected side-effects - --> tests/ui/arithmetic_side_effects.rs:742:33 + --> tests/ui/arithmetic_side_effects.rs:741:33 | LL | let _ = Duration::from_secs(86400 * shift(1)); | ^^^^^^^^^^^^^^^^ -error: aborting due to 132 previous errors +error: aborting due to 131 previous errors From fe0facf0f6f021f0c5d597fb214d96ae48571c91 Mon Sep 17 00:00:00 2001 From: Martin Nordholts Date: Thu, 1 Jan 2026 10:47:22 +0100 Subject: [PATCH 0136/1843] Finish transition from `semitransparent` to `semiopaque` for `rustc_macro_transparency` --- example/mini_core.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/mini_core.rs b/example/mini_core.rs index 64a5b431bd07..0aba44a88c5a 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -748,25 +748,25 @@ extern "C" { pub struct VaList<'a>(&'a mut VaListImpl); #[rustc_builtin_macro] -#[rustc_macro_transparency = "semitransparent"] +#[rustc_macro_transparency = "semiopaque"] pub macro stringify($($t:tt)*) { /* compiler built-in */ } #[rustc_builtin_macro] -#[rustc_macro_transparency = "semitransparent"] +#[rustc_macro_transparency = "semiopaque"] pub macro file() { /* compiler built-in */ } #[rustc_builtin_macro] -#[rustc_macro_transparency = "semitransparent"] +#[rustc_macro_transparency = "semiopaque"] pub macro line() { /* compiler built-in */ } #[rustc_builtin_macro] -#[rustc_macro_transparency = "semitransparent"] +#[rustc_macro_transparency = "semiopaque"] pub macro cfg() { /* compiler built-in */ } From a4f0937a9a28b4e1faacfc3ae1281e568d0ad59c Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Wed, 7 Jan 2026 11:03:40 +0100 Subject: [PATCH 0137/1843] clean-up --- .../src/methods/unnecessary_sort_by.rs | 71 ++++++++----------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/clippy_lints/src/methods/unnecessary_sort_by.rs b/clippy_lints/src/methods/unnecessary_sort_by.rs index 4a3e4c092f3b..dbaa801eaf50 100644 --- a/clippy_lints/src/methods/unnecessary_sort_by.rs +++ b/clippy_lints/src/methods/unnecessary_sort_by.rs @@ -9,7 +9,7 @@ use rustc_lint::LateContext; use rustc_middle::ty; use rustc_middle::ty::GenericArgKind; use rustc_span::sym; -use rustc_span::symbol::Ident; +use rustc_span::symbol::{Ident, Symbol}; use std::iter; use super::UNNECESSARY_SORT_BY; @@ -20,29 +20,29 @@ enum LintTrigger { } struct SortDetection { - vec_name: String, + vec_name: Sugg<'static>, } struct SortByKeyDetection { - vec_name: String, - closure_arg: String, - closure_body: String, + vec_name: Sugg<'static>, + closure_arg: Symbol, + closure_body: Sugg<'static>, reverse: bool, } /// Detect if the two expressions are mirrored (identical, except one /// contains a and the other replaces it with b) -fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: &Ident, b_expr: &Expr<'_>, b_ident: &Ident) -> bool { - match (&a_expr.kind, &b_expr.kind) { +fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: Ident, b_expr: &Expr<'_>, b_ident: Ident) -> bool { + match (a_expr.kind, b_expr.kind) { // Two arrays with mirrored contents (ExprKind::Array(left_exprs), ExprKind::Array(right_exprs)) => { - iter::zip(*left_exprs, *right_exprs).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) + iter::zip(left_exprs, right_exprs).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) }, // The two exprs are function calls. // Check to see that the function itself and its arguments are mirrored (ExprKind::Call(left_expr, left_args), ExprKind::Call(right_expr, right_args)) => { mirrored_exprs(left_expr, a_ident, right_expr, b_ident) - && iter::zip(*left_args, *right_args).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) + && iter::zip(left_args, right_args).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) }, // The two exprs are method calls. // Check to see that the function is the same and the arguments and receivers are mirrored @@ -51,12 +51,12 @@ fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: &Ident, b_expr: &Expr<'_>, b_ident ExprKind::MethodCall(right_segment, right_receiver, right_args, _), ) => { left_segment.ident == right_segment.ident - && iter::zip(*left_args, *right_args).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) + && iter::zip(left_args, right_args).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) && mirrored_exprs(left_receiver, a_ident, right_receiver, b_ident) }, // Two tuples with mirrored contents (ExprKind::Tup(left_exprs), ExprKind::Tup(right_exprs)) => { - iter::zip(*left_exprs, *right_exprs).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) + iter::zip(left_exprs, right_exprs).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident)) }, // Two binary ops, which are the same operation and which have mirrored arguments (ExprKind::Binary(left_op, left_left, left_right), ExprKind::Binary(right_op, right_left, right_right)) => { @@ -81,27 +81,27 @@ fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: &Ident, b_expr: &Expr<'_>, b_ident ( ExprKind::Path(QPath::Resolved( _, - Path { + &Path { segments: left_segments, .. }, )), ExprKind::Path(QPath::Resolved( _, - Path { + &Path { segments: right_segments, .. }, )), ) => { - (iter::zip(*left_segments, *right_segments).all(|(left, right)| left.ident == right.ident) + (iter::zip(left_segments, right_segments).all(|(left, right)| left.ident == right.ident) && left_segments .iter() - .all(|seg| &seg.ident != a_ident && &seg.ident != b_ident)) + .all(|seg| seg.ident != a_ident && seg.ident != b_ident)) || (left_segments.len() == 1 - && &left_segments[0].ident == a_ident + && left_segments[0].ident == a_ident && right_segments.len() == 1 - && &right_segments[0].ident == b_ident) + && right_segments[0].ident == b_ident) }, // Matching expressions, but one or both is borrowed ( @@ -123,7 +123,7 @@ fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, recv: &Expr<'_>, arg: &Exp && let &[ Param { pat: - Pat { + &Pat { kind: PatKind::Binding(_, _, left_ident, _), .. }, @@ -131,36 +131,26 @@ fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, recv: &Expr<'_>, arg: &Exp }, Param { pat: - Pat { + &Pat { kind: PatKind::Binding(_, _, right_ident, _), .. }, .. }, - ] = &closure_body.params + ] = closure_body.params && let ExprKind::MethodCall(method_path, left_expr, [right_expr], _) = closure_body.value.kind && method_path.ident.name == sym::cmp - && cx - .ty_based_def(closure_body.value) - .opt_parent(cx) - .is_diag_item(cx, sym::Ord) + && let Some(ord_trait) = cx.tcx.get_diagnostic_item(sym::Ord) + && cx.ty_based_def(closure_body.value).opt_parent(cx).opt_def_id() == Some(ord_trait) { let (closure_body, closure_arg, reverse) = if mirrored_exprs(left_expr, left_ident, right_expr, right_ident) { - ( - Sugg::hir(cx, left_expr, "..").to_string(), - left_ident.name.to_string(), - false, - ) + (Sugg::hir(cx, left_expr, "_"), left_ident.name, false) } else if mirrored_exprs(left_expr, right_ident, right_expr, left_ident) { - ( - Sugg::hir(cx, left_expr, "..").to_string(), - right_ident.name.to_string(), - true, - ) + (Sugg::hir(cx, left_expr, "_"), right_ident.name, true) } else { return None; }; - let vec_name = Sugg::hir(cx, recv, "..").to_string(); + let vec_name = Sugg::hir(cx, recv, "(_)"); if let ExprKind::Path(QPath::Resolved( _, @@ -168,12 +158,9 @@ fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, recv: &Expr<'_>, arg: &Exp segments: [PathSegment { ident: left_name, .. }], .. }, - )) = &left_expr.kind - && left_name == left_ident - && cx - .tcx - .get_diagnostic_item(sym::Ord) - .is_some_and(|id| implements_trait(cx, cx.typeck_results().expr_ty(left_expr), id, &[])) + )) = left_expr.kind + && *left_name == left_ident + && implements_trait(cx, cx.typeck_results().expr_ty(left_expr), ord_trait, &[]) { return Some(LintTrigger::Sort(SortDetection { vec_name })); } @@ -226,7 +213,7 @@ pub(super) fn check<'tcx>( { format!("{}::cmp::Reverse({})", std_or_core, trigger.closure_body) } else { - trigger.closure_body + trigger.closure_body.to_string() }, ), if trigger.reverse { From 0cfc4ee541d819e7cf93cdebb1c43fd0d9eb444b Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Wed, 7 Jan 2026 16:25:41 +0100 Subject: [PATCH 0138/1843] fix: respect applicability reduction due to `Sugg` By postponing the creation of `Sugg`s, we can properly account for their effect on applicability --- .../src/methods/unnecessary_sort_by.rs | 83 ++++++++++--------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/clippy_lints/src/methods/unnecessary_sort_by.rs b/clippy_lints/src/methods/unnecessary_sort_by.rs index dbaa801eaf50..c8cb85c05156 100644 --- a/clippy_lints/src/methods/unnecessary_sort_by.rs +++ b/clippy_lints/src/methods/unnecessary_sort_by.rs @@ -1,4 +1,4 @@ -use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::{MaybeDef, MaybeTypeckRes}; use clippy_utils::std_or_core; use clippy_utils::sugg::Sugg; @@ -14,19 +14,14 @@ use std::iter; use super::UNNECESSARY_SORT_BY; -enum LintTrigger { - Sort(SortDetection), - SortByKey(SortByKeyDetection), +enum LintTrigger<'tcx> { + Sort, + SortByKey(SortByKeyDetection<'tcx>), } -struct SortDetection { - vec_name: Sugg<'static>, -} - -struct SortByKeyDetection { - vec_name: Sugg<'static>, +struct SortByKeyDetection<'tcx> { closure_arg: Symbol, - closure_body: Sugg<'static>, + closure_body: &'tcx Expr<'tcx>, reverse: bool, } @@ -114,7 +109,7 @@ fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: Ident, b_expr: &Expr<'_>, b_ident: } } -fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, recv: &Expr<'_>, arg: &Expr<'_>) -> Option { +fn detect_lint<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, arg: &Expr<'_>) -> Option> { if let Some(method_id) = cx.typeck_results().type_dependent_def_id(expr.hir_id) && let Some(impl_id) = cx.tcx.impl_of_assoc(method_id) && cx.tcx.type_of(impl_id).instantiate_identity().is_slice() @@ -144,13 +139,12 @@ fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, recv: &Expr<'_>, arg: &Exp && cx.ty_based_def(closure_body.value).opt_parent(cx).opt_def_id() == Some(ord_trait) { let (closure_body, closure_arg, reverse) = if mirrored_exprs(left_expr, left_ident, right_expr, right_ident) { - (Sugg::hir(cx, left_expr, "_"), left_ident.name, false) + (left_expr, left_ident.name, false) } else if mirrored_exprs(left_expr, right_ident, right_expr, left_ident) { - (Sugg::hir(cx, left_expr, "_"), right_ident.name, true) + (left_expr, right_ident.name, true) } else { return None; }; - let vec_name = Sugg::hir(cx, recv, "(_)"); if let ExprKind::Path(QPath::Resolved( _, @@ -162,12 +156,11 @@ fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, recv: &Expr<'_>, arg: &Exp && *left_name == left_ident && implements_trait(cx, cx.typeck_results().expr_ty(left_expr), ord_trait, &[]) { - return Some(LintTrigger::Sort(SortDetection { vec_name })); + return Some(LintTrigger::Sort); } if !expr_borrows(cx, left_expr) { return Some(LintTrigger::SortByKey(SortByKeyDetection { - vec_name, closure_arg, closure_body, reverse, @@ -190,49 +183,57 @@ pub(super) fn check<'tcx>( arg: &'tcx Expr<'_>, is_unstable: bool, ) { - match detect_lint(cx, expr, recv, arg) { + match detect_lint(cx, expr, arg) { Some(LintTrigger::SortByKey(trigger)) => { let method = if is_unstable { "sort_unstable_by_key" } else { "sort_by_key" }; - span_lint_and_sugg( + span_lint_and_then( cx, UNNECESSARY_SORT_BY, expr.span, format!("consider using `{method}`"), - "try", - format!( - "{}.{}(|{}| {})", - trigger.vec_name, - method, - trigger.closure_arg, - if let Some(std_or_core) = std_or_core(cx) - && trigger.reverse - { - format!("{}::cmp::Reverse({})", std_or_core, trigger.closure_body) + |diag| { + let mut app = if trigger.reverse { + Applicability::MaybeIncorrect } else { - trigger.closure_body.to_string() - }, - ), - if trigger.reverse { - Applicability::MaybeIncorrect - } else { - Applicability::MachineApplicable + Applicability::MachineApplicable + }; + let recv = Sugg::hir_with_applicability(cx, recv, "(_)", &mut app); + let closure_body = Sugg::hir_with_applicability(cx, trigger.closure_body, "_", &mut app); + diag.span_suggestion( + expr.span, + "try", + format!( + "{recv}.{method}(|{}| {})", + trigger.closure_arg, + if let Some(std_or_core) = std_or_core(cx) + && trigger.reverse + { + format!("{std_or_core}::cmp::Reverse({closure_body})") + } else { + closure_body.to_string() + }, + ), + app, + ); }, ); }, - Some(LintTrigger::Sort(trigger)) => { + Some(LintTrigger::Sort) => { let method = if is_unstable { "sort_unstable" } else { "sort" }; - span_lint_and_sugg( + span_lint_and_then( cx, UNNECESSARY_SORT_BY, expr.span, format!("consider using `{method}`"), - "try", - format!("{}.{}()", trigger.vec_name, method), - Applicability::MachineApplicable, + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, recv, "(_)", &mut app); + diag.span_suggestion(expr.span, "try", format!("{recv}.{method}()"), app); + }, ); }, None => {}, From fe9715b5e8fb9e425fa14ce2830a3faafd3adf78 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 8 Jan 2026 18:44:38 +0000 Subject: [PATCH 0139/1843] Remove support for ScalarPair unadjusted arguments --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 13402edd13e2..d7a928f492f0 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -668,11 +668,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { if arg_layout.is_zst() { continue; } - if let BackendRepr::ScalarPair(_, _) = arg_layout.backend_repr { - llargument_tys.push(arg_layout.scalar_pair_element_llvm_type(self, 0, true)); - llargument_tys.push(arg_layout.scalar_pair_element_llvm_type(self, 1, true)); - continue; - } llargument_tys.push(arg_layout.immediate_llvm_type(self)); } From bdb1050a0f433be29982de0668102ec15a2c9b96 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Wed, 7 Jan 2026 16:42:15 +0100 Subject: [PATCH 0140/1843] fix: don't lint if `std` or `core` are required for a suggestion but unavailable --- .../src/methods/unnecessary_sort_by.rs | 23 ++++++++-------- tests/ui/unnecessary_sort_by_no_core.rs | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 tests/ui/unnecessary_sort_by_no_core.rs diff --git a/clippy_lints/src/methods/unnecessary_sort_by.rs b/clippy_lints/src/methods/unnecessary_sort_by.rs index c8cb85c05156..9dddbe814317 100644 --- a/clippy_lints/src/methods/unnecessary_sort_by.rs +++ b/clippy_lints/src/methods/unnecessary_sort_by.rs @@ -190,6 +190,12 @@ pub(super) fn check<'tcx>( } else { "sort_by_key" }; + let Some(std_or_core) = std_or_core(cx) else { + // To make it this far the crate has to reference diagnostic items defined in core. Either this is + // the `core` crate, there's an `extern crate core` somewhere, or another crate is defining the + // diagnostic items. It's fine to not lint in all those cases even if we might be able to. + return; + }; span_lint_and_then( cx, UNNECESSARY_SORT_BY, @@ -203,20 +209,15 @@ pub(super) fn check<'tcx>( }; let recv = Sugg::hir_with_applicability(cx, recv, "(_)", &mut app); let closure_body = Sugg::hir_with_applicability(cx, trigger.closure_body, "_", &mut app); + let closure_body = if trigger.reverse { + format!("{std_or_core}::cmp::Reverse({closure_body})") + } else { + closure_body.to_string() + }; diag.span_suggestion( expr.span, "try", - format!( - "{recv}.{method}(|{}| {})", - trigger.closure_arg, - if let Some(std_or_core) = std_or_core(cx) - && trigger.reverse - { - format!("{std_or_core}::cmp::Reverse({closure_body})") - } else { - closure_body.to_string() - }, - ), + format!("{recv}.{method}(|{}| {})", trigger.closure_arg, closure_body), app, ); }, diff --git a/tests/ui/unnecessary_sort_by_no_core.rs b/tests/ui/unnecessary_sort_by_no_core.rs new file mode 100644 index 000000000000..cb0da6339cb3 --- /dev/null +++ b/tests/ui/unnecessary_sort_by_no_core.rs @@ -0,0 +1,26 @@ +//@check-pass +#![feature(no_core)] +#![no_std] +#![no_core] +extern crate alloc; +extern crate core as mycore; +use alloc::vec; +use alloc::vec::Vec; +use mycore::cmp::Ord as _; + +fn issue_11524() -> Vec { + let mut vec = vec![1, 2, 3]; + + // We could lint and suggest `vec.sort_by_key(|a| a + 1);`, but we don't bother to -- see the + // comment in the lint at line 194 + vec.sort_by(|a, b| (a + 1).cmp(&(b + 1))); + vec +} + +fn issue_11524_2() -> Vec { + let mut vec = vec![1, 2, 3]; + + // Should not lint, as even `vec.sort_by_key(|b| core::cmp::Reverse(b + 1));` would not compile + vec.sort_by(|a, b| (b + 1).cmp(&(a + 1))); + vec +} From 459d77e863f0607f4f0fdc25be19db6f49fdc9c0 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Thu, 8 Jan 2026 22:23:16 +0200 Subject: [PATCH 0141/1843] Publish smol_str v0.3.5 --- src/tools/rust-analyzer/Cargo.lock | 2 +- src/tools/rust-analyzer/lib/smol_str/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock index 8188fbf96064..5bdde7c7c3e6 100644 --- a/src/tools/rust-analyzer/Cargo.lock +++ b/src/tools/rust-analyzer/Cargo.lock @@ -2629,7 +2629,7 @@ dependencies = [ [[package]] name = "smol_str" -version = "0.3.4" +version = "0.3.5" dependencies = [ "arbitrary", "borsh", diff --git a/src/tools/rust-analyzer/lib/smol_str/Cargo.toml b/src/tools/rust-analyzer/lib/smol_str/Cargo.toml index 118b25993ffe..4e7844b49e19 100644 --- a/src/tools/rust-analyzer/lib/smol_str/Cargo.toml +++ b/src/tools/rust-analyzer/lib/smol_str/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "smol_str" -version = "0.3.4" +version = "0.3.5" description = "small-string optimized string type with O(1) clone" license = "MIT OR Apache-2.0" repository = "https://github.com/rust-lang/rust-analyzer/tree/master/lib/smol_str" From d0d725133fb501ffb9f4572d2ea59454e68bb75a Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Thu, 8 Jan 2026 21:08:15 +0100 Subject: [PATCH 0142/1843] clean-up --- clippy_lints/src/floating_point_arithmetic.rs | 202 +++++++++--------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/clippy_lints/src/floating_point_arithmetic.rs b/clippy_lints/src/floating_point_arithmetic.rs index 5f022ba307ff..0a8f6c6ebf6b 100644 --- a/clippy_lints/src/floating_point_arithmetic.rs +++ b/clippy_lints/src/floating_point_arithmetic.rs @@ -1,6 +1,6 @@ use clippy_utils::consts::Constant::{F32, F64, Int}; use clippy_utils::consts::{ConstEvalCtxt, Constant}; -use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then}; use clippy_utils::res::{MaybeDef, MaybeTypeckRes}; use clippy_utils::{ eq_expr_value, get_parent_expr, has_ambiguous_literal_in_expr, higher, is_in_const_context, is_no_std_crate, @@ -128,16 +128,16 @@ fn get_specialized_log_method(cx: &LateContext<'_>, base: &Expr<'_>, ctxt: Synta // Adds type suffixes and parenthesis to method receivers if necessary fn prepare_receiver_sugg<'a>(cx: &LateContext<'_>, mut expr: &'a Expr<'a>) -> Sugg<'a> { - let mut suggestion = Sugg::hir(cx, expr, ".."); + let mut suggestion = Sugg::hir(cx, expr, "_"); - if let ExprKind::Unary(UnOp::Neg, inner_expr) = &expr.kind { + if let ExprKind::Unary(UnOp::Neg, inner_expr) = expr.kind { expr = inner_expr; } if let ty::Float(float_ty) = cx.typeck_results().expr_ty(expr).kind() // if the expression is a float literal and it is unsuffixed then // add a suffix so the suggestion is valid and unambiguous - && let ExprKind::Lit(lit) = &expr.kind + && let ExprKind::Lit(lit) = expr.kind && let ast::LitKind::Float(sym, ast::LitFloatType::Unsuffixed) = lit.node { let op = format!( @@ -166,7 +166,7 @@ fn check_log_base(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, ar expr.span, "logarithm for bases 2, 10 and e can be computed more accurately", "consider using", - format!("{}.{method}()", Sugg::hir(cx, receiver, "..").maybe_paren()), + format!("{}.{method}()", Sugg::hir(cx, receiver, "_").maybe_paren()), Applicability::MachineApplicable, ); } @@ -251,25 +251,25 @@ fn check_powf(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: // Check argument if let Some(value) = ConstEvalCtxt::new(cx).eval(&args[0]) { + let recv = Sugg::hir(cx, receiver, "_").maybe_paren(); let (lint, help, suggestion) = if F32(1.0 / 2.0) == value || F64(1.0 / 2.0) == value { ( SUBOPTIMAL_FLOPS, "square-root of a number can be computed more efficiently and accurately", - format!("{}.sqrt()", Sugg::hir(cx, receiver, "..").maybe_paren()), + format!("{recv}.sqrt()"), ) } else if F32(1.0 / 3.0) == value || F64(1.0 / 3.0) == value { ( IMPRECISE_FLOPS, "cube-root of a number can be computed more accurately", - format!("{}.cbrt()", Sugg::hir(cx, receiver, "..").maybe_paren()), + format!("{recv}.cbrt()"), ) } else if let Some(exponent) = get_integer_from_float_constant(&value) { ( SUBOPTIMAL_FLOPS, "exponentiation with integer powers can be computed more efficiently", format!( - "{}.powi({})", - Sugg::hir(cx, receiver, "..").maybe_paren(), + "{recv}.powi({})", numeric_literal::format(&exponent.to_string(), None, false) ), ) @@ -311,31 +311,36 @@ fn check_powi(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: rhs, ) = parent.kind { - let other_addend = if lhs.hir_id == expr.hir_id { rhs } else { lhs }; - - // Negate expr if original code has subtraction and expr is on the right side - let maybe_neg_sugg = |expr, hir_id| { - let sugg = Sugg::hir(cx, expr, ".."); - if matches!(op, BinOpKind::Sub) && hir_id == rhs.hir_id { - -sugg - } else { - sugg - } - }; - - span_lint_and_sugg( + span_lint_and_then( cx, SUBOPTIMAL_FLOPS, parent.span, "multiply and add expressions can be calculated more efficiently and accurately", - "consider using", - format!( - "{}.mul_add({}, {})", - Sugg::hir(cx, receiver, "..").maybe_paren(), - maybe_neg_sugg(receiver, expr.hir_id), - maybe_neg_sugg(other_addend, other_addend.hir_id), - ), - Applicability::MachineApplicable, + |diag| { + let other_addend = if lhs.hir_id == expr.hir_id { rhs } else { lhs }; + + // Negate expr if original code has subtraction and expr is on the right side + let maybe_neg_sugg = |expr, hir_id| { + let sugg = Sugg::hir(cx, expr, "_"); + if matches!(op, BinOpKind::Sub) && hir_id == rhs.hir_id { + -sugg + } else { + sugg + } + }; + + diag.span_suggestion( + parent.span, + "consider using", + format!( + "{}.mul_add({}, {})", + Sugg::hir(cx, receiver, "_").maybe_paren(), + maybe_neg_sugg(receiver, expr.hir_id), + maybe_neg_sugg(other_addend, other_addend.hir_id), + ), + Applicability::MachineApplicable, + ); + }, ); } } @@ -370,14 +375,14 @@ fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>) -> Option { { return Some(format!( "{}.hypot({})", - Sugg::hir(cx, lmul_lhs, "..").maybe_paren(), - Sugg::hir(cx, rmul_lhs, "..") + Sugg::hir(cx, lmul_lhs, "_").maybe_paren(), + Sugg::hir(cx, rmul_lhs, "_") )); } // check if expression of the form x.powi(2) + y.powi(2) - if let ExprKind::MethodCall(PathSegment { ident: lmethod, .. }, largs_0, [largs_1, ..], _) = &add_lhs.kind - && let ExprKind::MethodCall(PathSegment { ident: rmethod, .. }, rargs_0, [rargs_1, ..], _) = &add_rhs.kind + if let ExprKind::MethodCall(PathSegment { ident: lmethod, .. }, largs_0, [largs_1, ..], _) = add_lhs.kind + && let ExprKind::MethodCall(PathSegment { ident: rmethod, .. }, rargs_0, [rargs_1, ..], _) = add_rhs.kind && lmethod.name == sym::powi && rmethod.name == sym::powi && let ecx = ConstEvalCtxt::new(cx) @@ -388,8 +393,8 @@ fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>) -> Option { { return Some(format!( "{}.hypot({})", - Sugg::hir(cx, largs_0, "..").maybe_paren(), - Sugg::hir(cx, rargs_0, "..") + Sugg::hir(cx, largs_0, "_").maybe_paren(), + Sugg::hir(cx, rargs_0, "_") )); } } @@ -421,7 +426,7 @@ fn check_expm1(cx: &LateContext<'_>, expr: &Expr<'_>) { lhs, rhs, ) = expr.kind - && let ExprKind::MethodCall(path, self_arg, [], _) = &lhs.kind + && let ExprKind::MethodCall(path, self_arg, [], _) = lhs.kind && path.ident.name == sym::exp && cx.typeck_results().expr_ty(lhs).is_floating_point() && let Some(value) = ConstEvalCtxt::new(cx).eval(rhs) @@ -434,7 +439,7 @@ fn check_expm1(cx: &LateContext<'_>, expr: &Expr<'_>) { expr.span, "(e.pow(x) - 1) can be computed more accurately", "consider using", - format!("{}.exp_m1()", Sugg::hir(cx, self_arg, "..").maybe_paren()), + format!("{}.exp_m1()", Sugg::hir(cx, self_arg, "_").maybe_paren()), Applicability::MachineApplicable, ); } @@ -447,7 +452,7 @@ fn is_float_mul_expr<'a>(cx: &LateContext<'_>, expr: &'a Expr<'a>) -> Option<(&' }, lhs, rhs, - ) = &expr.kind + ) = expr.kind && cx.typeck_results().expr_ty(lhs).is_floating_point() && cx.typeck_results().expr_ty(rhs).is_floating_point() { @@ -476,18 +481,18 @@ fn check_mul_add(cx: &LateContext<'_>, expr: &Expr<'_>) { } let maybe_neg_sugg = |expr| { - let sugg = Sugg::hir(cx, expr, ".."); + let sugg = Sugg::hir(cx, expr, "_"); if let BinOpKind::Sub = op { -sugg } else { sugg } }; let (recv, arg1, arg2) = if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, lhs) && cx.typeck_results().expr_ty(rhs).is_floating_point() { - (inner_lhs, Sugg::hir(cx, inner_rhs, ".."), maybe_neg_sugg(rhs)) + (inner_lhs, Sugg::hir(cx, inner_rhs, "_"), maybe_neg_sugg(rhs)) } else if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, rhs) && cx.typeck_results().expr_ty(lhs).is_floating_point() { - (inner_lhs, maybe_neg_sugg(inner_rhs), Sugg::hir(cx, lhs, "..")) + (inner_lhs, maybe_neg_sugg(inner_rhs), Sugg::hir(cx, lhs, "_")) } else { return; }; @@ -558,12 +563,12 @@ fn is_zero(cx: &LateContext<'_>, expr: &Expr<'_>, ctxt: SyntaxContext) -> bool { /// If the two expressions are not negations of each other, then it /// returns None. fn are_negated<'a>(cx: &LateContext<'_>, expr1: &'a Expr<'a>, expr2: &'a Expr<'a>) -> Option<(bool, &'a Expr<'a>)> { - if let ExprKind::Unary(UnOp::Neg, expr1_negated) = &expr1.kind + if let ExprKind::Unary(UnOp::Neg, expr1_negated) = expr1.kind && eq_expr_value(cx, expr1_negated, expr2) { return Some((false, expr2)); } - if let ExprKind::Unary(UnOp::Neg, expr2_negated) = &expr2.kind + if let ExprKind::Unary(UnOp::Neg, expr2_negated) = expr2.kind && eq_expr_value(cx, expr1, expr2_negated) { return Some((true, expr1)); @@ -581,29 +586,20 @@ fn check_custom_abs(cx: &LateContext<'_>, expr: &Expr<'_>) { && let else_body_expr = peel_blocks(r#else) && let Some((if_expr_positive, body)) = are_negated(cx, if_body_expr, else_body_expr) { - let positive_abs_sugg = ( - "manual implementation of `abs` method", - format!("{}.abs()", Sugg::hir(cx, body, "..").maybe_paren()), - ); - let negative_abs_sugg = ( - "manual implementation of negation of `abs` method", - format!("-{}.abs()", Sugg::hir(cx, body, "..").maybe_paren()), - ); - let sugg = if is_testing_positive(cx, cond, body) { - if if_expr_positive { - positive_abs_sugg - } else { - negative_abs_sugg - } + let sugg_positive_abs = if is_testing_positive(cx, cond, body) { + if_expr_positive } else if is_testing_negative(cx, cond, body) { - if if_expr_positive { - negative_abs_sugg - } else { - positive_abs_sugg - } + !if_expr_positive } else { return; }; + let body = Sugg::hir(cx, body, "_").maybe_paren(); + let sugg = if sugg_positive_abs { + ("manual implementation of `abs` method", format!("{body}.abs()")) + } else { + #[rustfmt::skip] + ("manual implementation of negation of `abs` method", format!("-{body}.abs()")) + }; span_lint_and_sugg( cx, SUBOPTIMAL_FLOPS, @@ -637,10 +633,10 @@ fn check_log_division(cx: &LateContext<'_>, expr: &Expr<'_>) { }, lhs, rhs, - ) = &expr.kind + ) = expr.kind && are_same_base_logs(cx, lhs, rhs) - && let ExprKind::MethodCall(_, largs_self, ..) = &lhs.kind - && let ExprKind::MethodCall(_, rargs_self, ..) = &rhs.kind + && let ExprKind::MethodCall(_, largs_self, ..) = lhs.kind + && let ExprKind::MethodCall(_, rargs_self, ..) = rhs.kind { span_lint_and_sugg( cx, @@ -650,8 +646,8 @@ fn check_log_division(cx: &LateContext<'_>, expr: &Expr<'_>) { "consider using", format!( "{}.log({})", - Sugg::hir(cx, largs_self, "..").maybe_paren(), - Sugg::hir(cx, rargs_self, ".."), + Sugg::hir(cx, largs_self, "_").maybe_paren(), + Sugg::hir(cx, rargs_self, "_"), ), Applicability::MachineApplicable, ); @@ -665,14 +661,14 @@ fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { }, div_lhs, div_rhs, - ) = &expr.kind + ) = expr.kind && let ExprKind::Binary( Spanned { node: BinOpKind::Mul, .. }, mul_lhs, mul_rhs, - ) = &div_lhs.kind + ) = div_lhs.kind && let ecx = ConstEvalCtxt::new(cx) && let Some(rvalue) = ecx.eval(div_rhs) && let Some(lvalue) = ecx.eval(mul_rhs) @@ -681,48 +677,52 @@ fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { if (F32(f32_consts::PI) == rvalue || F64(f64_consts::PI) == rvalue) && (F32(180_f32) == lvalue || F64(180_f64) == lvalue) { - let mut proposal = format!("{}.to_degrees()", Sugg::hir(cx, mul_lhs, "..").maybe_paren()); - if let ExprKind::Lit(literal) = mul_lhs.kind - && let ast::LitKind::Float(ref value, float_type) = literal.node - && float_type == ast::LitFloatType::Unsuffixed - { - if value.as_str().ends_with('.') { - proposal = format!("{}0_f64.to_degrees()", Sugg::hir(cx, mul_lhs, "..")); - } else { - proposal = format!("{}_f64.to_degrees()", Sugg::hir(cx, mul_lhs, "..")); - } - } - span_lint_and_sugg( + span_lint_and_then( cx, SUBOPTIMAL_FLOPS, expr.span, "conversion to degrees can be done more accurately", - "consider using", - proposal, - Applicability::MachineApplicable, + |diag| { + let recv = Sugg::hir(cx, mul_lhs, "num"); + let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind + && let ast::LitKind::Float(ref value, float_type) = literal.node + && float_type == ast::LitFloatType::Unsuffixed + { + if value.as_str().ends_with('.') { + format!("{recv}0_f64.to_degrees()") + } else { + format!("{recv}_f64.to_degrees()") + } + } else { + format!("{}.to_degrees()", recv.maybe_paren()) + }; + diag.span_suggestion(expr.span, "consider using", proposal, Applicability::MachineApplicable); + }, ); } else if (F32(180_f32) == rvalue || F64(180_f64) == rvalue) && (F32(f32_consts::PI) == lvalue || F64(f64_consts::PI) == lvalue) { - let mut proposal = format!("{}.to_radians()", Sugg::hir(cx, mul_lhs, "..").maybe_paren()); - if let ExprKind::Lit(literal) = mul_lhs.kind - && let ast::LitKind::Float(ref value, float_type) = literal.node - && float_type == ast::LitFloatType::Unsuffixed - { - if value.as_str().ends_with('.') { - proposal = format!("{}0_f64.to_radians()", Sugg::hir(cx, mul_lhs, "..")); - } else { - proposal = format!("{}_f64.to_radians()", Sugg::hir(cx, mul_lhs, "..")); - } - } - span_lint_and_sugg( + span_lint_and_then( cx, SUBOPTIMAL_FLOPS, expr.span, "conversion to radians can be done more accurately", - "consider using", - proposal, - Applicability::MachineApplicable, + |diag| { + let recv = Sugg::hir(cx, mul_lhs, "num"); + let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind + && let ast::LitKind::Float(ref value, float_type) = literal.node + && float_type == ast::LitFloatType::Unsuffixed + { + if value.as_str().ends_with('.') { + format!("{recv}0_f64.to_radians()") + } else { + format!("{recv}_f64.to_radians()") + } + } else { + format!("{}.to_radians()", recv.maybe_paren()) + }; + diag.span_suggestion(expr.span, "consider using", proposal, Applicability::MachineApplicable); + }, ); } } @@ -735,7 +735,7 @@ impl<'tcx> LateLintPass<'tcx> for FloatingPointArithmetic { return; } - if let ExprKind::MethodCall(path, receiver, args, _) = &expr.kind { + if let ExprKind::MethodCall(path, receiver, args, _) = expr.kind { let recv_ty = cx.typeck_results().expr_ty(receiver); if recv_ty.is_floating_point() && !is_no_std_crate(cx) && cx.ty_based_def(expr).opt_parent(cx).is_impl(cx) { From eb101b1d2a2b7ee1407935ba8c5bee36ac9ee31d Mon Sep 17 00:00:00 2001 From: Paul Mabileau Date: Thu, 18 Dec 2025 12:55:04 +0100 Subject: [PATCH 0143/1843] Fix(lib/win/thread): Ensure `Sleep`'s usage passes over the requested duration under Win7 Fixes #149935. See the added comment for more details. Signed-off-by: Paul Mabileau --- library/std/src/sys/thread/windows.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/library/std/src/sys/thread/windows.rs b/library/std/src/sys/thread/windows.rs index 6a21b11e0312..ea18572489ee 100644 --- a/library/std/src/sys/thread/windows.rs +++ b/library/std/src/sys/thread/windows.rs @@ -8,7 +8,7 @@ use crate::sys::pal::time::WaitableTimer; use crate::sys::pal::{dur2timeout, to_u16s}; use crate::sys::{FromInner, c, stack_overflow}; use crate::thread::ThreadInit; -use crate::time::Duration; +use crate::time::{Duration, Instant}; use crate::{io, ptr}; pub const DEFAULT_MIN_STACK_SIZE: usize = 2 * 1024 * 1024; @@ -120,11 +120,28 @@ pub fn sleep(dur: Duration) { timer.set(dur)?; timer.wait() } + // Directly forward to `Sleep` for its zero duration behavior when indeed + // zero in order to skip the `Instant::now` calls, useless in this case. + if dur.is_zero() { + unsafe { c::Sleep(0) }; // Attempt to use high-precision sleep (Windows 10, version 1803+). - // On error fallback to the standard `Sleep` function. - // Also preserves the zero duration behavior of `Sleep`. - if dur.is_zero() || high_precision_sleep(dur).is_err() { - unsafe { c::Sleep(dur2timeout(dur)) } + // On error, fallback to the standard `Sleep` function. + } else if high_precision_sleep(dur).is_err() { + let start = Instant::now(); + unsafe { c::Sleep(dur2timeout(dur)) }; + + // See #149935: `Sleep` under Windows 7 and probably 8 as well seems a + // bit buggy for us as it can last less than the requested time while + // our API is meant to guarantee that. This is fixed by measuring the + // effective time difference and if needed, sleeping a bit more in + // order to ensure the duration is always exceeded. A fixed single + // millisecond works because `Sleep` operates based on a system-wide + // (until Windows 10 2004 that makes it process-local) interrupt timer + // that counts in "tick" units of ~15ms by default: a 1ms timeout + // therefore passes the next tick boundary. + if start.elapsed() < dur { + unsafe { c::Sleep(1) }; + } } } From cb9a079f608e6fed49a9303ff68ecca322e5b06b Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Thu, 8 Jan 2026 21:24:50 +0100 Subject: [PATCH 0144/1843] fix(float_point_arithmetic): respect reduced applicability --- clippy_lints/src/floating_point_arithmetic.rs | 142 ++++++++++-------- 1 file changed, 76 insertions(+), 66 deletions(-) diff --git a/clippy_lints/src/floating_point_arithmetic.rs b/clippy_lints/src/floating_point_arithmetic.rs index 0a8f6c6ebf6b..64a2af38a108 100644 --- a/clippy_lints/src/floating_point_arithmetic.rs +++ b/clippy_lints/src/floating_point_arithmetic.rs @@ -127,8 +127,8 @@ fn get_specialized_log_method(cx: &LateContext<'_>, base: &Expr<'_>, ctxt: Synta } // Adds type suffixes and parenthesis to method receivers if necessary -fn prepare_receiver_sugg<'a>(cx: &LateContext<'_>, mut expr: &'a Expr<'a>) -> Sugg<'a> { - let mut suggestion = Sugg::hir(cx, expr, "_"); +fn prepare_receiver_sugg<'a>(cx: &LateContext<'_>, mut expr: &'a Expr<'a>, app: &mut Applicability) -> Sugg<'a> { + let mut suggestion = Sugg::hir_with_applicability(cx, expr, "_", app); if let ExprKind::Unary(UnOp::Neg, inner_expr) = expr.kind { expr = inner_expr; @@ -160,14 +160,16 @@ fn prepare_receiver_sugg<'a>(cx: &LateContext<'_>, mut expr: &'a Expr<'a>) -> Su fn check_log_base(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { if let Some(method) = get_specialized_log_method(cx, &args[0], expr.span.ctxt()) { - span_lint_and_sugg( + span_lint_and_then( cx, SUBOPTIMAL_FLOPS, expr.span, "logarithm for bases 2, 10 and e can be computed more accurately", - "consider using", - format!("{}.{method}()", Sugg::hir(cx, receiver, "_").maybe_paren()), - Applicability::MachineApplicable, + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.{method}()"), app); + }, ); } } @@ -190,14 +192,16 @@ fn check_ln1p(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { _ => return, }; - span_lint_and_sugg( + span_lint_and_then( cx, IMPRECISE_FLOPS, expr.span, "ln(1 + x) can be computed more accurately", - "consider using", - format!("{}.ln_1p()", prepare_receiver_sugg(cx, recv)), - Applicability::MachineApplicable, + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = prepare_receiver_sugg(cx, recv, &mut app); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.ln_1p()"), app); + }, ); } } @@ -238,20 +242,23 @@ fn check_powf(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: None } { - span_lint_and_sugg( + span_lint_and_then( cx, SUBOPTIMAL_FLOPS, expr.span, "exponent for bases 2 and e can be computed more accurately", - "consider using", - format!("{}.{method}()", prepare_receiver_sugg(cx, &args[0])), - Applicability::MachineApplicable, + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = prepare_receiver_sugg(cx, &args[0], &mut app); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.{method}()"), app); + }, ); } // Check argument if let Some(value) = ConstEvalCtxt::new(cx).eval(&args[0]) { - let recv = Sugg::hir(cx, receiver, "_").maybe_paren(); + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(); let (lint, help, suggestion) = if F32(1.0 / 2.0) == value || F64(1.0 / 2.0) == value { ( SUBOPTIMAL_FLOPS, @@ -277,15 +284,7 @@ fn check_powf(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: return; }; - span_lint_and_sugg( - cx, - lint, - expr.span, - help, - "consider using", - suggestion, - Applicability::MachineApplicable, - ); + span_lint_and_sugg(cx, lint, expr.span, help, "consider using", suggestion, app); } } @@ -297,7 +296,8 @@ fn check_powi(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: if let Some(grandparent) = get_parent_expr(cx, parent) && let ExprKind::MethodCall(PathSegment { ident: method, .. }, receiver, ..) = grandparent.kind && method.name == sym::sqrt - && detect_hypot(cx, receiver).is_some() + // we don't care about the applicability as this is an early-return condition + && detect_hypot(cx, receiver, &mut Applicability::Unspecified).is_some() { return; } @@ -320,8 +320,8 @@ fn check_powi(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: let other_addend = if lhs.hir_id == expr.hir_id { rhs } else { lhs }; // Negate expr if original code has subtraction and expr is on the right side - let maybe_neg_sugg = |expr, hir_id| { - let sugg = Sugg::hir(cx, expr, "_"); + let maybe_neg_sugg = |expr, hir_id, app: &mut _| { + let sugg = Sugg::hir_with_applicability(cx, expr, "_", app); if matches!(op, BinOpKind::Sub) && hir_id == rhs.hir_id { -sugg } else { @@ -329,16 +329,17 @@ fn check_powi(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: } }; + let mut app = Applicability::MachineApplicable; diag.span_suggestion( parent.span, "consider using", format!( "{}.mul_add({}, {})", - Sugg::hir(cx, receiver, "_").maybe_paren(), - maybe_neg_sugg(receiver, expr.hir_id), - maybe_neg_sugg(other_addend, other_addend.hir_id), + Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(), + maybe_neg_sugg(receiver, expr.hir_id, &mut app), + maybe_neg_sugg(other_addend, other_addend.hir_id, &mut app), ), - Applicability::MachineApplicable, + app, ); }, ); @@ -346,7 +347,7 @@ fn check_powi(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: } } -fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>) -> Option { +fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>, app: &mut Applicability) -> Option { if let ExprKind::Binary( Spanned { node: BinOpKind::Add, .. @@ -375,8 +376,8 @@ fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>) -> Option { { return Some(format!( "{}.hypot({})", - Sugg::hir(cx, lmul_lhs, "_").maybe_paren(), - Sugg::hir(cx, rmul_lhs, "_") + Sugg::hir_with_applicability(cx, lmul_lhs, "_", app).maybe_paren(), + Sugg::hir_with_applicability(cx, rmul_lhs, "_", app) )); } @@ -393,8 +394,8 @@ fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>) -> Option { { return Some(format!( "{}.hypot({})", - Sugg::hir(cx, largs_0, "_").maybe_paren(), - Sugg::hir(cx, rargs_0, "_") + Sugg::hir_with_applicability(cx, largs_0, "_", app).maybe_paren(), + Sugg::hir_with_applicability(cx, rargs_0, "_", app) )); } } @@ -403,7 +404,8 @@ fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>) -> Option { } fn check_hypot(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { - if let Some(message) = detect_hypot(cx, receiver) { + let mut app = Applicability::MachineApplicable; + if let Some(message) = detect_hypot(cx, receiver, &mut app) { span_lint_and_sugg( cx, IMPRECISE_FLOPS, @@ -411,7 +413,7 @@ fn check_hypot(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { "hypotenuse can be computed more accurately", "consider using", message, - Applicability::MachineApplicable, + app, ); } } @@ -433,14 +435,16 @@ fn check_expm1(cx: &LateContext<'_>, expr: &Expr<'_>) { && (F32(1.0) == value || F64(1.0) == value) && cx.typeck_results().expr_ty(self_arg).is_floating_point() { - span_lint_and_sugg( + span_lint_and_then( cx, IMPRECISE_FLOPS, expr.span, "(e.pow(x) - 1) can be computed more accurately", - "consider using", - format!("{}.exp_m1()", Sugg::hir(cx, self_arg, "_").maybe_paren()), - Applicability::MachineApplicable, + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, self_arg, "_", &mut app).maybe_paren(); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.exp_m1()"), app); + }, ); } } @@ -475,24 +479,34 @@ fn check_mul_add(cx: &LateContext<'_>, expr: &Expr<'_>) { if let Some(parent) = get_parent_expr(cx, expr) && let ExprKind::MethodCall(PathSegment { ident: method, .. }, receiver, ..) = parent.kind && method.name == sym::sqrt - && detect_hypot(cx, receiver).is_some() + // we don't care about the applicability as this is an early-return condition + && detect_hypot(cx, receiver, &mut Applicability::Unspecified).is_some() { return; } - let maybe_neg_sugg = |expr| { - let sugg = Sugg::hir(cx, expr, "_"); + let maybe_neg_sugg = |expr, app: &mut _| { + let sugg = Sugg::hir_with_applicability(cx, expr, "_", app); if let BinOpKind::Sub = op { -sugg } else { sugg } }; + let mut app = Applicability::MachineApplicable; let (recv, arg1, arg2) = if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, lhs) && cx.typeck_results().expr_ty(rhs).is_floating_point() { - (inner_lhs, Sugg::hir(cx, inner_rhs, "_"), maybe_neg_sugg(rhs)) + ( + inner_lhs, + Sugg::hir_with_applicability(cx, inner_rhs, "_", &mut app), + maybe_neg_sugg(rhs, &mut app), + ) } else if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, rhs) && cx.typeck_results().expr_ty(lhs).is_floating_point() { - (inner_lhs, maybe_neg_sugg(inner_rhs), Sugg::hir(cx, lhs, "_")) + ( + inner_lhs, + maybe_neg_sugg(inner_rhs, &mut app), + Sugg::hir_with_applicability(cx, lhs, "_", &mut app), + ) } else { return; }; @@ -511,8 +525,8 @@ fn check_mul_add(cx: &LateContext<'_>, expr: &Expr<'_>) { expr.span, "multiply and add expressions can be calculated more efficiently and accurately", "consider using", - format!("{}.mul_add({arg1}, {arg2})", prepare_receiver_sugg(cx, recv)), - Applicability::MachineApplicable, + format!("{}.mul_add({arg1}, {arg2})", prepare_receiver_sugg(cx, recv, &mut app)), + app, ); } } @@ -593,22 +607,15 @@ fn check_custom_abs(cx: &LateContext<'_>, expr: &Expr<'_>) { } else { return; }; - let body = Sugg::hir(cx, body, "_").maybe_paren(); + let mut app = Applicability::MachineApplicable; + let body = Sugg::hir_with_applicability(cx, body, "_", &mut app).maybe_paren(); let sugg = if sugg_positive_abs { ("manual implementation of `abs` method", format!("{body}.abs()")) } else { #[rustfmt::skip] ("manual implementation of negation of `abs` method", format!("-{body}.abs()")) }; - span_lint_and_sugg( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - sugg.0, - "try", - sugg.1, - Applicability::MachineApplicable, - ); + span_lint_and_sugg(cx, SUBOPTIMAL_FLOPS, expr.span, sugg.0, "try", sugg.1, app); } } @@ -638,6 +645,7 @@ fn check_log_division(cx: &LateContext<'_>, expr: &Expr<'_>) { && let ExprKind::MethodCall(_, largs_self, ..) = lhs.kind && let ExprKind::MethodCall(_, rargs_self, ..) = rhs.kind { + let mut app = Applicability::MachineApplicable; span_lint_and_sugg( cx, SUBOPTIMAL_FLOPS, @@ -646,10 +654,10 @@ fn check_log_division(cx: &LateContext<'_>, expr: &Expr<'_>) { "consider using", format!( "{}.log({})", - Sugg::hir(cx, largs_self, "_").maybe_paren(), - Sugg::hir(cx, rargs_self, "_"), + Sugg::hir_with_applicability(cx, largs_self, "_", &mut app).maybe_paren(), + Sugg::hir_with_applicability(cx, rargs_self, "_", &mut app), ), - Applicability::MachineApplicable, + app, ); } } @@ -683,7 +691,8 @@ fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { expr.span, "conversion to degrees can be done more accurately", |diag| { - let recv = Sugg::hir(cx, mul_lhs, "num"); + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, mul_lhs, "num", &mut app); let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind && let ast::LitKind::Float(ref value, float_type) = literal.node && float_type == ast::LitFloatType::Unsuffixed @@ -696,7 +705,7 @@ fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { } else { format!("{}.to_degrees()", recv.maybe_paren()) }; - diag.span_suggestion(expr.span, "consider using", proposal, Applicability::MachineApplicable); + diag.span_suggestion(expr.span, "consider using", proposal, app); }, ); } else if (F32(180_f32) == rvalue || F64(180_f64) == rvalue) @@ -708,7 +717,8 @@ fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { expr.span, "conversion to radians can be done more accurately", |diag| { - let recv = Sugg::hir(cx, mul_lhs, "num"); + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, mul_lhs, "num", &mut app); let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind && let ast::LitKind::Float(ref value, float_type) = literal.node && float_type == ast::LitFloatType::Unsuffixed @@ -721,7 +731,7 @@ fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { } else { format!("{}.to_radians()", recv.maybe_paren()) }; - diag.span_suggestion(expr.span, "consider using", proposal, Applicability::MachineApplicable); + diag.span_suggestion(expr.span, "consider using", proposal, app); }, ); } From 7a377d66727e441c8472c18cb7aab3a597be1639 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Thu, 8 Jan 2026 22:03:26 +0100 Subject: [PATCH 0145/1843] move `floating_point_arithmetic.rs` to `floating_point_arithmetic/mod.rs` --- .../mod.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename clippy_lints/src/{floating_point_arithmetic.rs => floating_point_arithmetic/mod.rs} (100%) diff --git a/clippy_lints/src/floating_point_arithmetic.rs b/clippy_lints/src/floating_point_arithmetic/mod.rs similarity index 100% rename from clippy_lints/src/floating_point_arithmetic.rs rename to clippy_lints/src/floating_point_arithmetic/mod.rs From 44a41041fcb00f6fd1f76beeef53db936d39e31d Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Thu, 8 Jan 2026 22:46:58 +0100 Subject: [PATCH 0146/1843] extract each check into a separate module --- .../floating_point_arithmetic/custom_abs.rs | 100 +++ .../src/floating_point_arithmetic/expm1.rs | 42 ++ .../src/floating_point_arithmetic/hypot.rs | 82 +++ .../src/floating_point_arithmetic/lib.rs | 42 ++ .../src/floating_point_arithmetic/ln1p.rs | 41 ++ .../src/floating_point_arithmetic/log_base.rs | 44 ++ .../floating_point_arithmetic/log_division.rs | 52 ++ .../src/floating_point_arithmetic/mod.rs | 678 +----------------- .../src/floating_point_arithmetic/mul_add.rs | 94 +++ .../src/floating_point_arithmetic/powf.rs | 94 +++ .../src/floating_point_arithmetic/powi.rs | 70 ++ .../src/floating_point_arithmetic/radians.rs | 89 +++ 12 files changed, 774 insertions(+), 654 deletions(-) create mode 100644 clippy_lints/src/floating_point_arithmetic/custom_abs.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/expm1.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/hypot.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/lib.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/ln1p.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/log_base.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/log_division.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/mul_add.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/powf.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/powi.rs create mode 100644 clippy_lints/src/floating_point_arithmetic/radians.rs diff --git a/clippy_lints/src/floating_point_arithmetic/custom_abs.rs b/clippy_lints/src/floating_point_arithmetic/custom_abs.rs new file mode 100644 index 000000000000..d12a32e15881 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/custom_abs.rs @@ -0,0 +1,100 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::{F32, F64, Int}; +use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::sugg::Sugg; +use clippy_utils::{eq_expr_value, higher, peel_blocks}; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind, UnOp}; +use rustc_lint::LateContext; +use rustc_span::SyntaxContext; +use rustc_span::source_map::Spanned; + +use super::SUBOPTIMAL_FLOPS; + +/// Returns true iff expr is an expression which tests whether or not +/// test is positive or an expression which tests whether or not test +/// is nonnegative. +/// Used for check-custom-abs function below +fn is_testing_positive(cx: &LateContext<'_>, expr: &Expr<'_>, test: &Expr<'_>) -> bool { + if let ExprKind::Binary(Spanned { node: op, .. }, left, right) = expr.kind { + match op { + BinOpKind::Gt | BinOpKind::Ge => is_zero(cx, right, expr.span.ctxt()) && eq_expr_value(cx, left, test), + BinOpKind::Lt | BinOpKind::Le => is_zero(cx, left, expr.span.ctxt()) && eq_expr_value(cx, right, test), + _ => false, + } + } else { + false + } +} + +/// See [`is_testing_positive`] +fn is_testing_negative(cx: &LateContext<'_>, expr: &Expr<'_>, test: &Expr<'_>) -> bool { + if let ExprKind::Binary(Spanned { node: op, .. }, left, right) = expr.kind { + match op { + BinOpKind::Gt | BinOpKind::Ge => is_zero(cx, left, expr.span.ctxt()) && eq_expr_value(cx, right, test), + BinOpKind::Lt | BinOpKind::Le => is_zero(cx, right, expr.span.ctxt()) && eq_expr_value(cx, left, test), + _ => false, + } + } else { + false + } +} + +/// Returns true iff expr is some zero literal +fn is_zero(cx: &LateContext<'_>, expr: &Expr<'_>, ctxt: SyntaxContext) -> bool { + match ConstEvalCtxt::new(cx).eval_local(expr, ctxt) { + Some(Int(i)) => i == 0, + Some(F32(f)) => f == 0.0, + Some(F64(f)) => f == 0.0, + _ => false, + } +} + +/// If the two expressions are negations of each other, then it returns +/// a tuple, in which the first element is true iff expr1 is the +/// positive expressions, and the second element is the positive +/// one of the two expressions +/// If the two expressions are not negations of each other, then it +/// returns None. +fn are_negated<'a>(cx: &LateContext<'_>, expr1: &'a Expr<'a>, expr2: &'a Expr<'a>) -> Option<(bool, &'a Expr<'a>)> { + if let ExprKind::Unary(UnOp::Neg, expr1_negated) = expr1.kind + && eq_expr_value(cx, expr1_negated, expr2) + { + return Some((false, expr2)); + } + if let ExprKind::Unary(UnOp::Neg, expr2_negated) = expr2.kind + && eq_expr_value(cx, expr1, expr2_negated) + { + return Some((true, expr1)); + } + None +} + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>) { + if let Some(higher::If { + cond, + then, + r#else: Some(r#else), + }) = higher::If::hir(expr) + && let if_body_expr = peel_blocks(then) + && let else_body_expr = peel_blocks(r#else) + && let Some((if_expr_positive, body)) = are_negated(cx, if_body_expr, else_body_expr) + { + let sugg_positive_abs = if is_testing_positive(cx, cond, body) { + if_expr_positive + } else if is_testing_negative(cx, cond, body) { + !if_expr_positive + } else { + return; + }; + let mut app = Applicability::MachineApplicable; + let body = Sugg::hir_with_applicability(cx, body, "_", &mut app).maybe_paren(); + let sugg = if sugg_positive_abs { + ("manual implementation of `abs` method", format!("{body}.abs()")) + } else { + #[rustfmt::skip] + ("manual implementation of negation of `abs` method", format!("-{body}.abs()")) + }; + span_lint_and_sugg(cx, SUBOPTIMAL_FLOPS, expr.span, sugg.0, "try", sugg.1, app); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/expm1.rs b/clippy_lints/src/floating_point_arithmetic/expm1.rs new file mode 100644 index 000000000000..9a4c97569308 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/expm1.rs @@ -0,0 +1,42 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::{F32, F64}; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::sugg::Sugg; +use clippy_utils::sym; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; + +use super::IMPRECISE_FLOPS; + +// TODO: Lint expressions of the form `x.exp() - y` where y > 1 +// and suggest usage of `x.exp_m1() - (y - 1)` instead +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>) { + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Sub, .. + }, + lhs, + rhs, + ) = expr.kind + && let ExprKind::MethodCall(path, self_arg, [], _) = lhs.kind + && path.ident.name == sym::exp + && cx.typeck_results().expr_ty(lhs).is_floating_point() + && let Some(value) = ConstEvalCtxt::new(cx).eval(rhs) + && (F32(1.0) == value || F64(1.0) == value) + && cx.typeck_results().expr_ty(self_arg).is_floating_point() + { + span_lint_and_then( + cx, + IMPRECISE_FLOPS, + expr.span, + "(e.pow(x) - 1) can be computed more accurately", + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, self_arg, "_", &mut app).maybe_paren(); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.exp_m1()"), app); + }, + ); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/hypot.rs b/clippy_lints/src/floating_point_arithmetic/hypot.rs new file mode 100644 index 000000000000..49f8ba4bf825 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/hypot.rs @@ -0,0 +1,82 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::Int; +use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::sugg::Sugg; +use clippy_utils::{eq_expr_value, sym}; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind, PathSegment}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; + +use super::IMPRECISE_FLOPS; + +pub(super) fn detect(cx: &LateContext<'_>, receiver: &Expr<'_>, app: &mut Applicability) -> Option { + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Add, .. + }, + add_lhs, + add_rhs, + ) = receiver.kind + { + // check if expression of the form x * x + y * y + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Mul, .. + }, + lmul_lhs, + lmul_rhs, + ) = add_lhs.kind + && let ExprKind::Binary( + Spanned { + node: BinOpKind::Mul, .. + }, + rmul_lhs, + rmul_rhs, + ) = add_rhs.kind + && eq_expr_value(cx, lmul_lhs, lmul_rhs) + && eq_expr_value(cx, rmul_lhs, rmul_rhs) + { + return Some(format!( + "{}.hypot({})", + Sugg::hir_with_applicability(cx, lmul_lhs, "_", app).maybe_paren(), + Sugg::hir_with_applicability(cx, rmul_lhs, "_", app) + )); + } + + // check if expression of the form x.powi(2) + y.powi(2) + if let ExprKind::MethodCall(PathSegment { ident: lmethod, .. }, largs_0, [largs_1, ..], _) = add_lhs.kind + && let ExprKind::MethodCall(PathSegment { ident: rmethod, .. }, rargs_0, [rargs_1, ..], _) = add_rhs.kind + && lmethod.name == sym::powi + && rmethod.name == sym::powi + && let ecx = ConstEvalCtxt::new(cx) + && let Some(lvalue) = ecx.eval(largs_1) + && let Some(rvalue) = ecx.eval(rargs_1) + && Int(2) == lvalue + && Int(2) == rvalue + { + return Some(format!( + "{}.hypot({})", + Sugg::hir_with_applicability(cx, largs_0, "_", app).maybe_paren(), + Sugg::hir_with_applicability(cx, rargs_0, "_", app) + )); + } + } + + None +} + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { + let mut app = Applicability::MachineApplicable; + if let Some(message) = detect(cx, receiver, &mut app) { + span_lint_and_sugg( + cx, + IMPRECISE_FLOPS, + expr.span, + "hypotenuse can be computed more accurately", + "consider using", + message, + app, + ); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/lib.rs b/clippy_lints/src/floating_point_arithmetic/lib.rs new file mode 100644 index 000000000000..3fa041f97802 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/lib.rs @@ -0,0 +1,42 @@ +use clippy_utils::sugg::Sugg; +use rustc_ast::ast; +use rustc_errors::Applicability; +use rustc_hir::{Expr, ExprKind, UnOp}; +use rustc_lint::LateContext; +use rustc_middle::ty; + +// Adds type suffixes and parenthesis to method receivers if necessary +pub(super) fn prepare_receiver_sugg<'a>( + cx: &LateContext<'_>, + mut expr: &'a Expr<'a>, + app: &mut Applicability, +) -> Sugg<'a> { + let mut suggestion = Sugg::hir_with_applicability(cx, expr, "_", app); + + if let ExprKind::Unary(UnOp::Neg, inner_expr) = expr.kind { + expr = inner_expr; + } + + if let ty::Float(float_ty) = cx.typeck_results().expr_ty(expr).kind() + // if the expression is a float literal and it is unsuffixed then + // add a suffix so the suggestion is valid and unambiguous + && let ExprKind::Lit(lit) = expr.kind + && let ast::LitKind::Float(sym, ast::LitFloatType::Unsuffixed) = lit.node + { + let op = format!( + "{suggestion}{}{}", + // Check for float literals without numbers following the decimal + // separator such as `2.` and adds a trailing zero + if sym.as_str().ends_with('.') { "0" } else { "" }, + float_ty.name_str() + ) + .into(); + + suggestion = match suggestion { + Sugg::MaybeParen(_) | Sugg::UnOp(UnOp::Neg, _) => Sugg::MaybeParen(op), + _ => Sugg::NonParen(op), + }; + } + + suggestion.maybe_paren() +} diff --git a/clippy_lints/src/floating_point_arithmetic/ln1p.rs b/clippy_lints/src/floating_point_arithmetic/ln1p.rs new file mode 100644 index 000000000000..4c9aa96b5042 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/ln1p.rs @@ -0,0 +1,41 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::{F32, F64}; +use clippy_utils::diagnostics::span_lint_and_then; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; + +use super::IMPRECISE_FLOPS; + +// TODO: Lint expressions of the form `(x + y).ln()` where y > 1 and +// suggest usage of `(x + (y - 1)).ln_1p()` instead +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Add, .. + }, + lhs, + rhs, + ) = receiver.kind + { + let ecx = ConstEvalCtxt::new(cx); + let recv = match (ecx.eval(lhs), ecx.eval(rhs)) { + (Some(value), _) if F32(1.0) == value || F64(1.0) == value => rhs, + (_, Some(value)) if F32(1.0) == value || F64(1.0) == value => lhs, + _ => return, + }; + + span_lint_and_then( + cx, + IMPRECISE_FLOPS, + expr.span, + "ln(1 + x) can be computed more accurately", + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = super::lib::prepare_receiver_sugg(cx, recv, &mut app); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.ln_1p()"), app); + }, + ); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/log_base.rs b/clippy_lints/src/floating_point_arithmetic/log_base.rs new file mode 100644 index 000000000000..4ccc784655ed --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/log_base.rs @@ -0,0 +1,44 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::{F32, F64}; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::sugg::Sugg; +use rustc_errors::Applicability; +use rustc_hir::Expr; +use rustc_lint::LateContext; +use rustc_span::SyntaxContext; +use std::f32::consts as f32_consts; +use std::f64::consts as f64_consts; + +use super::SUBOPTIMAL_FLOPS; + +// Returns the specialized log method for a given base if base is constant +// and is one of 2, 10 and e +fn get_specialized_log_method(cx: &LateContext<'_>, base: &Expr<'_>, ctxt: SyntaxContext) -> Option<&'static str> { + if let Some(value) = ConstEvalCtxt::new(cx).eval_local(base, ctxt) { + if F32(2.0) == value || F64(2.0) == value { + return Some("log2"); + } else if F32(10.0) == value || F64(10.0) == value { + return Some("log10"); + } else if F32(f32_consts::E) == value || F64(f64_consts::E) == value { + return Some("ln"); + } + } + + None +} + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { + if let Some(method) = get_specialized_log_method(cx, &args[0], expr.span.ctxt()) { + span_lint_and_then( + cx, + SUBOPTIMAL_FLOPS, + expr.span, + "logarithm for bases 2, 10 and e can be computed more accurately", + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.{method}()"), app); + }, + ); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/log_division.rs b/clippy_lints/src/floating_point_arithmetic/log_division.rs new file mode 100644 index 000000000000..e3419ffad72a --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/log_division.rs @@ -0,0 +1,52 @@ +use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::sugg::Sugg; +use clippy_utils::{eq_expr_value, sym}; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind, PathSegment}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; + +use super::SUBOPTIMAL_FLOPS; + +fn are_same_base_logs(cx: &LateContext<'_>, expr_a: &Expr<'_>, expr_b: &Expr<'_>) -> bool { + if let ExprKind::MethodCall(PathSegment { ident: method_a, .. }, _, args_a, _) = expr_a.kind + && let ExprKind::MethodCall(PathSegment { ident: method_b, .. }, _, args_b, _) = expr_b.kind + { + return method_a.name == method_b.name + && args_a.len() == args_b.len() + && (matches!(method_a.name, sym::ln | sym::log2 | sym::log10) + || method_a.name == sym::log && args_a.len() == 1 && eq_expr_value(cx, &args_a[0], &args_b[0])); + } + + false +} + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>) { + // check if expression of the form x.logN() / y.logN() + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Div, .. + }, + lhs, + rhs, + ) = expr.kind + && are_same_base_logs(cx, lhs, rhs) + && let ExprKind::MethodCall(_, largs_self, ..) = lhs.kind + && let ExprKind::MethodCall(_, rargs_self, ..) = rhs.kind + { + let mut app = Applicability::MachineApplicable; + span_lint_and_sugg( + cx, + SUBOPTIMAL_FLOPS, + expr.span, + "log base can be expressed more clearly", + "consider using", + format!( + "{}.log({})", + Sugg::hir_with_applicability(cx, largs_self, "_", &mut app).maybe_paren(), + Sugg::hir_with_applicability(cx, rargs_self, "_", &mut app), + ), + app, + ); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/mod.rs b/clippy_lints/src/floating_point_arithmetic/mod.rs index 64a2af38a108..edc638c96bbf 100644 --- a/clippy_lints/src/floating_point_arithmetic/mod.rs +++ b/clippy_lints/src/floating_point_arithmetic/mod.rs @@ -1,22 +1,20 @@ -use clippy_utils::consts::Constant::{F32, F64, Int}; -use clippy_utils::consts::{ConstEvalCtxt, Constant}; -use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then}; use clippy_utils::res::{MaybeDef, MaybeTypeckRes}; -use clippy_utils::{ - eq_expr_value, get_parent_expr, has_ambiguous_literal_in_expr, higher, is_in_const_context, is_no_std_crate, - numeric_literal, peel_blocks, sugg, sym, -}; -use rustc_ast::ast; -use rustc_errors::Applicability; -use rustc_hir::{BinOpKind, Expr, ExprKind, PathSegment, UnOp}; +use clippy_utils::{is_in_const_context, is_no_std_crate, sym}; +use rustc_hir::{Expr, ExprKind}; use rustc_lint::{LateContext, LateLintPass}; -use rustc_middle::ty; use rustc_session::declare_lint_pass; -use rustc_span::SyntaxContext; -use rustc_span::source_map::Spanned; -use std::f32::consts as f32_consts; -use std::f64::consts as f64_consts; -use sugg::Sugg; + +mod custom_abs; +mod expm1; +mod hypot; +mod lib; +mod ln1p; +mod log_base; +mod log_division; +mod mul_add; +mod powf; +mod powi; +mod radians; declare_clippy_lint! { /// ### What it does @@ -110,634 +108,6 @@ declare_lint_pass!(FloatingPointArithmetic => [ SUBOPTIMAL_FLOPS ]); -// Returns the specialized log method for a given base if base is constant -// and is one of 2, 10 and e -fn get_specialized_log_method(cx: &LateContext<'_>, base: &Expr<'_>, ctxt: SyntaxContext) -> Option<&'static str> { - if let Some(value) = ConstEvalCtxt::new(cx).eval_local(base, ctxt) { - if F32(2.0) == value || F64(2.0) == value { - return Some("log2"); - } else if F32(10.0) == value || F64(10.0) == value { - return Some("log10"); - } else if F32(f32_consts::E) == value || F64(f64_consts::E) == value { - return Some("ln"); - } - } - - None -} - -// Adds type suffixes and parenthesis to method receivers if necessary -fn prepare_receiver_sugg<'a>(cx: &LateContext<'_>, mut expr: &'a Expr<'a>, app: &mut Applicability) -> Sugg<'a> { - let mut suggestion = Sugg::hir_with_applicability(cx, expr, "_", app); - - if let ExprKind::Unary(UnOp::Neg, inner_expr) = expr.kind { - expr = inner_expr; - } - - if let ty::Float(float_ty) = cx.typeck_results().expr_ty(expr).kind() - // if the expression is a float literal and it is unsuffixed then - // add a suffix so the suggestion is valid and unambiguous - && let ExprKind::Lit(lit) = expr.kind - && let ast::LitKind::Float(sym, ast::LitFloatType::Unsuffixed) = lit.node - { - let op = format!( - "{suggestion}{}{}", - // Check for float literals without numbers following the decimal - // separator such as `2.` and adds a trailing zero - if sym.as_str().ends_with('.') { "0" } else { "" }, - float_ty.name_str() - ) - .into(); - - suggestion = match suggestion { - Sugg::MaybeParen(_) | Sugg::UnOp(UnOp::Neg, _) => Sugg::MaybeParen(op), - _ => Sugg::NonParen(op), - }; - } - - suggestion.maybe_paren() -} - -fn check_log_base(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { - if let Some(method) = get_specialized_log_method(cx, &args[0], expr.span.ctxt()) { - span_lint_and_then( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - "logarithm for bases 2, 10 and e can be computed more accurately", - |diag| { - let mut app = Applicability::MachineApplicable; - let recv = Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(); - diag.span_suggestion(expr.span, "consider using", format!("{recv}.{method}()"), app); - }, - ); - } -} - -// TODO: Lint expressions of the form `(x + y).ln()` where y > 1 and -// suggest usage of `(x + (y - 1)).ln_1p()` instead -fn check_ln1p(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Add, .. - }, - lhs, - rhs, - ) = receiver.kind - { - let ecx = ConstEvalCtxt::new(cx); - let recv = match (ecx.eval(lhs), ecx.eval(rhs)) { - (Some(value), _) if F32(1.0) == value || F64(1.0) == value => rhs, - (_, Some(value)) if F32(1.0) == value || F64(1.0) == value => lhs, - _ => return, - }; - - span_lint_and_then( - cx, - IMPRECISE_FLOPS, - expr.span, - "ln(1 + x) can be computed more accurately", - |diag| { - let mut app = Applicability::MachineApplicable; - let recv = prepare_receiver_sugg(cx, recv, &mut app); - diag.span_suggestion(expr.span, "consider using", format!("{recv}.ln_1p()"), app); - }, - ); - } -} - -// Returns an integer if the float constant is a whole number and it can be -// converted to an integer without loss of precision. For now we only check -// ranges [-16777215, 16777216) for type f32 as whole number floats outside -// this range are lossy and ambiguous. -#[expect(clippy::cast_possible_truncation)] -fn get_integer_from_float_constant(value: &Constant) -> Option { - match value { - F32(num) if num.fract() == 0.0 => { - if (-16_777_215.0..16_777_216.0).contains(num) { - Some(num.round() as i32) - } else { - None - } - }, - F64(num) if num.fract() == 0.0 => { - if (-2_147_483_648.0..2_147_483_648.0).contains(num) { - Some(num.round() as i32) - } else { - None - } - }, - _ => None, - } -} - -fn check_powf(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { - // Check receiver - if let Some(value) = ConstEvalCtxt::new(cx).eval(receiver) - && let Some(method) = if F32(f32_consts::E) == value || F64(f64_consts::E) == value { - Some("exp") - } else if F32(2.0) == value || F64(2.0) == value { - Some("exp2") - } else { - None - } - { - span_lint_and_then( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - "exponent for bases 2 and e can be computed more accurately", - |diag| { - let mut app = Applicability::MachineApplicable; - let recv = prepare_receiver_sugg(cx, &args[0], &mut app); - diag.span_suggestion(expr.span, "consider using", format!("{recv}.{method}()"), app); - }, - ); - } - - // Check argument - if let Some(value) = ConstEvalCtxt::new(cx).eval(&args[0]) { - let mut app = Applicability::MachineApplicable; - let recv = Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(); - let (lint, help, suggestion) = if F32(1.0 / 2.0) == value || F64(1.0 / 2.0) == value { - ( - SUBOPTIMAL_FLOPS, - "square-root of a number can be computed more efficiently and accurately", - format!("{recv}.sqrt()"), - ) - } else if F32(1.0 / 3.0) == value || F64(1.0 / 3.0) == value { - ( - IMPRECISE_FLOPS, - "cube-root of a number can be computed more accurately", - format!("{recv}.cbrt()"), - ) - } else if let Some(exponent) = get_integer_from_float_constant(&value) { - ( - SUBOPTIMAL_FLOPS, - "exponentiation with integer powers can be computed more efficiently", - format!( - "{recv}.powi({})", - numeric_literal::format(&exponent.to_string(), None, false) - ), - ) - } else { - return; - }; - - span_lint_and_sugg(cx, lint, expr.span, help, "consider using", suggestion, app); - } -} - -fn check_powi(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { - if let Some(value) = ConstEvalCtxt::new(cx).eval(&args[0]) - && value == Int(2) - && let Some(parent) = get_parent_expr(cx, expr) - { - if let Some(grandparent) = get_parent_expr(cx, parent) - && let ExprKind::MethodCall(PathSegment { ident: method, .. }, receiver, ..) = grandparent.kind - && method.name == sym::sqrt - // we don't care about the applicability as this is an early-return condition - && detect_hypot(cx, receiver, &mut Applicability::Unspecified).is_some() - { - return; - } - - if let ExprKind::Binary( - Spanned { - node: op @ (BinOpKind::Add | BinOpKind::Sub), - .. - }, - lhs, - rhs, - ) = parent.kind - { - span_lint_and_then( - cx, - SUBOPTIMAL_FLOPS, - parent.span, - "multiply and add expressions can be calculated more efficiently and accurately", - |diag| { - let other_addend = if lhs.hir_id == expr.hir_id { rhs } else { lhs }; - - // Negate expr if original code has subtraction and expr is on the right side - let maybe_neg_sugg = |expr, hir_id, app: &mut _| { - let sugg = Sugg::hir_with_applicability(cx, expr, "_", app); - if matches!(op, BinOpKind::Sub) && hir_id == rhs.hir_id { - -sugg - } else { - sugg - } - }; - - let mut app = Applicability::MachineApplicable; - diag.span_suggestion( - parent.span, - "consider using", - format!( - "{}.mul_add({}, {})", - Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(), - maybe_neg_sugg(receiver, expr.hir_id, &mut app), - maybe_neg_sugg(other_addend, other_addend.hir_id, &mut app), - ), - app, - ); - }, - ); - } - } -} - -fn detect_hypot(cx: &LateContext<'_>, receiver: &Expr<'_>, app: &mut Applicability) -> Option { - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Add, .. - }, - add_lhs, - add_rhs, - ) = receiver.kind - { - // check if expression of the form x * x + y * y - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Mul, .. - }, - lmul_lhs, - lmul_rhs, - ) = add_lhs.kind - && let ExprKind::Binary( - Spanned { - node: BinOpKind::Mul, .. - }, - rmul_lhs, - rmul_rhs, - ) = add_rhs.kind - && eq_expr_value(cx, lmul_lhs, lmul_rhs) - && eq_expr_value(cx, rmul_lhs, rmul_rhs) - { - return Some(format!( - "{}.hypot({})", - Sugg::hir_with_applicability(cx, lmul_lhs, "_", app).maybe_paren(), - Sugg::hir_with_applicability(cx, rmul_lhs, "_", app) - )); - } - - // check if expression of the form x.powi(2) + y.powi(2) - if let ExprKind::MethodCall(PathSegment { ident: lmethod, .. }, largs_0, [largs_1, ..], _) = add_lhs.kind - && let ExprKind::MethodCall(PathSegment { ident: rmethod, .. }, rargs_0, [rargs_1, ..], _) = add_rhs.kind - && lmethod.name == sym::powi - && rmethod.name == sym::powi - && let ecx = ConstEvalCtxt::new(cx) - && let Some(lvalue) = ecx.eval(largs_1) - && let Some(rvalue) = ecx.eval(rargs_1) - && Int(2) == lvalue - && Int(2) == rvalue - { - return Some(format!( - "{}.hypot({})", - Sugg::hir_with_applicability(cx, largs_0, "_", app).maybe_paren(), - Sugg::hir_with_applicability(cx, rargs_0, "_", app) - )); - } - } - - None -} - -fn check_hypot(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>) { - let mut app = Applicability::MachineApplicable; - if let Some(message) = detect_hypot(cx, receiver, &mut app) { - span_lint_and_sugg( - cx, - IMPRECISE_FLOPS, - expr.span, - "hypotenuse can be computed more accurately", - "consider using", - message, - app, - ); - } -} - -// TODO: Lint expressions of the form `x.exp() - y` where y > 1 -// and suggest usage of `x.exp_m1() - (y - 1)` instead -fn check_expm1(cx: &LateContext<'_>, expr: &Expr<'_>) { - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Sub, .. - }, - lhs, - rhs, - ) = expr.kind - && let ExprKind::MethodCall(path, self_arg, [], _) = lhs.kind - && path.ident.name == sym::exp - && cx.typeck_results().expr_ty(lhs).is_floating_point() - && let Some(value) = ConstEvalCtxt::new(cx).eval(rhs) - && (F32(1.0) == value || F64(1.0) == value) - && cx.typeck_results().expr_ty(self_arg).is_floating_point() - { - span_lint_and_then( - cx, - IMPRECISE_FLOPS, - expr.span, - "(e.pow(x) - 1) can be computed more accurately", - |diag| { - let mut app = Applicability::MachineApplicable; - let recv = Sugg::hir_with_applicability(cx, self_arg, "_", &mut app).maybe_paren(); - diag.span_suggestion(expr.span, "consider using", format!("{recv}.exp_m1()"), app); - }, - ); - } -} - -fn is_float_mul_expr<'a>(cx: &LateContext<'_>, expr: &'a Expr<'a>) -> Option<(&'a Expr<'a>, &'a Expr<'a>)> { - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Mul, .. - }, - lhs, - rhs, - ) = expr.kind - && cx.typeck_results().expr_ty(lhs).is_floating_point() - && cx.typeck_results().expr_ty(rhs).is_floating_point() - { - return Some((lhs, rhs)); - } - - None -} - -fn check_mul_add(cx: &LateContext<'_>, expr: &Expr<'_>) { - if let ExprKind::Binary( - Spanned { - node: op @ (BinOpKind::Add | BinOpKind::Sub), - .. - }, - lhs, - rhs, - ) = &expr.kind - { - if let Some(parent) = get_parent_expr(cx, expr) - && let ExprKind::MethodCall(PathSegment { ident: method, .. }, receiver, ..) = parent.kind - && method.name == sym::sqrt - // we don't care about the applicability as this is an early-return condition - && detect_hypot(cx, receiver, &mut Applicability::Unspecified).is_some() - { - return; - } - - let maybe_neg_sugg = |expr, app: &mut _| { - let sugg = Sugg::hir_with_applicability(cx, expr, "_", app); - if let BinOpKind::Sub = op { -sugg } else { sugg } - }; - - let mut app = Applicability::MachineApplicable; - let (recv, arg1, arg2) = if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, lhs) - && cx.typeck_results().expr_ty(rhs).is_floating_point() - { - ( - inner_lhs, - Sugg::hir_with_applicability(cx, inner_rhs, "_", &mut app), - maybe_neg_sugg(rhs, &mut app), - ) - } else if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, rhs) - && cx.typeck_results().expr_ty(lhs).is_floating_point() - { - ( - inner_lhs, - maybe_neg_sugg(inner_rhs, &mut app), - Sugg::hir_with_applicability(cx, lhs, "_", &mut app), - ) - } else { - return; - }; - - // Check if any variable in the expression has an ambiguous type (could be f32 or f64) - // see: https://github.com/rust-lang/rust-clippy/issues/14897 - if (matches!(recv.kind, ExprKind::Path(_)) || matches!(recv.kind, ExprKind::Call(_, _))) - && has_ambiguous_literal_in_expr(cx, recv) - { - return; - } - - span_lint_and_sugg( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - "multiply and add expressions can be calculated more efficiently and accurately", - "consider using", - format!("{}.mul_add({arg1}, {arg2})", prepare_receiver_sugg(cx, recv, &mut app)), - app, - ); - } -} - -/// Returns true iff expr is an expression which tests whether or not -/// test is positive or an expression which tests whether or not test -/// is nonnegative. -/// Used for check-custom-abs function below -fn is_testing_positive(cx: &LateContext<'_>, expr: &Expr<'_>, test: &Expr<'_>) -> bool { - if let ExprKind::Binary(Spanned { node: op, .. }, left, right) = expr.kind { - match op { - BinOpKind::Gt | BinOpKind::Ge => is_zero(cx, right, expr.span.ctxt()) && eq_expr_value(cx, left, test), - BinOpKind::Lt | BinOpKind::Le => is_zero(cx, left, expr.span.ctxt()) && eq_expr_value(cx, right, test), - _ => false, - } - } else { - false - } -} - -/// See [`is_testing_positive`] -fn is_testing_negative(cx: &LateContext<'_>, expr: &Expr<'_>, test: &Expr<'_>) -> bool { - if let ExprKind::Binary(Spanned { node: op, .. }, left, right) = expr.kind { - match op { - BinOpKind::Gt | BinOpKind::Ge => is_zero(cx, left, expr.span.ctxt()) && eq_expr_value(cx, right, test), - BinOpKind::Lt | BinOpKind::Le => is_zero(cx, right, expr.span.ctxt()) && eq_expr_value(cx, left, test), - _ => false, - } - } else { - false - } -} - -/// Returns true iff expr is some zero literal -fn is_zero(cx: &LateContext<'_>, expr: &Expr<'_>, ctxt: SyntaxContext) -> bool { - match ConstEvalCtxt::new(cx).eval_local(expr, ctxt) { - Some(Int(i)) => i == 0, - Some(F32(f)) => f == 0.0, - Some(F64(f)) => f == 0.0, - _ => false, - } -} - -/// If the two expressions are negations of each other, then it returns -/// a tuple, in which the first element is true iff expr1 is the -/// positive expressions, and the second element is the positive -/// one of the two expressions -/// If the two expressions are not negations of each other, then it -/// returns None. -fn are_negated<'a>(cx: &LateContext<'_>, expr1: &'a Expr<'a>, expr2: &'a Expr<'a>) -> Option<(bool, &'a Expr<'a>)> { - if let ExprKind::Unary(UnOp::Neg, expr1_negated) = expr1.kind - && eq_expr_value(cx, expr1_negated, expr2) - { - return Some((false, expr2)); - } - if let ExprKind::Unary(UnOp::Neg, expr2_negated) = expr2.kind - && eq_expr_value(cx, expr1, expr2_negated) - { - return Some((true, expr1)); - } - None -} - -fn check_custom_abs(cx: &LateContext<'_>, expr: &Expr<'_>) { - if let Some(higher::If { - cond, - then, - r#else: Some(r#else), - }) = higher::If::hir(expr) - && let if_body_expr = peel_blocks(then) - && let else_body_expr = peel_blocks(r#else) - && let Some((if_expr_positive, body)) = are_negated(cx, if_body_expr, else_body_expr) - { - let sugg_positive_abs = if is_testing_positive(cx, cond, body) { - if_expr_positive - } else if is_testing_negative(cx, cond, body) { - !if_expr_positive - } else { - return; - }; - let mut app = Applicability::MachineApplicable; - let body = Sugg::hir_with_applicability(cx, body, "_", &mut app).maybe_paren(); - let sugg = if sugg_positive_abs { - ("manual implementation of `abs` method", format!("{body}.abs()")) - } else { - #[rustfmt::skip] - ("manual implementation of negation of `abs` method", format!("-{body}.abs()")) - }; - span_lint_and_sugg(cx, SUBOPTIMAL_FLOPS, expr.span, sugg.0, "try", sugg.1, app); - } -} - -fn are_same_base_logs(cx: &LateContext<'_>, expr_a: &Expr<'_>, expr_b: &Expr<'_>) -> bool { - if let ExprKind::MethodCall(PathSegment { ident: method_a, .. }, _, args_a, _) = expr_a.kind - && let ExprKind::MethodCall(PathSegment { ident: method_b, .. }, _, args_b, _) = expr_b.kind - { - return method_a.name == method_b.name - && args_a.len() == args_b.len() - && (matches!(method_a.name, sym::ln | sym::log2 | sym::log10) - || method_a.name == sym::log && args_a.len() == 1 && eq_expr_value(cx, &args_a[0], &args_b[0])); - } - - false -} - -fn check_log_division(cx: &LateContext<'_>, expr: &Expr<'_>) { - // check if expression of the form x.logN() / y.logN() - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Div, .. - }, - lhs, - rhs, - ) = expr.kind - && are_same_base_logs(cx, lhs, rhs) - && let ExprKind::MethodCall(_, largs_self, ..) = lhs.kind - && let ExprKind::MethodCall(_, rargs_self, ..) = rhs.kind - { - let mut app = Applicability::MachineApplicable; - span_lint_and_sugg( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - "log base can be expressed more clearly", - "consider using", - format!( - "{}.log({})", - Sugg::hir_with_applicability(cx, largs_self, "_", &mut app).maybe_paren(), - Sugg::hir_with_applicability(cx, rargs_self, "_", &mut app), - ), - app, - ); - } -} - -fn check_radians(cx: &LateContext<'_>, expr: &Expr<'_>) { - if let ExprKind::Binary( - Spanned { - node: BinOpKind::Div, .. - }, - div_lhs, - div_rhs, - ) = expr.kind - && let ExprKind::Binary( - Spanned { - node: BinOpKind::Mul, .. - }, - mul_lhs, - mul_rhs, - ) = div_lhs.kind - && let ecx = ConstEvalCtxt::new(cx) - && let Some(rvalue) = ecx.eval(div_rhs) - && let Some(lvalue) = ecx.eval(mul_rhs) - { - // TODO: also check for constant values near PI/180 or 180/PI - if (F32(f32_consts::PI) == rvalue || F64(f64_consts::PI) == rvalue) - && (F32(180_f32) == lvalue || F64(180_f64) == lvalue) - { - span_lint_and_then( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - "conversion to degrees can be done more accurately", - |diag| { - let mut app = Applicability::MachineApplicable; - let recv = Sugg::hir_with_applicability(cx, mul_lhs, "num", &mut app); - let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind - && let ast::LitKind::Float(ref value, float_type) = literal.node - && float_type == ast::LitFloatType::Unsuffixed - { - if value.as_str().ends_with('.') { - format!("{recv}0_f64.to_degrees()") - } else { - format!("{recv}_f64.to_degrees()") - } - } else { - format!("{}.to_degrees()", recv.maybe_paren()) - }; - diag.span_suggestion(expr.span, "consider using", proposal, app); - }, - ); - } else if (F32(180_f32) == rvalue || F64(180_f64) == rvalue) - && (F32(f32_consts::PI) == lvalue || F64(f64_consts::PI) == lvalue) - { - span_lint_and_then( - cx, - SUBOPTIMAL_FLOPS, - expr.span, - "conversion to radians can be done more accurately", - |diag| { - let mut app = Applicability::MachineApplicable; - let recv = Sugg::hir_with_applicability(cx, mul_lhs, "num", &mut app); - let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind - && let ast::LitKind::Float(ref value, float_type) = literal.node - && float_type == ast::LitFloatType::Unsuffixed - { - if value.as_str().ends_with('.') { - format!("{recv}0_f64.to_radians()") - } else { - format!("{recv}_f64.to_radians()") - } - } else { - format!("{}.to_radians()", recv.maybe_paren()) - }; - diag.span_suggestion(expr.span, "consider using", proposal, app); - }, - ); - } - } -} - impl<'tcx> LateLintPass<'tcx> for FloatingPointArithmetic { fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) { // All of these operations are currently not const and are in std. @@ -750,22 +120,22 @@ impl<'tcx> LateLintPass<'tcx> for FloatingPointArithmetic { if recv_ty.is_floating_point() && !is_no_std_crate(cx) && cx.ty_based_def(expr).opt_parent(cx).is_impl(cx) { match path.ident.name { - sym::ln => check_ln1p(cx, expr, receiver), - sym::log => check_log_base(cx, expr, receiver, args), - sym::powf => check_powf(cx, expr, receiver, args), - sym::powi => check_powi(cx, expr, receiver, args), - sym::sqrt => check_hypot(cx, expr, receiver), + sym::ln => ln1p::check(cx, expr, receiver), + sym::log => log_base::check(cx, expr, receiver, args), + sym::powf => powf::check(cx, expr, receiver, args), + sym::powi => powi::check(cx, expr, receiver, args), + sym::sqrt => hypot::check(cx, expr, receiver), _ => {}, } } } else { if !is_no_std_crate(cx) { - check_expm1(cx, expr); - check_mul_add(cx, expr); - check_custom_abs(cx, expr); - check_log_division(cx, expr); + expm1::check(cx, expr); + mul_add::check(cx, expr); + custom_abs::check(cx, expr); + log_division::check(cx, expr); } - check_radians(cx, expr); + radians::check(cx, expr); } } } diff --git a/clippy_lints/src/floating_point_arithmetic/mul_add.rs b/clippy_lints/src/floating_point_arithmetic/mul_add.rs new file mode 100644 index 000000000000..03a9d3b05f88 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/mul_add.rs @@ -0,0 +1,94 @@ +use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::sugg::Sugg; +use clippy_utils::{get_parent_expr, has_ambiguous_literal_in_expr, sym}; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind, PathSegment}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; + +use super::SUBOPTIMAL_FLOPS; + +fn is_float_mul_expr<'a>(cx: &LateContext<'_>, expr: &'a Expr<'a>) -> Option<(&'a Expr<'a>, &'a Expr<'a>)> { + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Mul, .. + }, + lhs, + rhs, + ) = expr.kind + && cx.typeck_results().expr_ty(lhs).is_floating_point() + && cx.typeck_results().expr_ty(rhs).is_floating_point() + { + return Some((lhs, rhs)); + } + + None +} + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>) { + if let ExprKind::Binary( + Spanned { + node: op @ (BinOpKind::Add | BinOpKind::Sub), + .. + }, + lhs, + rhs, + ) = &expr.kind + { + if let Some(parent) = get_parent_expr(cx, expr) + && let ExprKind::MethodCall(PathSegment { ident: method, .. }, receiver, ..) = parent.kind + && method.name == sym::sqrt + // we don't care about the applicability as this is an early-return condition + && super::hypot::detect(cx, receiver, &mut Applicability::Unspecified).is_some() + { + return; + } + + let maybe_neg_sugg = |expr, app: &mut _| { + let sugg = Sugg::hir_with_applicability(cx, expr, "_", app); + if let BinOpKind::Sub = op { -sugg } else { sugg } + }; + + let mut app = Applicability::MachineApplicable; + let (recv, arg1, arg2) = if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, lhs) + && cx.typeck_results().expr_ty(rhs).is_floating_point() + { + ( + inner_lhs, + Sugg::hir_with_applicability(cx, inner_rhs, "_", &mut app), + maybe_neg_sugg(rhs, &mut app), + ) + } else if let Some((inner_lhs, inner_rhs)) = is_float_mul_expr(cx, rhs) + && cx.typeck_results().expr_ty(lhs).is_floating_point() + { + ( + inner_lhs, + maybe_neg_sugg(inner_rhs, &mut app), + Sugg::hir_with_applicability(cx, lhs, "_", &mut app), + ) + } else { + return; + }; + + // Check if any variable in the expression has an ambiguous type (could be f32 or f64) + // see: https://github.com/rust-lang/rust-clippy/issues/14897 + if (matches!(recv.kind, ExprKind::Path(_)) || matches!(recv.kind, ExprKind::Call(_, _))) + && has_ambiguous_literal_in_expr(cx, recv) + { + return; + } + + span_lint_and_sugg( + cx, + SUBOPTIMAL_FLOPS, + expr.span, + "multiply and add expressions can be calculated more efficiently and accurately", + "consider using", + format!( + "{}.mul_add({arg1}, {arg2})", + super::lib::prepare_receiver_sugg(cx, recv, &mut app) + ), + app, + ); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/powf.rs b/clippy_lints/src/floating_point_arithmetic/powf.rs new file mode 100644 index 000000000000..8e4a7388e784 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/powf.rs @@ -0,0 +1,94 @@ +use clippy_utils::consts::Constant::{F32, F64}; +use clippy_utils::consts::{ConstEvalCtxt, Constant}; +use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then}; +use clippy_utils::numeric_literal; +use clippy_utils::sugg::Sugg; +use rustc_errors::Applicability; +use rustc_hir::Expr; +use rustc_lint::LateContext; +use std::f32::consts as f32_consts; +use std::f64::consts as f64_consts; + +use super::{IMPRECISE_FLOPS, SUBOPTIMAL_FLOPS}; + +// Returns an integer if the float constant is a whole number and it can be +// converted to an integer without loss of precision. For now we only check +// ranges [-16777215, 16777216) for type f32 as whole number floats outside +// this range are lossy and ambiguous. +#[expect(clippy::cast_possible_truncation)] +fn get_integer_from_float_constant(value: &Constant) -> Option { + match value { + F32(num) if num.fract() == 0.0 => { + if (-16_777_215.0..16_777_216.0).contains(num) { + Some(num.round() as i32) + } else { + None + } + }, + F64(num) if num.fract() == 0.0 => { + if (-2_147_483_648.0..2_147_483_648.0).contains(num) { + Some(num.round() as i32) + } else { + None + } + }, + _ => None, + } +} + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { + // Check receiver + if let Some(value) = ConstEvalCtxt::new(cx).eval(receiver) + && let Some(method) = if F32(f32_consts::E) == value || F64(f64_consts::E) == value { + Some("exp") + } else if F32(2.0) == value || F64(2.0) == value { + Some("exp2") + } else { + None + } + { + span_lint_and_then( + cx, + SUBOPTIMAL_FLOPS, + expr.span, + "exponent for bases 2 and e can be computed more accurately", + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = super::lib::prepare_receiver_sugg(cx, &args[0], &mut app); + diag.span_suggestion(expr.span, "consider using", format!("{recv}.{method}()"), app); + }, + ); + } + + // Check argument + if let Some(value) = ConstEvalCtxt::new(cx).eval(&args[0]) { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(); + let (lint, help, suggestion) = if F32(1.0 / 2.0) == value || F64(1.0 / 2.0) == value { + ( + SUBOPTIMAL_FLOPS, + "square-root of a number can be computed more efficiently and accurately", + format!("{recv}.sqrt()"), + ) + } else if F32(1.0 / 3.0) == value || F64(1.0 / 3.0) == value { + ( + IMPRECISE_FLOPS, + "cube-root of a number can be computed more accurately", + format!("{recv}.cbrt()"), + ) + } else if let Some(exponent) = get_integer_from_float_constant(&value) { + ( + SUBOPTIMAL_FLOPS, + "exponentiation with integer powers can be computed more efficiently", + format!( + "{recv}.powi({})", + numeric_literal::format(&exponent.to_string(), None, false) + ), + ) + } else { + return; + }; + + span_lint_and_sugg(cx, lint, expr.span, help, "consider using", suggestion, app); + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/powi.rs b/clippy_lints/src/floating_point_arithmetic/powi.rs new file mode 100644 index 000000000000..a61a2a82c023 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/powi.rs @@ -0,0 +1,70 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::Int; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::sugg::Sugg; +use clippy_utils::{get_parent_expr, sym}; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind, PathSegment}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; + +use super::SUBOPTIMAL_FLOPS; + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, receiver: &Expr<'_>, args: &[Expr<'_>]) { + if let Some(value) = ConstEvalCtxt::new(cx).eval(&args[0]) + && value == Int(2) + && let Some(parent) = get_parent_expr(cx, expr) + { + if let Some(grandparent) = get_parent_expr(cx, parent) + && let ExprKind::MethodCall(PathSegment { ident: method, .. }, receiver, ..) = grandparent.kind + && method.name == sym::sqrt + // we don't care about the applicability as this is an early-return condition + && super::hypot::detect(cx, receiver, &mut Applicability::Unspecified).is_some() + { + return; + } + + if let ExprKind::Binary( + Spanned { + node: op @ (BinOpKind::Add | BinOpKind::Sub), + .. + }, + lhs, + rhs, + ) = parent.kind + { + span_lint_and_then( + cx, + SUBOPTIMAL_FLOPS, + parent.span, + "multiply and add expressions can be calculated more efficiently and accurately", + |diag| { + let other_addend = if lhs.hir_id == expr.hir_id { rhs } else { lhs }; + + // Negate expr if original code has subtraction and expr is on the right side + let maybe_neg_sugg = |expr, hir_id, app: &mut _| { + let sugg = Sugg::hir_with_applicability(cx, expr, "_", app); + if matches!(op, BinOpKind::Sub) && hir_id == rhs.hir_id { + -sugg + } else { + sugg + } + }; + + let mut app = Applicability::MachineApplicable; + diag.span_suggestion( + parent.span, + "consider using", + format!( + "{}.mul_add({}, {})", + Sugg::hir_with_applicability(cx, receiver, "_", &mut app).maybe_paren(), + maybe_neg_sugg(receiver, expr.hir_id, &mut app), + maybe_neg_sugg(other_addend, other_addend.hir_id, &mut app), + ), + app, + ); + }, + ); + } + } +} diff --git a/clippy_lints/src/floating_point_arithmetic/radians.rs b/clippy_lints/src/floating_point_arithmetic/radians.rs new file mode 100644 index 000000000000..2021f00a97e8 --- /dev/null +++ b/clippy_lints/src/floating_point_arithmetic/radians.rs @@ -0,0 +1,89 @@ +use clippy_utils::consts::ConstEvalCtxt; +use clippy_utils::consts::Constant::{F32, F64}; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::sugg::Sugg; +use rustc_ast::ast; +use rustc_errors::Applicability; +use rustc_hir::{BinOpKind, Expr, ExprKind}; +use rustc_lint::LateContext; +use rustc_span::source_map::Spanned; +use std::f32::consts as f32_consts; +use std::f64::consts as f64_consts; + +use super::SUBOPTIMAL_FLOPS; + +pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>) { + if let ExprKind::Binary( + Spanned { + node: BinOpKind::Div, .. + }, + div_lhs, + div_rhs, + ) = expr.kind + && let ExprKind::Binary( + Spanned { + node: BinOpKind::Mul, .. + }, + mul_lhs, + mul_rhs, + ) = div_lhs.kind + && let ecx = ConstEvalCtxt::new(cx) + && let Some(rvalue) = ecx.eval(div_rhs) + && let Some(lvalue) = ecx.eval(mul_rhs) + { + // TODO: also check for constant values near PI/180 or 180/PI + if (F32(f32_consts::PI) == rvalue || F64(f64_consts::PI) == rvalue) + && (F32(180_f32) == lvalue || F64(180_f64) == lvalue) + { + span_lint_and_then( + cx, + SUBOPTIMAL_FLOPS, + expr.span, + "conversion to degrees can be done more accurately", + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, mul_lhs, "num", &mut app); + let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind + && let ast::LitKind::Float(ref value, float_type) = literal.node + && float_type == ast::LitFloatType::Unsuffixed + { + if value.as_str().ends_with('.') { + format!("{recv}0_f64.to_degrees()") + } else { + format!("{recv}_f64.to_degrees()") + } + } else { + format!("{}.to_degrees()", recv.maybe_paren()) + }; + diag.span_suggestion(expr.span, "consider using", proposal, app); + }, + ); + } else if (F32(180_f32) == rvalue || F64(180_f64) == rvalue) + && (F32(f32_consts::PI) == lvalue || F64(f64_consts::PI) == lvalue) + { + span_lint_and_then( + cx, + SUBOPTIMAL_FLOPS, + expr.span, + "conversion to radians can be done more accurately", + |diag| { + let mut app = Applicability::MachineApplicable; + let recv = Sugg::hir_with_applicability(cx, mul_lhs, "num", &mut app); + let proposal = if let ExprKind::Lit(literal) = mul_lhs.kind + && let ast::LitKind::Float(ref value, float_type) = literal.node + && float_type == ast::LitFloatType::Unsuffixed + { + if value.as_str().ends_with('.') { + format!("{recv}0_f64.to_radians()") + } else { + format!("{recv}_f64.to_radians()") + } + } else { + format!("{}.to_radians()", recv.maybe_paren()) + }; + diag.span_suggestion(expr.span, "consider using", proposal, app); + }, + ); + } + } +} From 26be33ae18b2aaa376eba9e755b54bc11e104a7b Mon Sep 17 00:00:00 2001 From: A4-Tacks Date: Thu, 8 Jan 2026 15:07:24 +0800 Subject: [PATCH 0147/1843] Fix not disable string escape highlights Example --- with config `strings: false` ```rust fn main() { format_args!("foo\nbar\invalid"); } ``` **Before this PR** ```rust fn main() { format_args!("foo\nbar\invalid"); // ^^ EscapeSequence // ^^ InvalidEscapeSequence } ``` **After this PR** ```rust fn main() { format_args!("foo\nbar\invalid"); } ``` --- .../crates/ide/src/syntax_highlighting.rs | 16 ++++--- .../ide/src/syntax_highlighting/escape.rs | 43 ++++++++++++----- .../ide/src/syntax_highlighting/highlights.rs | 8 +++- .../test_data/highlight_strings_disabled.html | 47 +++++++++++++++++++ .../ide/src/syntax_highlighting/tests.rs | 17 +++++++ 5 files changed, 113 insertions(+), 18 deletions(-) create mode 100644 src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings_disabled.html diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting.rs index e7c5f95a250e..e64fd6488f2a 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting.rs @@ -513,21 +513,21 @@ fn string_injections( ); if !string.is_raw() { - highlight_escape_string(hl, &string); + highlight_escape_string(hl, config, &string); } } } else if let Some(byte_string) = ast::ByteString::cast(token.clone()) { if !byte_string.is_raw() { - highlight_escape_string(hl, &byte_string); + highlight_escape_string(hl, config, &byte_string); } } else if let Some(c_string) = ast::CString::cast(token.clone()) { if !c_string.is_raw() { - highlight_escape_string(hl, &c_string); + highlight_escape_string(hl, config, &c_string); } } else if let Some(char) = ast::Char::cast(token.clone()) { - highlight_escape_char(hl, &char) + highlight_escape_char(hl, config, &char) } else if let Some(byte) = ast::Byte::cast(token) { - highlight_escape_byte(hl, &byte) + highlight_escape_byte(hl, config, &byte) } ControlFlow::Continue(()) } @@ -586,7 +586,11 @@ fn descend_token( fn filter_by_config(highlight: &mut Highlight, config: &HighlightConfig<'_>) -> bool { match &mut highlight.tag { - HlTag::StringLiteral if !config.strings => return false, + HlTag::StringLiteral | HlTag::EscapeSequence | HlTag::InvalidEscapeSequence + if !config.strings => + { + return false; + } HlTag::Comment if !config.comments => return false, // If punctuation is disabled, make the macro bang part of the macro call again. tag @ HlTag::Punctuation(HlPunct::MacroBang) => { diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/escape.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/escape.rs index 094f88f3a864..4da69cc43d9e 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/escape.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/escape.rs @@ -1,10 +1,14 @@ //! Syntax highlighting for escape sequences use crate::syntax_highlighting::highlights::Highlights; -use crate::{HlRange, HlTag}; +use crate::{HighlightConfig, HlRange, HlTag}; use syntax::ast::{Byte, Char, IsString}; use syntax::{AstToken, TextRange, TextSize}; -pub(super) fn highlight_escape_string(stack: &mut Highlights, string: &T) { +pub(super) fn highlight_escape_string( + stack: &mut Highlights, + config: &HighlightConfig<'_>, + string: &T, +) { let text = string.text(); let start = string.syntax().text_range().start(); string.escaped_char_ranges(&mut |piece_range, char| { @@ -13,16 +17,23 @@ pub(super) fn highlight_escape_string(stack: &mut Highlights, strin Ok(_) => HlTag::EscapeSequence, Err(_) => HlTag::InvalidEscapeSequence, }; - stack.add(HlRange { - range: piece_range + start, - highlight: highlight.into(), - binding_hash: None, - }); + stack.add_with( + config, + HlRange { + range: piece_range + start, + highlight: highlight.into(), + binding_hash: None, + }, + ); } }); } -pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char) { +pub(super) fn highlight_escape_char( + stack: &mut Highlights, + config: &HighlightConfig<'_>, + char: &Char, +) { if char.value().is_err() { // We do not emit invalid escapes highlighting here. The lexer would likely be in a bad // state and this token contains junk, since `'` is not a reliable delimiter (consider @@ -43,10 +54,17 @@ pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char) { char.syntax().text_range().start() + TextSize::from(1), TextSize::from(text.len() as u32), ); - stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None }) + stack.add_with( + config, + HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None }, + ) } -pub(super) fn highlight_escape_byte(stack: &mut Highlights, byte: &Byte) { +pub(super) fn highlight_escape_byte( + stack: &mut Highlights, + config: &HighlightConfig<'_>, + byte: &Byte, +) { if byte.value().is_err() { // See `highlight_escape_char` for why no error highlighting here. return; @@ -65,5 +83,8 @@ pub(super) fn highlight_escape_byte(stack: &mut Highlights, byte: &Byte) { byte.syntax().text_range().start() + TextSize::from(2), TextSize::from(text.len() as u32), ); - stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None }) + stack.add_with( + config, + HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None }, + ) } diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlights.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlights.rs index 340290eafedb..6fe4d0844338 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlights.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlights.rs @@ -4,7 +4,7 @@ use std::iter; use stdx::equal_range_by; use syntax::TextRange; -use crate::{HlRange, HlTag}; +use crate::{HighlightConfig, HlRange, HlTag}; pub(super) struct Highlights { root: Node, @@ -26,6 +26,12 @@ impl Highlights { self.root.add(hl_range); } + pub(super) fn add_with(&mut self, config: &HighlightConfig<'_>, mut hl_range: HlRange) { + if super::filter_by_config(&mut hl_range.highlight, config) { + self.root.add(hl_range); + } + } + pub(super) fn to_vec(&self) -> Vec { let mut res = Vec::new(); self.root.flatten(&mut res); diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings_disabled.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings_disabled.html new file mode 100644 index 000000000000..344d0c2ff03b --- /dev/null +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings_disabled.html @@ -0,0 +1,47 @@ + + +
fn main() {
+    format_args!("foo\nbar");
+    format_args!("foo\invalid");
+}
\ No newline at end of file diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs index 89a5e434f90c..8b529cf10f7f 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs @@ -1498,6 +1498,23 @@ fn main() { ); } +#[test] +fn test_strings_highlighting_disabled() { + // Test that comments are not highlighted when disabled + check_highlighting_with_config( + r#" +//- minicore: fmt +fn main() { + format_args!("foo\nbar"); + format_args!("foo\invalid"); +} +"#, + HighlightConfig { strings: false, ..HL_CONFIG }, + expect_file!["./test_data/highlight_strings_disabled.html"], + false, + ); +} + #[test] fn regression_20952() { check_highlighting( From 7d0b1e144917b46d3e2b5c18ef6d434b098dab08 Mon Sep 17 00:00:00 2001 From: Coca Date: Fri, 2 Jan 2026 20:59:22 +0000 Subject: [PATCH 0148/1843] `transmuting_null`: Add checks for `without_provenance` and `without_provenance_mut` Currently `without_provenance`/`without_provenance_mut` do not have a `rustc_diagnostic_item` so this change is dependent on them being added before being ready to be used. changelog: [`transmuting_null`]: now checks for [`ptr::without_provenance`](https://doc.rust-lang.org/core/ptr/fn.without_provenance.html) and [`ptr::without_provenance_mut`](https://doc.rust-lang.org/core/ptr/fn.without_provenance_mut.html) which create null pointers --- clippy_lints/src/transmute/transmuting_null.rs | 12 ++++++++++++ tests/ui/transmuting_null.rs | 11 +++++++++++ tests/ui/transmuting_null.stderr | 14 +++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/clippy_lints/src/transmute/transmuting_null.rs b/clippy_lints/src/transmute/transmuting_null.rs index 3f435f255d91..4f06d98703f6 100644 --- a/clippy_lints/src/transmute/transmuting_null.rs +++ b/clippy_lints/src/transmute/transmuting_null.rs @@ -42,6 +42,18 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, arg: &'t return true; } + // Catching: + // `std::mem::transmute(std::ptr::without_provenance::(0))` + // `std::mem::transmute(std::ptr::without_provenance_mut::(0))` + if let ExprKind::Call(func1, [arg1]) = arg.kind + && (func1.basic_res().is_diag_item(cx, sym::ptr_without_provenance) + || func1.basic_res().is_diag_item(cx, sym::ptr_without_provenance_mut)) + && is_integer_const(cx, arg1, 0) + { + span_lint(cx, TRANSMUTING_NULL, expr.span, LINT_MSG); + return true; + } + // Catching: // `std::mem::transmute({ 0 as *const u64 })` and similar const blocks if let ExprKind::Block(block, _) = arg.kind diff --git a/tests/ui/transmuting_null.rs b/tests/ui/transmuting_null.rs index 00aa35dff803..efa4c5cfdc2d 100644 --- a/tests/ui/transmuting_null.rs +++ b/tests/ui/transmuting_null.rs @@ -47,9 +47,20 @@ fn transumute_single_expr_blocks() { } } +fn transmute_pointer_creators() { + unsafe { + let _: &u64 = std::mem::transmute(std::ptr::without_provenance::(0)); + //~^ transmuting_null + + let _: &u64 = std::mem::transmute(std::ptr::without_provenance_mut::(0)); + //~^ transmuting_null + } +} + fn main() { one_liners(); transmute_const(); transmute_const_int(); transumute_single_expr_blocks(); + transmute_pointer_creators(); } diff --git a/tests/ui/transmuting_null.stderr b/tests/ui/transmuting_null.stderr index e1de391813bd..3c6c28f31d0d 100644 --- a/tests/ui/transmuting_null.stderr +++ b/tests/ui/transmuting_null.stderr @@ -37,5 +37,17 @@ error: transmuting a known null pointer into a reference LL | let _: &u64 = std::mem::transmute(const { u64::MIN as *const u64 }); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error: aborting due to 6 previous errors +error: transmuting a known null pointer into a reference + --> tests/ui/transmuting_null.rs:52:23 + | +LL | let _: &u64 = std::mem::transmute(std::ptr::without_provenance::(0)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: transmuting a known null pointer into a reference + --> tests/ui/transmuting_null.rs:55:23 + | +LL | let _: &u64 = std::mem::transmute(std::ptr::without_provenance_mut::(0)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 8 previous errors From 6a9dae4f3e679c298c6d6ad83f65d2d9c51a02f0 Mon Sep 17 00:00:00 2001 From: Philipp Krones Date: Fri, 9 Jan 2026 10:37:00 +0100 Subject: [PATCH 0149/1843] Merge commit '500e0ff18726cd44b23004a02fc1b99f52c11ab1' into clippy-subtree-update --- COPYRIGHT | 2 +- Cargo.toml | 2 +- LICENSE-APACHE | 2 +- LICENSE-MIT | 2 +- README.md | 2 +- clippy_lints/src/attrs/allow_attributes.rs | 9 +- clippy_lints/src/attrs/mod.rs | 6 +- .../src/attrs/should_panic_without_expect.rs | 2 +- clippy_lints/src/bool_assert_comparison.rs | 33 +++-- clippy_lints/src/booleans.rs | 3 +- clippy_lints/src/cargo/mod.rs | 2 +- clippy_lints/src/cfg_not_test.rs | 13 +- clippy_lints/src/checked_conversions.rs | 5 +- .../src/derive/derive_ord_xor_partial_ord.rs | 9 +- clippy_lints/src/derive/mod.rs | 2 +- .../src/doc/include_in_doc_without_cfg.rs | 2 +- clippy_lints/src/doc/mod.rs | 10 +- clippy_lints/src/double_parens.rs | 2 + clippy_lints/src/implicit_saturating_sub.rs | 27 +++- clippy_lints/src/incompatible_msrv.rs | 10 +- clippy_lints/src/inherent_impl.rs | 23 ++- clippy_lints/src/large_include_file.rs | 2 +- clippy_lints/src/large_stack_arrays.rs | 9 ++ clippy_lints/src/loops/for_kv_map.rs | 24 +++- clippy_lints/src/loops/mod.rs | 2 +- clippy_lints/src/loops/never_loop.rs | 4 +- clippy_lints/src/manual_ignore_case_cmp.rs | 10 +- clippy_lints/src/manual_ilog2.rs | 4 +- clippy_lints/src/matches/manual_ok_err.rs | 2 +- clippy_lints/src/matches/match_as_ref.rs | 5 +- clippy_lints/src/matches/match_bool.rs | 96 ++++++------- .../src/matches/redundant_pattern_match.rs | 23 ++- clippy_lints/src/methods/is_empty.rs | 5 +- clippy_lints/src/methods/iter_kv_map.rs | 7 +- clippy_lints/src/methods/unnecessary_fold.rs | 111 +++++++++----- .../src/methods/unnecessary_to_owned.rs | 22 +-- .../src/missing_enforced_import_rename.rs | 3 +- clippy_lints/src/multiple_bound_locations.rs | 2 +- clippy_lints/src/mutex_atomic.rs | 2 +- clippy_lints/src/needless_bool.rs | 6 +- clippy_lints/src/needless_for_each.rs | 125 +++++++++------- clippy_lints/src/new_without_default.rs | 36 +++-- clippy_lints/src/operators/cmp_owned.rs | 58 +++----- clippy_lints/src/operators/manual_div_ceil.rs | 127 +++++++++------- .../src/operators/manual_is_multiple_of.rs | 4 +- clippy_lints/src/question_mark.rs | 7 +- clippy_lints/src/single_range_in_vec_init.rs | 2 +- clippy_lints/src/strings.rs | 8 +- clippy_lints/src/strlen_on_c_strings.rs | 4 +- .../src/transmute/transmuting_null.rs | 20 ++- clippy_lints/src/useless_conversion.rs | 14 +- clippy_lints/src/utils/author.rs | 1 - .../src/lint_without_lint_pass.rs | 4 +- clippy_utils/README.md | 4 +- clippy_utils/src/ast_utils/mod.rs | 4 +- clippy_utils/src/attrs.rs | 6 +- clippy_utils/src/consts.rs | 8 +- clippy_utils/src/hir_utils.rs | 4 +- clippy_utils/src/lib.rs | 29 +++- clippy_utils/src/res.rs | 4 +- clippy_utils/src/sugg.rs | 2 +- clippy_utils/src/sym.rs | 1 + rust-toolchain.toml | 2 +- rustc_tools_util/README.md | 2 +- .../clippy.toml | 3 +- .../conf_missing_enforced_import_rename.fixed | 1 + .../conf_missing_enforced_import_rename.rs | 1 + ...conf_missing_enforced_import_rename.stderr | 2 +- tests/ui/bool_assert_comparison.fixed | 13 ++ tests/ui/bool_assert_comparison.rs | 13 ++ tests/ui/bool_assert_comparison.stderr | 26 +++- tests/ui/checked_conversions.fixed | 16 +++ tests/ui/checked_conversions.rs | 16 +++ tests/ui/checked_conversions.stderr | 42 +++--- tests/ui/cmp_owned/with_suggestion.fixed | 29 ++++ tests/ui/cmp_owned/with_suggestion.rs | 29 ++++ tests/ui/cmp_owned/with_suggestion.stderr | 8 +- tests/ui/derive_ord_xor_partial_ord.rs | 14 ++ tests/ui/double_parens.fixed | 16 +++ tests/ui/double_parens.rs | 16 +++ tests/ui/for_kv_map.fixed | 17 +++ tests/ui/for_kv_map.rs | 17 +++ tests/ui/for_kv_map.stderr | 14 +- tests/ui/impl.rs | 42 +++++- tests/ui/impl.stderr | 90 ++++++++++-- tests/ui/implicit_saturating_sub.fixed | 8 ++ tests/ui/implicit_saturating_sub.rs | 8 ++ tests/ui/implicit_saturating_sub.stderr | 8 +- tests/ui/iter_kv_map.fixed | 6 + tests/ui/iter_kv_map.rs | 6 + tests/ui/iter_kv_map.stderr | 8 +- tests/ui/manual_div_ceil.fixed | 29 ++++ tests/ui/manual_div_ceil.rs | 29 ++++ tests/ui/manual_div_ceil.stderr | 20 ++- tests/ui/manual_div_ceil_with_feature.fixed | 29 ++++ tests/ui/manual_div_ceil_with_feature.rs | 29 ++++ tests/ui/manual_div_ceil_with_feature.stderr | 20 ++- tests/ui/manual_ignore_case_cmp.fixed | 20 +++ tests/ui/manual_ignore_case_cmp.rs | 20 +++ tests/ui/manual_ignore_case_cmp.stderr | 26 +++- tests/ui/manual_ilog2.fixed | 17 +++ tests/ui/manual_ilog2.rs | 17 +++ tests/ui/manual_ilog2.stderr | 14 +- tests/ui/manual_is_multiple_of.fixed | 16 +++ tests/ui/manual_is_multiple_of.rs | 16 +++ tests/ui/manual_is_multiple_of.stderr | 8 +- tests/ui/manual_ok_err.fixed | 10 ++ tests/ui/manual_ok_err.rs | 14 ++ tests/ui/manual_ok_err.stderr | 13 +- tests/ui/match_as_ref.fixed | 10 ++ tests/ui/match_as_ref.rs | 14 ++ tests/ui/match_as_ref.stderr | 23 ++- tests/ui/match_bool.fixed | 11 ++ tests/ui/match_bool.rs | 15 ++ tests/ui/match_bool.stderr | 12 +- tests/ui/mutex_atomic.fixed | 12 ++ tests/ui/mutex_atomic.rs | 12 ++ tests/ui/mutex_atomic.stderr | 10 +- tests/ui/needless_bool_assign.fixed | 19 +++ tests/ui/needless_bool_assign.rs | 23 +++ tests/ui/needless_bool_assign.stderr | 12 +- tests/ui/needless_for_each_fixable.fixed | 8 ++ tests/ui/needless_for_each_fixable.rs | 8 ++ tests/ui/needless_for_each_fixable.stderr | 19 ++- tests/ui/never_loop_iterator_reduction.rs | 9 +- tests/ui/never_loop_iterator_reduction.stderr | 2 +- tests/ui/new_without_default.fixed | 55 +++++++ tests/ui/new_without_default.rs | 36 +++++ tests/ui/new_without_default.stderr | 55 ++++++- tests/ui/question_mark.fixed | 17 ++- tests/ui/question_mark.rs | 22 ++- tests/ui/question_mark.stderr | 19 ++- .../redundant_pattern_matching_option.fixed | 13 ++ tests/ui/redundant_pattern_matching_option.rs | 13 ++ .../redundant_pattern_matching_option.stderr | 14 +- .../redundant_pattern_matching_result.fixed | 20 +++ tests/ui/redundant_pattern_matching_result.rs | 24 ++++ .../redundant_pattern_matching_result.stderr | 19 ++- .../ui/single_range_in_vec_init_unfixable.rs | 12 ++ .../single_range_in_vec_init_unfixable.stderr | 16 +++ tests/ui/str_to_string.fixed | 14 ++ tests/ui/str_to_string.rs | 14 ++ tests/ui/str_to_string.stderr | 8 +- tests/ui/string_from_utf8_as_bytes.fixed | 10 ++ tests/ui/string_from_utf8_as_bytes.rs | 10 ++ tests/ui/string_from_utf8_as_bytes.stderr | 10 +- tests/ui/transmuting_null.rs | 11 ++ tests/ui/transmuting_null.stderr | 14 +- tests/ui/unnecessary_fold.fixed | 11 ++ tests/ui/unnecessary_fold.rs | 11 ++ tests/ui/unnecessary_fold.stderr | 17 ++- tests/ui/unnecessary_to_owned.fixed | 9 ++ tests/ui/unnecessary_to_owned.rs | 9 ++ tests/ui/unnecessary_to_owned.stderr | 8 +- tests/ui/useless_conversion.fixed | 10 ++ tests/ui/useless_conversion.rs | 10 ++ tests/ui/useless_conversion.stderr | 135 ++++++++++-------- triagebot.toml | 1 - 158 files changed, 2147 insertions(+), 518 deletions(-) create mode 100644 tests/ui/single_range_in_vec_init_unfixable.rs create mode 100644 tests/ui/single_range_in_vec_init_unfixable.stderr diff --git a/COPYRIGHT b/COPYRIGHT index f402dcf465a3..d3b4c9e5fb2c 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,6 +1,6 @@ // REUSE-IgnoreStart -Copyright 2014-2025 The Rust Project Developers +Copyright (c) The Rust Project Contributors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/Cargo.toml b/Cargo.toml index 67078adea2b4..7379dcbb7b37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ walkdir = "2.3" filetime = "0.2.9" itertools = "0.12" pulldown-cmark = { version = "0.11", default-features = false, features = ["html"] } -askama = { version = "0.14", default-features = false, features = ["alloc", "config", "derive"] } +askama = { version = "0.15", default-features = false, features = ["alloc", "config", "derive"] } [dev-dependencies.toml] version = "0.9.7" diff --git a/LICENSE-APACHE b/LICENSE-APACHE index 9990a0cec474..773ae298cc19 100644 --- a/LICENSE-APACHE +++ b/LICENSE-APACHE @@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2014-2025 The Rust Project Developers +Copyright (c) The Rust Project Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/LICENSE-MIT b/LICENSE-MIT index 5d6e36ef6bfc..9549420685cc 100644 --- a/LICENSE-MIT +++ b/LICENSE-MIT @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2014-2025 The Rust Project Developers +Copyright (c) The Rust Project Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated diff --git a/README.md b/README.md index 78498c73ae78..8bccd040c1f9 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ If you want to contribute to Clippy, you can find more information in [CONTRIBUT -Copyright 2014-2025 The Rust Project Developers +Copyright (c) The Rust Project Contributors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/clippy_lints/src/attrs/allow_attributes.rs b/clippy_lints/src/attrs/allow_attributes.rs index 84b65d3185e3..0235b130b6c0 100644 --- a/clippy_lints/src/attrs/allow_attributes.rs +++ b/clippy_lints/src/attrs/allow_attributes.rs @@ -1,10 +1,10 @@ use super::ALLOW_ATTRIBUTES; use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::is_from_proc_macro; +use rustc_ast::attr::AttributeExt; use rustc_ast::{AttrStyle, Attribute}; use rustc_errors::Applicability; use rustc_lint::{EarlyContext, LintContext}; -use rustc_ast::attr::AttributeExt; // Separate each crate's features. pub fn check<'cx>(cx: &EarlyContext<'cx>, attr: &'cx Attribute) { @@ -15,12 +15,7 @@ pub fn check<'cx>(cx: &EarlyContext<'cx>, attr: &'cx Attribute) { { #[expect(clippy::collapsible_span_lint_calls, reason = "rust-clippy#7797")] span_lint_and_then(cx, ALLOW_ATTRIBUTES, path_span, "#[allow] attribute found", |diag| { - diag.span_suggestion( - path_span, - "replace it with", - "expect", - Applicability::MachineApplicable, - ); + diag.span_suggestion(path_span, "replace it with", "expect", Applicability::MachineApplicable); }); } } diff --git a/clippy_lints/src/attrs/mod.rs b/clippy_lints/src/attrs/mod.rs index 366f5873a1aa..42c321df61c1 100644 --- a/clippy_lints/src/attrs/mod.rs +++ b/clippy_lints/src/attrs/mod.rs @@ -16,7 +16,7 @@ mod utils; use clippy_config::Conf; use clippy_utils::diagnostics::span_lint_and_help; use clippy_utils::msrvs::{self, Msrv, MsrvStack}; -use rustc_ast::{self as ast, AttrArgs, AttrKind, Attribute, MetaItemInner, MetaItemKind, AttrItemKind}; +use rustc_ast::{self as ast, AttrArgs, AttrItemKind, AttrKind, Attribute, MetaItemInner, MetaItemKind}; use rustc_hir::{ImplItem, Item, ItemKind, TraitItem}; use rustc_lint::{EarlyContext, EarlyLintPass, LateContext, LateLintPass}; use rustc_session::impl_lint_pass; @@ -604,7 +604,9 @@ impl EarlyLintPass for PostExpansionEarlyAttributes { if attr.has_name(sym::ignore) && match &attr.kind { - AttrKind::Normal(normal_attr) => !matches!(normal_attr.item.args, AttrItemKind::Unparsed(AttrArgs::Eq { .. })), + AttrKind::Normal(normal_attr) => { + !matches!(normal_attr.item.args, AttrItemKind::Unparsed(AttrArgs::Eq { .. })) + }, AttrKind::DocComment(..) => true, } { diff --git a/clippy_lints/src/attrs/should_panic_without_expect.rs b/clippy_lints/src/attrs/should_panic_without_expect.rs index b854a3070bef..1a9abd88a46a 100644 --- a/clippy_lints/src/attrs/should_panic_without_expect.rs +++ b/clippy_lints/src/attrs/should_panic_without_expect.rs @@ -2,7 +2,7 @@ use super::{Attribute, SHOULD_PANIC_WITHOUT_EXPECT}; use clippy_utils::diagnostics::span_lint_and_sugg; use rustc_ast::token::{Token, TokenKind}; use rustc_ast::tokenstream::TokenTree; -use rustc_ast::{AttrArgs, AttrKind, AttrItemKind}; +use rustc_ast::{AttrArgs, AttrItemKind, AttrKind}; use rustc_errors::Applicability; use rustc_lint::EarlyContext; use rustc_span::sym; diff --git a/clippy_lints/src/bool_assert_comparison.rs b/clippy_lints/src/bool_assert_comparison.rs index f31b67f470f9..165941a859f7 100644 --- a/clippy_lints/src/bool_assert_comparison.rs +++ b/clippy_lints/src/bool_assert_comparison.rs @@ -1,5 +1,6 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::macros::{find_assert_eq_args, root_macro_call_first_node}; +use clippy_utils::source::walk_span_to_context; use clippy_utils::sugg::Sugg; use clippy_utils::sym; use clippy_utils::ty::{implements_trait, is_copy}; @@ -130,22 +131,24 @@ impl<'tcx> LateLintPass<'tcx> for BoolAssertComparison { let mut suggestions = vec![(name_span, non_eq_mac.to_string()), (lit_span, String::new())]; - if let Some(sugg) = Sugg::hir_opt(cx, non_lit_expr) { - let sugg = if bool_value ^ eq_macro { - !sugg.maybe_paren() - } else if ty::Bool == *non_lit_ty.kind() { - sugg - } else { - !!sugg.maybe_paren() - }; - suggestions.push((non_lit_expr.span, sugg.to_string())); + let mut applicability = Applicability::MachineApplicable; + let sugg = Sugg::hir_with_context(cx, non_lit_expr, macro_call.span.ctxt(), "..", &mut applicability); + let sugg = if bool_value ^ eq_macro { + !sugg.maybe_paren() + } else if ty::Bool == *non_lit_ty.kind() { + sugg + } else { + !!sugg.maybe_paren() + }; + let non_lit_expr_span = + walk_span_to_context(non_lit_expr.span, macro_call.span.ctxt()).unwrap_or(non_lit_expr.span); + suggestions.push((non_lit_expr_span, sugg.to_string())); - diag.multipart_suggestion( - format!("replace it with `{non_eq_mac}!(..)`"), - suggestions, - Applicability::MachineApplicable, - ); - } + diag.multipart_suggestion( + format!("replace it with `{non_eq_mac}!(..)`"), + suggestions, + applicability, + ); }, ); } diff --git a/clippy_lints/src/booleans.rs b/clippy_lints/src/booleans.rs index a04a56d72bc0..0bd459d8b021 100644 --- a/clippy_lints/src/booleans.rs +++ b/clippy_lints/src/booleans.rs @@ -15,6 +15,7 @@ use rustc_lint::{LateContext, LateLintPass, Level}; use rustc_session::impl_lint_pass; use rustc_span::def_id::LocalDefId; use rustc_span::{Span, Symbol, SyntaxContext}; +use std::fmt::Write as _; declare_clippy_lint! { /// ### What it does @@ -356,7 +357,7 @@ impl SuggestContext<'_, '_, '_> { if app != Applicability::MachineApplicable { return None; } - self.output.push_str(&(!snip).to_string()); + let _cannot_fail = write!(&mut self.output, "{}", &(!snip)); } }, True | False | Not(_) => { diff --git a/clippy_lints/src/cargo/mod.rs b/clippy_lints/src/cargo/mod.rs index 60371dcd7715..08d92adbacef 100644 --- a/clippy_lints/src/cargo/mod.rs +++ b/clippy_lints/src/cargo/mod.rs @@ -132,7 +132,7 @@ declare_clippy_lint! { /// Because this can be caused purely by the dependencies /// themselves, it's not always possible to fix this issue. /// In those cases, you can allow that specific crate using - /// the `allowed_duplicate_crates` configuration option. + /// the `allowed-duplicate-crates` configuration option. /// /// ### Example /// ```toml diff --git a/clippy_lints/src/cfg_not_test.rs b/clippy_lints/src/cfg_not_test.rs index ec543d02c9dd..88d07be0d4d4 100644 --- a/clippy_lints/src/cfg_not_test.rs +++ b/clippy_lints/src/cfg_not_test.rs @@ -1,10 +1,9 @@ use clippy_utils::diagnostics::span_lint_and_then; +use rustc_ast::attr::data_structures::CfgEntry; +use rustc_ast::{AttrItemKind, EarlyParsedAttribute}; use rustc_lint::{EarlyContext, EarlyLintPass}; use rustc_session::declare_lint_pass; -use rustc_ast::AttrItemKind; -use rustc_ast::EarlyParsedAttribute; use rustc_span::sym; -use rustc_ast::attr::data_structures::CfgEntry; declare_clippy_lint! { /// ### What it does @@ -40,7 +39,7 @@ impl EarlyLintPass for CfgNotTest { unreachable!() }; - if contains_not_test(&cfg, false) { + if contains_not_test(cfg, false) { span_lint_and_then( cx, CFG_NOT_TEST, @@ -58,11 +57,9 @@ impl EarlyLintPass for CfgNotTest { fn contains_not_test(cfg: &CfgEntry, not: bool) -> bool { match cfg { - CfgEntry::All(subs, _) | CfgEntry::Any(subs, _) => subs.iter().any(|item| { - contains_not_test(item, not) - }), + CfgEntry::All(subs, _) | CfgEntry::Any(subs, _) => subs.iter().any(|item| contains_not_test(item, not)), CfgEntry::Not(sub, _) => contains_not_test(sub, !not), CfgEntry::NameValue { name: sym::test, .. } => not, - _ => false + _ => false, } } diff --git a/clippy_lints/src/checked_conversions.rs b/clippy_lints/src/checked_conversions.rs index 9b3822f9d8f0..8303897d1294 100644 --- a/clippy_lints/src/checked_conversions.rs +++ b/clippy_lints/src/checked_conversions.rs @@ -1,7 +1,7 @@ use clippy_config::Conf; use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::msrvs::{self, Msrv}; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::snippet_with_context; use clippy_utils::{SpanlessEq, is_in_const_context, is_integer_literal, sym}; use rustc_errors::Applicability; use rustc_hir::{BinOpKind, Expr, ExprKind, QPath, TyKind}; @@ -80,7 +80,8 @@ impl LateLintPass<'_> for CheckedConversions { && self.msrv.meets(cx, msrvs::TRY_FROM) { let mut applicability = Applicability::MachineApplicable; - let snippet = snippet_with_applicability(cx, cv.expr_to_cast.span, "_", &mut applicability); + let (snippet, _) = + snippet_with_context(cx, cv.expr_to_cast.span, item.span.ctxt(), "_", &mut applicability); span_lint_and_sugg( cx, CHECKED_CONVERSIONS, diff --git a/clippy_lints/src/derive/derive_ord_xor_partial_ord.rs b/clippy_lints/src/derive/derive_ord_xor_partial_ord.rs index 2bd5e2cbfb1a..316d800a70c9 100644 --- a/clippy_lints/src/derive/derive_ord_xor_partial_ord.rs +++ b/clippy_lints/src/derive/derive_ord_xor_partial_ord.rs @@ -1,15 +1,16 @@ use clippy_utils::diagnostics::span_lint_hir_and_then; +use clippy_utils::fulfill_or_allowed; use rustc_hir::{self as hir, HirId}; use rustc_lint::LateContext; use rustc_middle::ty::Ty; -use rustc_span::{Span, sym}; +use rustc_span::sym; use super::DERIVE_ORD_XOR_PARTIAL_ORD; /// Implementation of the `DERIVE_ORD_XOR_PARTIAL_ORD` lint. pub(super) fn check<'tcx>( cx: &LateContext<'tcx>, - span: Span, + item: &hir::Item<'_>, trait_ref: &hir::TraitRef<'_>, ty: Ty<'tcx>, adt_hir_id: HirId, @@ -19,6 +20,8 @@ pub(super) fn check<'tcx>( && let Some(partial_ord_trait_def_id) = cx.tcx.lang_items().partial_ord_trait() && let Some(def_id) = &trait_ref.trait_def_id() && *def_id == ord_trait_def_id + && let item_hir_id = cx.tcx.local_def_id_to_hir_id(item.owner_id) + && !fulfill_or_allowed(cx, DERIVE_ORD_XOR_PARTIAL_ORD, [adt_hir_id]) { // Look for the PartialOrd implementations for `ty` cx.tcx.for_each_relevant_impl(partial_ord_trait_def_id, ty, |impl_id| { @@ -39,7 +42,7 @@ pub(super) fn check<'tcx>( "you are deriving `Ord` but have implemented `PartialOrd` explicitly" }; - span_lint_hir_and_then(cx, DERIVE_ORD_XOR_PARTIAL_ORD, adt_hir_id, span, mess, |diag| { + span_lint_hir_and_then(cx, DERIVE_ORD_XOR_PARTIAL_ORD, item_hir_id, item.span, mess, |diag| { if let Some(local_def_id) = impl_id.as_local() { let hir_id = cx.tcx.local_def_id_to_hir_id(local_def_id); diag.span_note(cx.tcx.hir_span(hir_id), "`PartialOrd` implemented here"); diff --git a/clippy_lints/src/derive/mod.rs b/clippy_lints/src/derive/mod.rs index eafe7c4bb9f2..86614201c406 100644 --- a/clippy_lints/src/derive/mod.rs +++ b/clippy_lints/src/derive/mod.rs @@ -208,7 +208,7 @@ impl<'tcx> LateLintPass<'tcx> for Derive { let is_automatically_derived = cx.tcx.is_automatically_derived(item.owner_id.to_def_id()); derived_hash_with_manual_eq::check(cx, item.span, trait_ref, ty, adt_hir_id, is_automatically_derived); - derive_ord_xor_partial_ord::check(cx, item.span, trait_ref, ty, adt_hir_id, is_automatically_derived); + derive_ord_xor_partial_ord::check(cx, item, trait_ref, ty, adt_hir_id, is_automatically_derived); if is_automatically_derived { unsafe_derive_deserialize::check(cx, item, trait_ref, ty, adt_hir_id); diff --git a/clippy_lints/src/doc/include_in_doc_without_cfg.rs b/clippy_lints/src/doc/include_in_doc_without_cfg.rs index f8e9e870f629..6c800f47b68a 100644 --- a/clippy_lints/src/doc/include_in_doc_without_cfg.rs +++ b/clippy_lints/src/doc/include_in_doc_without_cfg.rs @@ -1,6 +1,6 @@ use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::source::snippet_opt; -use rustc_ast::{AttrArgs, AttrKind, AttrStyle, Attribute, AttrItemKind}; +use rustc_ast::{AttrArgs, AttrItemKind, AttrKind, AttrStyle, Attribute}; use rustc_errors::Applicability; use rustc_lint::EarlyContext; diff --git a/clippy_lints/src/doc/mod.rs b/clippy_lints/src/doc/mod.rs index b11b2f8392c1..2b41275ee3a4 100644 --- a/clippy_lints/src/doc/mod.rs +++ b/clippy_lints/src/doc/mod.rs @@ -869,10 +869,12 @@ fn check_attrs(cx: &LateContext<'_>, valid_idents: &FxHashSet, attrs: &[ }), true, ); - let mut doc = fragments.iter().fold(String::new(), |mut acc, fragment| { - add_doc_fragment(&mut acc, fragment); - acc - }); + + let mut doc = String::with_capacity(fragments.iter().map(|frag| frag.doc.as_str().len() + 1).sum()); + + for fragment in &fragments { + add_doc_fragment(&mut doc, fragment); + } doc.pop(); if doc.trim().is_empty() { diff --git a/clippy_lints/src/double_parens.rs b/clippy_lints/src/double_parens.rs index 8defbeeaa5f2..351d29d87432 100644 --- a/clippy_lints/src/double_parens.rs +++ b/clippy_lints/src/double_parens.rs @@ -114,6 +114,8 @@ fn check_source(cx: &EarlyContext<'_>, inner: &Expr) -> bool { && inner.starts_with('(') && inner.ends_with(')') && outer_after_inner.trim_start().starts_with(')') + // Don't lint macro repetition patterns like `($($result),*)` where parens are necessary + && !inner.trim_start_matches('(').trim_start().starts_with("$(") { true } else { diff --git a/clippy_lints/src/implicit_saturating_sub.rs b/clippy_lints/src/implicit_saturating_sub.rs index 7b6f8729cb75..516f9e3aa60c 100644 --- a/clippy_lints/src/implicit_saturating_sub.rs +++ b/clippy_lints/src/implicit_saturating_sub.rs @@ -1,9 +1,13 @@ +use std::borrow::Cow; + use clippy_config::Conf; use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then}; use clippy_utils::msrvs::{self, Msrv}; +use clippy_utils::source::snippet_with_applicability; use clippy_utils::sugg::{Sugg, make_binop}; use clippy_utils::{ - SpanlessEq, eq_expr_value, higher, is_in_const_context, is_integer_literal, peel_blocks, peel_blocks_with_stmt, sym, + SpanlessEq, eq_expr_value, higher, is_in_const_context, is_integer_literal, is_integer_literal_untyped, + peel_blocks, peel_blocks_with_stmt, sym, }; use rustc_ast::ast::LitKind; use rustc_data_structures::packed::Pu128; @@ -238,10 +242,21 @@ fn check_subtraction( if eq_expr_value(cx, left, big_expr) && eq_expr_value(cx, right, little_expr) { // This part of the condition is voluntarily split from the one before to ensure that // if `snippet_opt` fails, it won't try the next conditions. - if (!is_in_const_context(cx) || msrv.meets(cx, msrvs::SATURATING_SUB_CONST)) - && let Some(big_expr_sugg) = Sugg::hir_opt(cx, big_expr).map(Sugg::maybe_paren) - && let Some(little_expr_sugg) = Sugg::hir_opt(cx, little_expr) - { + if !is_in_const_context(cx) || msrv.meets(cx, msrvs::SATURATING_SUB_CONST) { + let mut applicability = Applicability::MachineApplicable; + let big_expr_sugg = (if is_integer_literal_untyped(big_expr) { + let get_snippet = |span: Span| { + let snippet = snippet_with_applicability(cx, span, "..", &mut applicability); + let big_expr_ty = cx.typeck_results().expr_ty(big_expr); + Cow::Owned(format!("{snippet}_{big_expr_ty}")) + }; + Sugg::hir_from_snippet(cx, big_expr, get_snippet) + } else { + Sugg::hir_with_applicability(cx, big_expr, "..", &mut applicability) + }) + .maybe_paren(); + let little_expr_sugg = Sugg::hir_with_applicability(cx, little_expr, "..", &mut applicability); + let sugg = format!( "{}{big_expr_sugg}.saturating_sub({little_expr_sugg}){}", if is_composited { "{ " } else { "" }, @@ -254,7 +269,7 @@ fn check_subtraction( "manual arithmetic check found", "replace it with", sugg, - Applicability::MachineApplicable, + applicability, ); } } else if eq_expr_value(cx, left, little_expr) diff --git a/clippy_lints/src/incompatible_msrv.rs b/clippy_lints/src/incompatible_msrv.rs index e0149a23fccf..8d538ff1acba 100644 --- a/clippy_lints/src/incompatible_msrv.rs +++ b/clippy_lints/src/incompatible_msrv.rs @@ -3,14 +3,13 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::msrvs::Msrv; use clippy_utils::{is_in_const_context, is_in_test, sym}; use rustc_data_structures::fx::FxHashMap; -use rustc_hir::{self as hir, AmbigArg, Expr, ExprKind, HirId, RustcVersion, StabilityLevel, StableSince}; +use rustc_hir::attrs::AttributeKind; +use rustc_hir::{self as hir, AmbigArg, Expr, ExprKind, HirId, RustcVersion, StabilityLevel, StableSince, find_attr}; use rustc_lint::{LateContext, LateLintPass}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::impl_lint_pass; use rustc_span::def_id::{CrateNum, DefId}; use rustc_span::{ExpnKind, Span}; -use rustc_hir::attrs::AttributeKind; -use rustc_hir::find_attr; declare_clippy_lint! { /// ### What it does @@ -270,6 +269,9 @@ impl<'tcx> LateLintPass<'tcx> for IncompatibleMsrv { /// attribute. fn is_under_cfg_attribute(cx: &LateContext<'_>, hir_id: HirId) -> bool { cx.tcx.hir_parent_id_iter(hir_id).any(|id| { - find_attr!(cx.tcx.hir_attrs(id), AttributeKind::CfgTrace(..) | AttributeKind::CfgAttrTrace) + find_attr!( + cx.tcx.hir_attrs(id), + AttributeKind::CfgTrace(..) | AttributeKind::CfgAttrTrace + ) }) } diff --git a/clippy_lints/src/inherent_impl.rs b/clippy_lints/src/inherent_impl.rs index f59c7615d745..14928a1be13b 100644 --- a/clippy_lints/src/inherent_impl.rs +++ b/clippy_lints/src/inherent_impl.rs @@ -101,7 +101,21 @@ impl<'tcx> LateLintPass<'tcx> for MultipleInherentImpl { InherentImplLintScope::Crate => Criterion::Crate, }; let is_test = is_cfg_test(cx.tcx, hir_id) || is_in_cfg_test(cx.tcx, hir_id); - match type_map.entry((impl_ty, criterion, is_test)) { + let predicates = { + // Gets the predicates (bounds) for the given impl block, + // sorted for consistent comparison to allow distinguishing between impl blocks + // with different generic bounds. + let mut predicates = cx + .tcx + .predicates_of(impl_id) + .predicates + .iter() + .map(|(clause, _)| *clause) + .collect::>(); + predicates.sort_by_key(|c| format!("{c:?}")); + predicates + }; + match type_map.entry((impl_ty, predicates, criterion, is_test)) { Entry::Vacant(e) => { // Store the id for the first impl block of this type. The span is retrieved lazily. e.insert(IdOrSpan::Id(impl_id)); @@ -152,15 +166,12 @@ impl<'tcx> LateLintPass<'tcx> for MultipleInherentImpl { fn get_impl_span(cx: &LateContext<'_>, id: LocalDefId) -> Option { let id = cx.tcx.local_def_id_to_hir_id(id); if let Node::Item(&Item { - kind: ItemKind::Impl(impl_item), + kind: ItemKind::Impl(_), span, .. }) = cx.tcx.hir_node(id) { - (!span.from_expansion() - && impl_item.generics.params.is_empty() - && !fulfill_or_allowed(cx, MULTIPLE_INHERENT_IMPL, [id])) - .then_some(span) + (!span.from_expansion() && !fulfill_or_allowed(cx, MULTIPLE_INHERENT_IMPL, [id])).then_some(span) } else { None } diff --git a/clippy_lints/src/large_include_file.rs b/clippy_lints/src/large_include_file.rs index 5c37747b8c9b..d77e0beeaf4c 100644 --- a/clippy_lints/src/large_include_file.rs +++ b/clippy_lints/src/large_include_file.rs @@ -3,7 +3,7 @@ use clippy_config::Conf; use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::macros::root_macro_call_first_node; use clippy_utils::source::snippet_opt; -use rustc_ast::{AttrArgs, AttrKind, Attribute, LitKind}; +use rustc_ast::{AttrArgs, AttrItemKind, AttrKind, Attribute, LitKind}; use rustc_hir::{Expr, ExprKind}; use rustc_lint::{EarlyContext, EarlyLintPass, LateContext, LateLintPass}; use rustc_session::impl_lint_pass; diff --git a/clippy_lints/src/large_stack_arrays.rs b/clippy_lints/src/large_stack_arrays.rs index 261b03abba17..32c23a2d0362 100644 --- a/clippy_lints/src/large_stack_arrays.rs +++ b/clippy_lints/src/large_stack_arrays.rs @@ -94,6 +94,15 @@ impl<'tcx> LateLintPass<'tcx> for LargeStackArrays { }) && u128::from(self.maximum_allowed_size) < u128::from(element_count) * u128::from(element_size) { + // libtest might generate a large array containing the test cases, and no span will be associated + // to it. In this case it is better not to complain. + // + // Note that this condition is not checked explicitly by a unit test. Do not remove it without + // ensuring that stays fixed. + if expr.span.is_dummy() { + return; + } + span_lint_and_then( cx, LARGE_STACK_ARRAYS, diff --git a/clippy_lints/src/loops/for_kv_map.rs b/clippy_lints/src/loops/for_kv_map.rs index 39b2391c98ec..7fb8e51377a2 100644 --- a/clippy_lints/src/loops/for_kv_map.rs +++ b/clippy_lints/src/loops/for_kv_map.rs @@ -1,16 +1,22 @@ use super::FOR_KV_MAP; use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::MaybeDef; -use clippy_utils::source::snippet; +use clippy_utils::source::{snippet_with_applicability, walk_span_to_context}; use clippy_utils::{pat_is_wild, sugg}; use rustc_errors::Applicability; use rustc_hir::{BorrowKind, Expr, ExprKind, Mutability, Pat, PatKind}; use rustc_lint::LateContext; use rustc_middle::ty; -use rustc_span::sym; +use rustc_span::{Span, sym}; /// Checks for the `FOR_KV_MAP` lint. -pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, pat: &'tcx Pat<'_>, arg: &'tcx Expr<'_>, body: &'tcx Expr<'_>) { +pub(super) fn check<'tcx>( + cx: &LateContext<'tcx>, + pat: &'tcx Pat<'_>, + arg: &'tcx Expr<'_>, + body: &'tcx Expr<'_>, + span: Span, +) { let pat_span = pat.span; if let PatKind::Tuple(pat, _) = pat.kind @@ -34,21 +40,25 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, pat: &'tcx Pat<'_>, arg: &'tcx _ => arg, }; - if matches!(ty.opt_diag_name(cx), Some(sym::HashMap | sym::BTreeMap)) { + if matches!(ty.opt_diag_name(cx), Some(sym::HashMap | sym::BTreeMap)) + && let Some(arg_span) = walk_span_to_context(arg_span, span.ctxt()) + { span_lint_and_then( cx, FOR_KV_MAP, arg_span, format!("you seem to want to iterate on a map's {kind}s"), |diag| { - let map = sugg::Sugg::hir(cx, arg, "map"); + let mut applicability = Applicability::MachineApplicable; + let map = sugg::Sugg::hir_with_context(cx, arg, span.ctxt(), "map", &mut applicability); + let pat = snippet_with_applicability(cx, new_pat_span, kind, &mut applicability); diag.multipart_suggestion( "use the corresponding method", vec![ - (pat_span, snippet(cx, new_pat_span, kind).into_owned()), + (pat_span, pat.to_string()), (arg_span, format!("{}.{kind}s{mutbl}()", map.maybe_paren())), ], - Applicability::MachineApplicable, + applicability, ); }, ); diff --git a/clippy_lints/src/loops/mod.rs b/clippy_lints/src/loops/mod.rs index ddc783069385..83574cab6b67 100644 --- a/clippy_lints/src/loops/mod.rs +++ b/clippy_lints/src/loops/mod.rs @@ -943,7 +943,7 @@ impl Loops { explicit_counter_loop::check(cx, pat, arg, body, expr, label); } self.check_for_loop_arg(cx, pat, arg); - for_kv_map::check(cx, pat, arg, body); + for_kv_map::check(cx, pat, arg, body, span); mut_range_bound::check(cx, arg, body); single_element_loop::check(cx, pat, arg, body, expr); same_item_push::check(cx, pat, arg, body, expr, self.msrv); diff --git a/clippy_lints/src/loops/never_loop.rs b/clippy_lints/src/loops/never_loop.rs index a037af3433c3..e7b9b1cd3881 100644 --- a/clippy_lints/src/loops/never_loop.rs +++ b/clippy_lints/src/loops/never_loop.rs @@ -4,8 +4,8 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::higher::ForLoop; use clippy_utils::macros::root_macro_call_first_node; use clippy_utils::source::{snippet, snippet_with_context}; -use clippy_utils::sym; use clippy_utils::visitors::{Descend, for_each_expr_without_closures}; +use clippy_utils::{contains_return, sym}; use rustc_errors::Applicability; use rustc_hir::{ Block, Closure, Destination, Expr, ExprKind, HirId, InlineAsm, InlineAsmOperand, Node, Pat, Stmt, StmtKind, @@ -82,7 +82,7 @@ pub(super) fn check_iterator_reduction<'tcx>( ) { let closure_body = cx.tcx.hir_body(closure.body).value; let body_ty = cx.typeck_results().expr_ty(closure_body); - if body_ty.is_never() { + if body_ty.is_never() && !contains_return(closure_body) { span_lint_and_then( cx, NEVER_LOOP, diff --git a/clippy_lints/src/manual_ignore_case_cmp.rs b/clippy_lints/src/manual_ignore_case_cmp.rs index 25057b4aeaa2..1c20a8f81efb 100644 --- a/clippy_lints/src/manual_ignore_case_cmp.rs +++ b/clippy_lints/src/manual_ignore_case_cmp.rs @@ -1,7 +1,7 @@ use crate::manual_ignore_case_cmp::MatchType::{Literal, ToAscii}; use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::MaybeDef; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::snippet_with_context; use clippy_utils::sym; use rustc_ast::LitKind; use rustc_errors::Applicability; @@ -111,14 +111,12 @@ impl LateLintPass<'_> for ManualIgnoreCaseCmp { "manual case-insensitive ASCII comparison", |diag| { let mut app = Applicability::MachineApplicable; + let (left_snip, _) = snippet_with_context(cx, left_span, expr.span.ctxt(), "..", &mut app); + let (right_snip, _) = snippet_with_context(cx, right_span, expr.span.ctxt(), "..", &mut app); diag.span_suggestion_verbose( expr.span, "consider using `.eq_ignore_ascii_case()` instead", - format!( - "{neg}{}.eq_ignore_ascii_case({deref}{})", - snippet_with_applicability(cx, left_span, "_", &mut app), - snippet_with_applicability(cx, right_span, "_", &mut app) - ), + format!("{neg}{left_snip}.eq_ignore_ascii_case({deref}{right_snip})"), app, ); }, diff --git a/clippy_lints/src/manual_ilog2.rs b/clippy_lints/src/manual_ilog2.rs index 1c61db530606..4b411a60f3bf 100644 --- a/clippy_lints/src/manual_ilog2.rs +++ b/clippy_lints/src/manual_ilog2.rs @@ -1,7 +1,7 @@ use clippy_config::Conf; use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::msrvs::{self, Msrv}; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::snippet_with_context; use clippy_utils::{is_from_proc_macro, sym}; use rustc_ast::LitKind; use rustc_data_structures::packed::Pu128; @@ -102,7 +102,7 @@ impl LateLintPass<'_> for ManualIlog2 { fn emit(cx: &LateContext<'_>, recv: &Expr<'_>, full_expr: &Expr<'_>) { let mut app = Applicability::MachineApplicable; - let recv = snippet_with_applicability(cx, recv.span, "_", &mut app); + let (recv, _) = snippet_with_context(cx, recv.span, full_expr.span.ctxt(), "_", &mut app); span_lint_and_sugg( cx, MANUAL_ILOG2, diff --git a/clippy_lints/src/matches/manual_ok_err.rs b/clippy_lints/src/matches/manual_ok_err.rs index c35c3d1f62e6..1fc8bb9acce2 100644 --- a/clippy_lints/src/matches/manual_ok_err.rs +++ b/clippy_lints/src/matches/manual_ok_err.rs @@ -135,7 +135,7 @@ fn apply_lint(cx: &LateContext<'_>, expr: &Expr<'_>, scrutinee: &Expr<'_>, is_ok } else { Applicability::MachineApplicable }; - let scrut = Sugg::hir_with_applicability(cx, scrutinee, "..", &mut app).maybe_paren(); + let scrut = Sugg::hir_with_context(cx, scrutinee, expr.span.ctxt(), "..", &mut app).maybe_paren(); let scrutinee_ty = cx.typeck_results().expr_ty(scrutinee); let (_, _, mutability) = peel_and_count_ty_refs(scrutinee_ty); diff --git a/clippy_lints/src/matches/match_as_ref.rs b/clippy_lints/src/matches/match_as_ref.rs index 795355f25f9e..12fe44ef2f13 100644 --- a/clippy_lints/src/matches/match_as_ref.rs +++ b/clippy_lints/src/matches/match_as_ref.rs @@ -46,6 +46,7 @@ pub(crate) fn check(cx: &LateContext<'_>, ex: &Expr<'_>, arms: &[Arm<'_>], expr: let cast = if input_ty == output_ty { "" } else { ".map(|x| x as _)" }; let mut applicability = Applicability::MachineApplicable; + let ctxt = expr.span.ctxt(); span_lint_and_then( cx, MATCH_AS_REF, @@ -59,7 +60,7 @@ pub(crate) fn check(cx: &LateContext<'_>, ex: &Expr<'_>, arms: &[Arm<'_>], expr: "use `Option::as_ref()`", format!( "{}.as_ref(){cast}", - Sugg::hir_with_applicability(cx, ex, "_", &mut applicability).maybe_paren(), + Sugg::hir_with_context(cx, ex, ctxt, "_", &mut applicability).maybe_paren(), ), applicability, ); @@ -69,7 +70,7 @@ pub(crate) fn check(cx: &LateContext<'_>, ex: &Expr<'_>, arms: &[Arm<'_>], expr: format!("use `Option::{method}()` directly"), format!( "{}.{method}(){cast}", - Sugg::hir_with_applicability(cx, ex, "_", &mut applicability).maybe_paren(), + Sugg::hir_with_context(cx, ex, ctxt, "_", &mut applicability).maybe_paren(), ), applicability, ); diff --git a/clippy_lints/src/matches/match_bool.rs b/clippy_lints/src/matches/match_bool.rs index a2c8741f4f74..3e76231b6ef1 100644 --- a/clippy_lints/src/matches/match_bool.rs +++ b/clippy_lints/src/matches/match_bool.rs @@ -16,6 +16,7 @@ pub(crate) fn check(cx: &LateContext<'_>, scrutinee: &Expr<'_>, arms: &[Arm<'_>] && arms .iter() .all(|arm| arm.pat.walk_short(|p| !matches!(p.kind, PatKind::Binding(..)))) + && arms.len() == 2 { span_lint_and_then( cx, @@ -23,59 +24,58 @@ pub(crate) fn check(cx: &LateContext<'_>, scrutinee: &Expr<'_>, arms: &[Arm<'_>] expr.span, "`match` on a boolean expression", move |diag| { - if arms.len() == 2 { - let mut app = Applicability::MachineApplicable; - let test_sugg = if let PatKind::Expr(arm_bool) = arms[0].pat.kind { - let test = Sugg::hir_with_applicability(cx, scrutinee, "_", &mut app); - if let PatExprKind::Lit { lit, .. } = arm_bool.kind { - match &lit.node { - LitKind::Bool(true) => Some(test), - LitKind::Bool(false) => Some(!test), - _ => None, - } - .map(|test| { - if let Some(guard) = &arms[0] - .guard - .map(|g| Sugg::hir_with_applicability(cx, g, "_", &mut app)) - { - test.and(guard) - } else { - test - } - }) - } else { - None + let mut app = Applicability::MachineApplicable; + let ctxt = expr.span.ctxt(); + let test_sugg = if let PatKind::Expr(arm_bool) = arms[0].pat.kind { + let test = Sugg::hir_with_context(cx, scrutinee, ctxt, "_", &mut app); + if let PatExprKind::Lit { lit, .. } = arm_bool.kind { + match &lit.node { + LitKind::Bool(true) => Some(test), + LitKind::Bool(false) => Some(!test), + _ => None, } + .map(|test| { + if let Some(guard) = &arms[0] + .guard + .map(|g| Sugg::hir_with_context(cx, g, ctxt, "_", &mut app)) + { + test.and(guard) + } else { + test + } + }) } else { None + } + } else { + None + }; + + if let Some(test_sugg) = test_sugg { + let ctxt = expr.span.ctxt(); + let (true_expr, false_expr) = (arms[0].body, arms[1].body); + let sugg = match (is_unit_expr(true_expr), is_unit_expr(false_expr)) { + (false, false) => Some(format!( + "if {} {} else {}", + test_sugg, + expr_block(cx, true_expr, ctxt, "..", Some(expr.span), &mut app), + expr_block(cx, false_expr, ctxt, "..", Some(expr.span), &mut app) + )), + (false, true) => Some(format!( + "if {} {}", + test_sugg, + expr_block(cx, true_expr, ctxt, "..", Some(expr.span), &mut app) + )), + (true, false) => Some(format!( + "if {} {}", + !test_sugg, + expr_block(cx, false_expr, ctxt, "..", Some(expr.span), &mut app) + )), + (true, true) => None, }; - if let Some(test_sugg) = test_sugg { - let ctxt = expr.span.ctxt(); - let (true_expr, false_expr) = (arms[0].body, arms[1].body); - let sugg = match (is_unit_expr(true_expr), is_unit_expr(false_expr)) { - (false, false) => Some(format!( - "if {} {} else {}", - test_sugg, - expr_block(cx, true_expr, ctxt, "..", Some(expr.span), &mut app), - expr_block(cx, false_expr, ctxt, "..", Some(expr.span), &mut app) - )), - (false, true) => Some(format!( - "if {} {}", - test_sugg, - expr_block(cx, true_expr, ctxt, "..", Some(expr.span), &mut app) - )), - (true, false) => Some(format!( - "if {} {}", - !test_sugg, - expr_block(cx, false_expr, ctxt, "..", Some(expr.span), &mut app) - )), - (true, true) => None, - }; - - if let Some(sugg) = sugg { - diag.span_suggestion(expr.span, "consider using an `if`/`else` expression", sugg, app); - } + if let Some(sugg) = sugg { + diag.span_suggestion(expr.span, "consider using an `if`/`else` expression", sugg, app); } } }, diff --git a/clippy_lints/src/matches/redundant_pattern_match.rs b/clippy_lints/src/matches/redundant_pattern_match.rs index bc3783750e5c..bf0c0c4aec3c 100644 --- a/clippy_lints/src/matches/redundant_pattern_match.rs +++ b/clippy_lints/src/matches/redundant_pattern_match.rs @@ -1,7 +1,6 @@ use super::REDUNDANT_PATTERN_MATCHING; use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then}; use clippy_utils::res::{MaybeDef, MaybeTypeckRes}; -use clippy_utils::source::walk_span_to_context; use clippy_utils::sugg::{Sugg, make_unop}; use clippy_utils::ty::needs_ordered_drop; use clippy_utils::visitors::{any_temporaries_need_ordered_drop, for_each_expr_without_closures}; @@ -25,7 +24,7 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) { .. }) = higher::WhileLet::hir(expr) { - find_method_sugg_for_if_let(cx, expr, let_pat, let_expr, "while", false); + find_method_sugg_for_if_let(cx, expr, let_pat, let_expr, "while", false, let_span); find_if_let_true(cx, let_pat, let_expr, let_span); } } @@ -39,7 +38,7 @@ pub(super) fn check_if_let<'tcx>( let_span: Span, ) { find_if_let_true(cx, pat, scrutinee, let_span); - find_method_sugg_for_if_let(cx, expr, pat, scrutinee, "if", has_else); + find_method_sugg_for_if_let(cx, expr, pat, scrutinee, "if", has_else, let_span); } /// Looks for: @@ -182,6 +181,7 @@ fn find_method_sugg_for_if_let<'tcx>( let_expr: &'tcx Expr<'_>, keyword: &'static str, has_else: bool, + let_span: Span, ) { // also look inside refs // if we have &None for example, peel it so we can detect "if let None = x" @@ -239,15 +239,9 @@ fn find_method_sugg_for_if_let<'tcx>( let expr_span = expr.span; let ctxt = expr.span.ctxt(); - // if/while let ... = ... { ... } - // ^^^ - let Some(res_span) = walk_span_to_context(result_expr.span.source_callsite(), ctxt) else { - return; - }; - // if/while let ... = ... { ... } // ^^^^^^^^^^^^^^^^^^^^^^ - let span = expr_span.until(res_span.shrink_to_hi()); + let span = expr_span.until(let_span.shrink_to_hi()); let mut app = if needs_drop { Applicability::MaybeIncorrect @@ -273,13 +267,14 @@ pub(super) fn check_match<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, op if let Ok(arms) = arms.try_into() // TODO: use `slice::as_array` once stabilized && let Some((good_method, maybe_guard)) = found_good_method(cx, arms) { - let span = is_expn_of(expr.span, sym::matches).unwrap_or(expr.span.to(op.span)); + let expr_span = is_expn_of(expr.span, sym::matches).unwrap_or(expr.span); + let result_expr = match &op.kind { ExprKind::AddrOf(_, _, borrowed) => borrowed, _ => op, }; let mut app = Applicability::MachineApplicable; - let receiver_sugg = Sugg::hir_with_applicability(cx, result_expr, "_", &mut app).maybe_paren(); + let receiver_sugg = Sugg::hir_with_context(cx, result_expr, expr_span.ctxt(), "_", &mut app).maybe_paren(); let mut sugg = format!("{receiver_sugg}.{good_method}"); if let Some(guard) = maybe_guard { @@ -302,14 +297,14 @@ pub(super) fn check_match<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, op return; } - let guard = Sugg::hir(cx, guard, ".."); + let guard = Sugg::hir_with_context(cx, guard, expr_span.ctxt(), "..", &mut app); let _ = write!(sugg, " && {}", guard.maybe_paren()); } span_lint_and_sugg( cx, REDUNDANT_PATTERN_MATCHING, - span, + expr_span, format!("redundant pattern matching, consider using `{good_method}`"), "try", sugg, diff --git a/clippy_lints/src/methods/is_empty.rs b/clippy_lints/src/methods/is_empty.rs index 834456ff6668..8135c67d0d78 100644 --- a/clippy_lints/src/methods/is_empty.rs +++ b/clippy_lints/src/methods/is_empty.rs @@ -3,10 +3,9 @@ use clippy_utils::diagnostics::span_lint; use clippy_utils::macros::{is_assert_macro, root_macro_call}; use clippy_utils::res::MaybeResPath; use clippy_utils::{find_binding_init, get_parent_expr, is_inside_always_const_context}; -use rustc_hir::{Expr, HirId}; -use rustc_lint::{LateContext, LintContext}; use rustc_hir::attrs::AttributeKind; -use rustc_hir::find_attr; +use rustc_hir::{Expr, HirId, find_attr}; +use rustc_lint::{LateContext, LintContext}; use super::CONST_IS_EMPTY; diff --git a/clippy_lints/src/methods/iter_kv_map.rs b/clippy_lints/src/methods/iter_kv_map.rs index 16db8663941e..366bfaed73d4 100644 --- a/clippy_lints/src/methods/iter_kv_map.rs +++ b/clippy_lints/src/methods/iter_kv_map.rs @@ -2,7 +2,7 @@ use super::ITER_KV_MAP; use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::msrvs::{self, Msrv}; use clippy_utils::res::MaybeDef; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::{snippet_with_applicability, snippet_with_context}; use clippy_utils::{pat_is_wild, sym}; use rustc_hir::{Body, Expr, ExprKind, PatKind}; use rustc_lint::LateContext; @@ -58,6 +58,8 @@ pub(super) fn check<'tcx>( applicability, ); } else { + let (body_snippet, _) = + snippet_with_context(cx, body_expr.span, expr.span.ctxt(), "..", &mut applicability); span_lint_and_sugg( cx, ITER_KV_MAP, @@ -65,9 +67,8 @@ pub(super) fn check<'tcx>( format!("iterating on a map's {replacement_kind}s"), "try", format!( - "{recv_snippet}.{into_prefix}{replacement_kind}s().map(|{}{bound_ident}| {})", + "{recv_snippet}.{into_prefix}{replacement_kind}s().map(|{}{bound_ident}| {body_snippet})", annotation.prefix_str(), - snippet_with_applicability(cx, body_expr.span, "/* body */", &mut applicability) ), applicability, ); diff --git a/clippy_lints/src/methods/unnecessary_fold.rs b/clippy_lints/src/methods/unnecessary_fold.rs index 9dae6fbb48dd..c3f031edff2e 100644 --- a/clippy_lints/src/methods/unnecessary_fold.rs +++ b/clippy_lints/src/methods/unnecessary_fold.rs @@ -1,10 +1,10 @@ -use clippy_utils::diagnostics::span_lint_and_sugg; +use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::{MaybeDef, MaybeQPath, MaybeResPath, MaybeTypeckRes}; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::snippet_with_context; use clippy_utils::{DefinedTy, ExprUseNode, expr_use_ctxt, peel_blocks, strip_pat_refs}; use rustc_ast::ast; use rustc_data_structures::packed::Pu128; -use rustc_errors::Applicability; +use rustc_errors::{Applicability, Diag}; use rustc_hir as hir; use rustc_hir::PatKind; use rustc_hir::def::{DefKind, Res}; @@ -59,6 +59,34 @@ struct Replacement { method_name: &'static str, has_args: bool, has_generic_return: bool, + is_short_circuiting: bool, +} + +impl Replacement { + fn default_applicability(&self) -> Applicability { + if self.is_short_circuiting { + Applicability::MaybeIncorrect + } else { + Applicability::MachineApplicable + } + } + + fn maybe_add_note(&self, diag: &mut Diag<'_, ()>) { + if self.is_short_circuiting { + diag.note(format!( + "the `{}` method is short circuiting and may change the program semantics if the iterator has side effects", + self.method_name + )); + } + } + + fn maybe_turbofish(&self, ty: Ty<'_>) -> String { + if self.has_generic_return { + format!("::<{ty}>") + } else { + String::new() + } + } } fn check_fold_with_op( @@ -86,32 +114,30 @@ fn check_fold_with_op( && left_expr.res_local_id() == Some(first_arg_id) && (replacement.has_args || right_expr.res_local_id() == Some(second_arg_id)) { - let mut applicability = Applicability::MachineApplicable; - - let turbofish = if replacement.has_generic_return { - format!("::<{}>", cx.typeck_results().expr_ty_adjusted(right_expr).peel_refs()) - } else { - String::new() - }; - - let sugg = if replacement.has_args { - format!( - "{method}{turbofish}(|{second_arg_ident}| {r})", - method = replacement.method_name, - r = snippet_with_applicability(cx, right_expr.span, "EXPR", &mut applicability), - ) - } else { - format!("{method}{turbofish}()", method = replacement.method_name) - }; - - span_lint_and_sugg( + let span = fold_span.with_hi(expr.span.hi()); + span_lint_and_then( cx, UNNECESSARY_FOLD, - fold_span.with_hi(expr.span.hi()), + span, "this `.fold` can be written more succinctly using another method", - "try", - sugg, - applicability, + |diag| { + let mut applicability = replacement.default_applicability(); + let turbofish = + replacement.maybe_turbofish(cx.typeck_results().expr_ty_adjusted(right_expr).peel_refs()); + let (r_snippet, _) = + snippet_with_context(cx, right_expr.span, expr.span.ctxt(), "EXPR", &mut applicability); + let sugg = if replacement.has_args { + format!( + "{method}{turbofish}(|{second_arg_ident}| {r_snippet})", + method = replacement.method_name, + ) + } else { + format!("{method}{turbofish}()", method = replacement.method_name) + }; + + diag.span_suggestion(span, "try", sugg, applicability); + replacement.maybe_add_note(diag); + }, ); return true; } @@ -131,22 +157,25 @@ fn check_fold_with_method( // Check if the function belongs to the operator && cx.tcx.is_diagnostic_item(method, fn_did) { - let applicability = Applicability::MachineApplicable; - - let turbofish = if replacement.has_generic_return { - format!("::<{}>", cx.typeck_results().expr_ty(expr)) - } else { - String::new() - }; - - span_lint_and_sugg( + let span = fold_span.with_hi(expr.span.hi()); + span_lint_and_then( cx, UNNECESSARY_FOLD, - fold_span.with_hi(expr.span.hi()), + span, "this `.fold` can be written more succinctly using another method", - "try", - format!("{method}{turbofish}()", method = replacement.method_name), - applicability, + |diag| { + diag.span_suggestion( + span, + "try", + format!( + "{method}{turbofish}()", + method = replacement.method_name, + turbofish = replacement.maybe_turbofish(cx.typeck_results().expr_ty(expr)) + ), + replacement.default_applicability(), + ); + replacement.maybe_add_note(diag); + }, ); } } @@ -171,6 +200,7 @@ pub(super) fn check<'tcx>( method_name: "any", has_args: true, has_generic_return: false, + is_short_circuiting: true, }; check_fold_with_op(cx, expr, acc, fold_span, hir::BinOpKind::Or, replacement); }, @@ -179,6 +209,7 @@ pub(super) fn check<'tcx>( method_name: "all", has_args: true, has_generic_return: false, + is_short_circuiting: true, }; check_fold_with_op(cx, expr, acc, fold_span, hir::BinOpKind::And, replacement); }, @@ -187,6 +218,7 @@ pub(super) fn check<'tcx>( method_name: "sum", has_args: false, has_generic_return: needs_turbofish(cx, expr), + is_short_circuiting: false, }; if !check_fold_with_op(cx, expr, acc, fold_span, hir::BinOpKind::Add, replacement) { check_fold_with_method(cx, expr, acc, fold_span, sym::add, replacement); @@ -197,6 +229,7 @@ pub(super) fn check<'tcx>( method_name: "product", has_args: false, has_generic_return: needs_turbofish(cx, expr), + is_short_circuiting: false, }; if !check_fold_with_op(cx, expr, acc, fold_span, hir::BinOpKind::Mul, replacement) { check_fold_with_method(cx, expr, acc, fold_span, sym::mul, replacement); diff --git a/clippy_lints/src/methods/unnecessary_to_owned.rs b/clippy_lints/src/methods/unnecessary_to_owned.rs index a6a39cb6ab30..74e8dbc15a6c 100644 --- a/clippy_lints/src/methods/unnecessary_to_owned.rs +++ b/clippy_lints/src/methods/unnecessary_to_owned.rs @@ -3,7 +3,7 @@ use super::unnecessary_iter_cloned::{self, is_into_iter}; use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then}; use clippy_utils::msrvs::{self, Msrv}; use clippy_utils::res::MaybeDef; -use clippy_utils::source::{SpanRangeExt, snippet}; +use clippy_utils::source::{SpanRangeExt, snippet, snippet_with_context}; use clippy_utils::ty::{get_iterator_item_ty, implements_trait, is_copy, peel_and_count_ty_refs}; use clippy_utils::visitors::find_all_ret_expressions; use clippy_utils::{fn_def_id, get_parent_expr, is_expr_temporary_value, return_ty, sym}; @@ -131,8 +131,10 @@ fn check_addr_of_expr( && (*referent_ty != receiver_ty || (matches!(referent_ty.kind(), ty::Array(..)) && is_copy(cx, *referent_ty)) || is_cow_into_owned(cx, method_name, method_parent_id)) - && let Some(receiver_snippet) = receiver.span.get_source_text(cx) { + let mut applicability = Applicability::MachineApplicable; + let (receiver_snippet, _) = snippet_with_context(cx, receiver.span, expr.span.ctxt(), "..", &mut applicability); + if receiver_ty == target_ty && n_target_refs >= n_receiver_refs { span_lint_and_sugg( cx, @@ -145,7 +147,7 @@ fn check_addr_of_expr( "", width = n_target_refs - n_receiver_refs ), - Applicability::MachineApplicable, + applicability, ); return true; } @@ -165,8 +167,8 @@ fn check_addr_of_expr( parent.span, format!("unnecessary use of `{method_name}`"), "use", - receiver_snippet.to_owned(), - Applicability::MachineApplicable, + receiver_snippet.to_string(), + applicability, ); } else { span_lint_and_sugg( @@ -176,7 +178,7 @@ fn check_addr_of_expr( format!("unnecessary use of `{method_name}`"), "remove this", String::new(), - Applicability::MachineApplicable, + applicability, ); } return true; @@ -191,7 +193,7 @@ fn check_addr_of_expr( format!("unnecessary use of `{method_name}`"), "use", format!("{receiver_snippet}.as_ref()"), - Applicability::MachineApplicable, + applicability, ); return true; } @@ -409,8 +411,10 @@ fn check_other_call_arg<'tcx>( None } && can_change_type(cx, maybe_arg, receiver_ty) - && let Some(receiver_snippet) = receiver.span.get_source_text(cx) { + let mut applicability = Applicability::MachineApplicable; + let (receiver_snippet, _) = snippet_with_context(cx, receiver.span, expr.span.ctxt(), "..", &mut applicability); + span_lint_and_sugg( cx, UNNECESSARY_TO_OWNED, @@ -418,7 +422,7 @@ fn check_other_call_arg<'tcx>( format!("unnecessary use of `{method_name}`"), "use", format!("{:&>n_refs$}{receiver_snippet}", ""), - Applicability::MachineApplicable, + applicability, ); return true; } diff --git a/clippy_lints/src/missing_enforced_import_rename.rs b/clippy_lints/src/missing_enforced_import_rename.rs index eeea6dfd5f4b..5dd38cf059c2 100644 --- a/clippy_lints/src/missing_enforced_import_rename.rs +++ b/clippy_lints/src/missing_enforced_import_rename.rs @@ -82,7 +82,8 @@ impl LateLintPass<'_> for ImportRename { && let Some(import) = match snip.split_once(" as ") { None => Some(snip.as_str()), Some((import, rename)) => { - if rename.trim() == name.as_str() { + let trimmed_rename = rename.trim(); + if trimmed_rename == "_" || trimmed_rename == name.as_str() { None } else { Some(import.trim()) diff --git a/clippy_lints/src/multiple_bound_locations.rs b/clippy_lints/src/multiple_bound_locations.rs index 741f38f97560..5b6b4f112455 100644 --- a/clippy_lints/src/multiple_bound_locations.rs +++ b/clippy_lints/src/multiple_bound_locations.rs @@ -31,7 +31,7 @@ declare_clippy_lint! { /// ``` #[clippy::version = "1.78.0"] pub MULTIPLE_BOUND_LOCATIONS, - suspicious, + style, "defining generic bounds in multiple locations" } diff --git a/clippy_lints/src/mutex_atomic.rs b/clippy_lints/src/mutex_atomic.rs index 2fef8404f824..ad44d65b4d66 100644 --- a/clippy_lints/src/mutex_atomic.rs +++ b/clippy_lints/src/mutex_atomic.rs @@ -143,7 +143,7 @@ fn check_expr<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'tcx>, ty_ascription: &T && new.ident.name == sym::new { let mut applicability = Applicability::MaybeIncorrect; - let arg = Sugg::hir_with_applicability(cx, arg, "_", &mut applicability); + let arg = Sugg::hir_with_context(cx, arg, expr.span.ctxt(), "_", &mut applicability); let mut suggs = vec![(expr.span, format!("std::sync::atomic::{atomic_name}::new({arg})"))]; match ty_ascription { TypeAscriptionKind::Required(ty_ascription) => { diff --git a/clippy_lints/src/needless_bool.rs b/clippy_lints/src/needless_bool.rs index 854e927aa2f7..6b5db9dcf3e2 100644 --- a/clippy_lints/src/needless_bool.rs +++ b/clippy_lints/src/needless_bool.rs @@ -1,5 +1,5 @@ use clippy_utils::diagnostics::{span_lint, span_lint_and_sugg}; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::snippet_with_context; use clippy_utils::sugg::Sugg; use clippy_utils::{ SpanlessEq, get_parent_expr, higher, is_block_like, is_else_clause, is_parent_stmt, is_receiver_of_method_call, @@ -171,8 +171,8 @@ impl<'tcx> LateLintPass<'tcx> for NeedlessBool { && SpanlessEq::new(cx).eq_expr(lhs_a, lhs_b) { let mut applicability = Applicability::MachineApplicable; - let cond = Sugg::hir_with_applicability(cx, cond, "..", &mut applicability); - let lhs = snippet_with_applicability(cx, lhs_a.span, "..", &mut applicability); + let cond = Sugg::hir_with_context(cx, cond, e.span.ctxt(), "..", &mut applicability); + let (lhs, _) = snippet_with_context(cx, lhs_a.span, e.span.ctxt(), "..", &mut applicability); let mut sugg = if a == b { format!("{cond}; {lhs} = {a:?};") } else { diff --git a/clippy_lints/src/needless_for_each.rs b/clippy_lints/src/needless_for_each.rs index d03188f1d39b..55a5a16c0099 100644 --- a/clippy_lints/src/needless_for_each.rs +++ b/clippy_lints/src/needless_for_each.rs @@ -56,8 +56,20 @@ declare_lint_pass!(NeedlessForEach => [NEEDLESS_FOR_EACH]); impl<'tcx> LateLintPass<'tcx> for NeedlessForEach { fn check_stmt(&mut self, cx: &LateContext<'tcx>, stmt: &'tcx Stmt<'_>) { - if let StmtKind::Expr(expr) | StmtKind::Semi(expr) = stmt.kind - && let ExprKind::MethodCall(method_name, for_each_recv, [for_each_arg], _) = expr.kind + if let StmtKind::Expr(expr) | StmtKind::Semi(expr) = stmt.kind { + check_expr(cx, expr, stmt.span); + } + } + + fn check_block(&mut self, cx: &LateContext<'tcx>, block: &'tcx Block<'_>) { + if let Some(expr) = block.expr { + check_expr(cx, expr, expr.span); + } + } +} + +fn check_expr(cx: &LateContext<'_>, expr: &Expr<'_>, outer_span: Span) { + if let ExprKind::MethodCall(method_name, for_each_recv, [for_each_arg], _) = expr.kind && let ExprKind::MethodCall(_, iter_recv, [], _) = for_each_recv.kind // Skip the lint if the call chain is too long. e.g. `v.field.iter().for_each()` or // `v.foo().iter().for_each()` must be skipped. @@ -76,69 +88,74 @@ impl<'tcx> LateLintPass<'tcx> for NeedlessForEach { // Skip the lint if the body is not safe, so as not to suggest `for … in … unsafe {}` // and suggesting `for … in … { unsafe { } }` is a little ugly. && !matches!(body.value.kind, ExprKind::Block(Block { rules: BlockCheckMode::UnsafeBlock(_), .. }, ..)) + { + let mut applicability = Applicability::MachineApplicable; + + // If any closure parameter has an explicit type specified, applying the lint would necessarily + // remove that specification, possibly breaking type inference + if fn_decl + .inputs + .iter() + .any(|input| matches!(input.kind, TyKind::Infer(..))) { - let mut applicability = Applicability::MachineApplicable; + applicability = Applicability::MaybeIncorrect; + } - // If any closure parameter has an explicit type specified, applying the lint would necessarily - // remove that specification, possibly breaking type inference - if fn_decl - .inputs - .iter() - .any(|input| matches!(input.kind, TyKind::Infer(..))) - { - applicability = Applicability::MaybeIncorrect; - } + let mut ret_collector = RetCollector::default(); + ret_collector.visit_expr(body.value); - let mut ret_collector = RetCollector::default(); - ret_collector.visit_expr(body.value); + // Skip the lint if `return` is used in `Loop` in order not to suggest using `'label`. + if ret_collector.ret_in_loop { + return; + } - // Skip the lint if `return` is used in `Loop` in order not to suggest using `'label`. - if ret_collector.ret_in_loop { - return; - } + let ret_suggs = if ret_collector.spans.is_empty() { + None + } else { + applicability = Applicability::MaybeIncorrect; + Some( + ret_collector + .spans + .into_iter() + .map(|span| (span, "continue".to_string())) + .collect(), + ) + }; - let ret_suggs = if ret_collector.spans.is_empty() { - None + let body_param_sugg = snippet_with_applicability(cx, body.params[0].pat.span, "..", &mut applicability); + let for_each_rev_sugg = snippet_with_applicability(cx, for_each_recv.span, "..", &mut applicability); + let (body_value_sugg, is_macro_call) = + snippet_with_context(cx, body.value.span, for_each_recv.span.ctxt(), "..", &mut applicability); + + let sugg = format!( + "for {} in {} {}", + body_param_sugg, + for_each_rev_sugg, + if is_macro_call { + format!("{{ {body_value_sugg}; }}") } else { - applicability = Applicability::MaybeIncorrect; - Some( - ret_collector - .spans - .into_iter() - .map(|span| (span, "continue".to_string())) - .collect(), - ) - }; - - let body_param_sugg = snippet_with_applicability(cx, body.params[0].pat.span, "..", &mut applicability); - let for_each_rev_sugg = snippet_with_applicability(cx, for_each_recv.span, "..", &mut applicability); - let (body_value_sugg, is_macro_call) = - snippet_with_context(cx, body.value.span, for_each_recv.span.ctxt(), "..", &mut applicability); - - let sugg = format!( - "for {} in {} {}", - body_param_sugg, - for_each_rev_sugg, - if is_macro_call { - format!("{{ {body_value_sugg}; }}") - } else { - match body.value.kind { - ExprKind::Block(block, _) if is_let_desugar(block) => { - format!("{{ {body_value_sugg} }}") - }, - ExprKind::Block(_, _) => body_value_sugg.to_string(), - _ => format!("{{ {body_value_sugg}; }}"), - } + match body.value.kind { + ExprKind::Block(block, _) if is_let_desugar(block) => { + format!("{{ {body_value_sugg} }}") + }, + ExprKind::Block(_, _) => body_value_sugg.to_string(), + _ => format!("{{ {body_value_sugg}; }}"), } - ); + } + ); - span_lint_and_then(cx, NEEDLESS_FOR_EACH, stmt.span, "needless use of `for_each`", |diag| { - diag.span_suggestion(stmt.span, "try", sugg, applicability); + span_lint_and_then( + cx, + NEEDLESS_FOR_EACH, + outer_span, + "needless use of `for_each`", + |diag| { + diag.span_suggestion(outer_span, "try", sugg, applicability); if let Some(ret_suggs) = ret_suggs { diag.multipart_suggestion("...and replace `return` with `continue`", ret_suggs, applicability); } - }); - } + }, + ); } } diff --git a/clippy_lints/src/new_without_default.rs b/clippy_lints/src/new_without_default.rs index 67493d54b552..e151c06c6c20 100644 --- a/clippy_lints/src/new_without_default.rs +++ b/clippy_lints/src/new_without_default.rs @@ -1,16 +1,15 @@ use clippy_utils::diagnostics::span_lint_hir_and_then; use clippy_utils::return_ty; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::{indent_of, reindent_multiline, snippet_with_applicability}; use clippy_utils::sugg::DiagExt; use rustc_errors::Applicability; use rustc_hir as hir; -use rustc_hir::HirIdSet; +use rustc_hir::attrs::AttributeKind; +use rustc_hir::{Attribute, HirIdSet}; use rustc_lint::{LateContext, LateLintPass, LintContext}; use rustc_middle::ty::AssocKind; use rustc_session::impl_lint_pass; use rustc_span::sym; -use rustc_hir::Attribute; -use rustc_hir::attrs::AttributeKind; declare_clippy_lint! { /// ### What it does @@ -59,6 +58,7 @@ pub struct NewWithoutDefault { impl_lint_pass!(NewWithoutDefault => [NEW_WITHOUT_DEFAULT]); impl<'tcx> LateLintPass<'tcx> for NewWithoutDefault { + #[expect(clippy::too_many_lines)] fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'_>) { let hir::ItemKind::Impl(hir::Impl { of_trait: None, @@ -139,16 +139,34 @@ impl<'tcx> LateLintPass<'tcx> for NewWithoutDefault { sugg.push_str(&snippet_with_applicability(cx.sess(), *attr_span, "_", &mut app)); sugg.push('\n'); } - } sugg }; let generics_sugg = snippet_with_applicability(cx, generics.span, "", &mut app); let where_clause_sugg = if generics.has_where_clause_predicates { - format!( - "\n{}\n", - snippet_with_applicability(cx, generics.where_clause_span, "", &mut app) - ) + let where_clause_sugg = + snippet_with_applicability(cx, generics.where_clause_span, "", &mut app).to_string(); + let mut where_clause_sugg = reindent_multiline(&where_clause_sugg, true, Some(4)); + if impl_item.generics.has_where_clause_predicates { + if !where_clause_sugg.ends_with(',') { + where_clause_sugg.push(','); + } + + let additional_where_preds = + snippet_with_applicability(cx, impl_item.generics.where_clause_span, "", &mut app); + let ident = indent_of(cx, generics.where_clause_span).unwrap_or(0); + // Remove the leading `where ` keyword + let additional_where_preds = additional_where_preds.trim_start_matches("where").trim_start(); + where_clause_sugg.push('\n'); + where_clause_sugg.extend(std::iter::repeat_n(' ', ident)); + where_clause_sugg.push_str(additional_where_preds); + } + format!("\n{where_clause_sugg}\n") + } else if impl_item.generics.has_where_clause_predicates { + let where_clause_sugg = + snippet_with_applicability(cx, impl_item.generics.where_clause_span, "", &mut app); + let where_clause_sugg = reindent_multiline(&where_clause_sugg, true, Some(4)); + format!("\n{}\n", where_clause_sugg.trim_start()) } else { String::new() }; diff --git a/clippy_lints/src/operators/cmp_owned.rs b/clippy_lints/src/operators/cmp_owned.rs index 05358de5b348..39097833a6c5 100644 --- a/clippy_lints/src/operators/cmp_owned.rs +++ b/clippy_lints/src/operators/cmp_owned.rs @@ -1,6 +1,6 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::MaybeQPath; -use clippy_utils::source::snippet; +use clippy_utils::source::snippet_with_context; use clippy_utils::ty::{implements_trait, is_copy}; use rustc_errors::Applicability; use rustc_hir::{BinOpKind, Expr, ExprKind, UnOp}; @@ -94,51 +94,37 @@ fn check_op(cx: &LateContext<'_>, expr: &Expr<'_>, other: &Expr<'_>, left: bool) return; } - let arg_snip = snippet(cx, arg_span, ".."); - let expr_snip; - let eq_impl; - if with_deref.is_implemented() && !arg_ty.peel_refs().is_str() { - expr_snip = format!("*{arg_snip}"); - eq_impl = with_deref; + let mut applicability = Applicability::MachineApplicable; + let (arg_snip, _) = snippet_with_context(cx, arg_span, expr.span.ctxt(), "..", &mut applicability); + let (expr_snip, eq_impl) = if with_deref.is_implemented() && !arg_ty.peel_refs().is_str() { + (format!("*{arg_snip}"), with_deref) } else { - expr_snip = arg_snip.to_string(); - eq_impl = without_deref; - } + (arg_snip.to_string(), without_deref) + }; - let span; - let hint; - if (eq_impl.ty_eq_other && left) || (eq_impl.other_eq_ty && !left) { - span = expr.span; - hint = expr_snip; + let (span, hint) = if (eq_impl.ty_eq_other && left) || (eq_impl.other_eq_ty && !left) { + (expr.span, expr_snip) } else { - span = expr.span.to(other.span); + let span = expr.span.to(other.span); let cmp_span = if other.span < expr.span { other.span.between(expr.span) } else { expr.span.between(other.span) }; - if eq_impl.ty_eq_other { - hint = format!( - "{expr_snip}{}{}", - snippet(cx, cmp_span, ".."), - snippet(cx, other.span, "..") - ); - } else { - hint = format!( - "{}{}{expr_snip}", - snippet(cx, other.span, ".."), - snippet(cx, cmp_span, "..") - ); - } - } - diag.span_suggestion( - span, - "try", - hint, - Applicability::MachineApplicable, // snippet - ); + let (cmp_snippet, _) = snippet_with_context(cx, cmp_span, expr.span.ctxt(), "..", &mut applicability); + let (other_snippet, _) = + snippet_with_context(cx, other.span, expr.span.ctxt(), "..", &mut applicability); + + if eq_impl.ty_eq_other { + (span, format!("{expr_snip}{cmp_snippet}{other_snippet}")) + } else { + (span, format!("{other_snippet}{cmp_snippet}{expr_snip}")) + } + }; + + diag.span_suggestion(span, "try", hint, applicability); }, ); } diff --git a/clippy_lints/src/operators/manual_div_ceil.rs b/clippy_lints/src/operators/manual_div_ceil.rs index 98aa47421537..5ed923d719bc 100644 --- a/clippy_lints/src/operators/manual_div_ceil.rs +++ b/clippy_lints/src/operators/manual_div_ceil.rs @@ -1,5 +1,6 @@ use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::msrvs::{self, Msrv}; +use clippy_utils::res::{MaybeDef, MaybeQPath}; use clippy_utils::sugg::{Sugg, has_enclosing_paren}; use clippy_utils::{SpanlessEq, sym}; use rustc_ast::{BinOpKind, LitIntType, LitKind, UnOp}; @@ -7,7 +8,7 @@ use rustc_data_structures::packed::Pu128; use rustc_errors::Applicability; use rustc_hir::{Expr, ExprKind}; use rustc_lint::LateContext; -use rustc_middle::ty::{self}; +use rustc_middle::ty::{self, Ty}; use rustc_span::source_map::Spanned; use super::MANUAL_DIV_CEIL; @@ -16,59 +17,84 @@ pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, op: BinOpKind, lhs: & let mut applicability = Applicability::MachineApplicable; if op == BinOpKind::Div - && check_int_ty_and_feature(cx, lhs) - && check_int_ty_and_feature(cx, rhs) - && let ExprKind::Binary(inner_op, inner_lhs, inner_rhs) = lhs.kind + && check_int_ty_and_feature(cx, cx.typeck_results().expr_ty(lhs)) + && check_int_ty_and_feature(cx, cx.typeck_results().expr_ty(rhs)) && msrv.meets(cx, msrvs::DIV_CEIL) { - // (x + (y - 1)) / y - if let ExprKind::Binary(sub_op, sub_lhs, sub_rhs) = inner_rhs.kind - && inner_op.node == BinOpKind::Add - && sub_op.node == BinOpKind::Sub - && check_literal(sub_rhs) - && check_eq_expr(cx, sub_lhs, rhs) - { - build_suggestion(cx, expr, inner_lhs, rhs, &mut applicability); - return; - } + match lhs.kind { + ExprKind::Binary(inner_op, inner_lhs, inner_rhs) => { + // (x + (y - 1)) / y + if let ExprKind::Binary(sub_op, sub_lhs, sub_rhs) = inner_rhs.kind + && inner_op.node == BinOpKind::Add + && sub_op.node == BinOpKind::Sub + && check_literal(sub_rhs) + && check_eq_expr(cx, sub_lhs, rhs) + { + build_suggestion(cx, expr, inner_lhs, rhs, &mut applicability); + return; + } - // ((y - 1) + x) / y - if let ExprKind::Binary(sub_op, sub_lhs, sub_rhs) = inner_lhs.kind - && inner_op.node == BinOpKind::Add - && sub_op.node == BinOpKind::Sub - && check_literal(sub_rhs) - && check_eq_expr(cx, sub_lhs, rhs) - { - build_suggestion(cx, expr, inner_rhs, rhs, &mut applicability); - return; - } + // ((y - 1) + x) / y + if let ExprKind::Binary(sub_op, sub_lhs, sub_rhs) = inner_lhs.kind + && inner_op.node == BinOpKind::Add + && sub_op.node == BinOpKind::Sub + && check_literal(sub_rhs) + && check_eq_expr(cx, sub_lhs, rhs) + { + build_suggestion(cx, expr, inner_rhs, rhs, &mut applicability); + return; + } - // (x + y - 1) / y - if let ExprKind::Binary(add_op, add_lhs, add_rhs) = inner_lhs.kind - && inner_op.node == BinOpKind::Sub - && add_op.node == BinOpKind::Add - && check_literal(inner_rhs) - && check_eq_expr(cx, add_rhs, rhs) - { - build_suggestion(cx, expr, add_lhs, rhs, &mut applicability); - } + // (x + y - 1) / y + if let ExprKind::Binary(add_op, add_lhs, add_rhs) = inner_lhs.kind + && inner_op.node == BinOpKind::Sub + && add_op.node == BinOpKind::Add + && check_literal(inner_rhs) + && check_eq_expr(cx, add_rhs, rhs) + { + build_suggestion(cx, expr, add_lhs, rhs, &mut applicability); + } - // (x + (Y - 1)) / Y - if inner_op.node == BinOpKind::Add && differ_by_one(inner_rhs, rhs) { - build_suggestion(cx, expr, inner_lhs, rhs, &mut applicability); - } + // (x + (Y - 1)) / Y + if inner_op.node == BinOpKind::Add && differ_by_one(inner_rhs, rhs) { + build_suggestion(cx, expr, inner_lhs, rhs, &mut applicability); + } - // ((Y - 1) + x) / Y - if inner_op.node == BinOpKind::Add && differ_by_one(inner_lhs, rhs) { - build_suggestion(cx, expr, inner_rhs, rhs, &mut applicability); - } + // ((Y - 1) + x) / Y + if inner_op.node == BinOpKind::Add && differ_by_one(inner_lhs, rhs) { + build_suggestion(cx, expr, inner_rhs, rhs, &mut applicability); + } - // (x - (-Y - 1)) / Y - if inner_op.node == BinOpKind::Sub - && let ExprKind::Unary(UnOp::Neg, abs_div_rhs) = rhs.kind - && differ_by_one(abs_div_rhs, inner_rhs) - { - build_suggestion(cx, expr, inner_lhs, rhs, &mut applicability); + // (x - (-Y - 1)) / Y + if inner_op.node == BinOpKind::Sub + && let ExprKind::Unary(UnOp::Neg, abs_div_rhs) = rhs.kind + && differ_by_one(abs_div_rhs, inner_rhs) + { + build_suggestion(cx, expr, inner_lhs, rhs, &mut applicability); + } + }, + ExprKind::MethodCall(method, receiver, [next_multiple_of_arg], _) => { + // x.next_multiple_of(Y) / Y + if method.ident.name == sym::next_multiple_of + && check_int_ty(cx.typeck_results().expr_ty(receiver)) + && check_eq_expr(cx, next_multiple_of_arg, rhs) + { + build_suggestion(cx, expr, receiver, rhs, &mut applicability); + } + }, + ExprKind::Call(callee, [receiver, next_multiple_of_arg]) => { + // int_type::next_multiple_of(x, Y) / Y + if let Some(impl_ty_binder) = callee + .ty_rel_def_if_named(cx, sym::next_multiple_of) + .assoc_fn_parent(cx) + .opt_impl_ty(cx) + && check_int_ty(impl_ty_binder.skip_binder()) + && check_eq_expr(cx, next_multiple_of_arg, rhs) + { + build_suggestion(cx, expr, receiver, rhs, &mut applicability); + } + }, + _ => (), } } } @@ -91,8 +117,11 @@ fn differ_by_one(small_expr: &Expr<'_>, large_expr: &Expr<'_>) -> bool { } } -fn check_int_ty_and_feature(cx: &LateContext<'_>, expr: &Expr<'_>) -> bool { - let expr_ty = cx.typeck_results().expr_ty(expr); +fn check_int_ty(expr_ty: Ty<'_>) -> bool { + matches!(expr_ty.peel_refs().kind(), ty::Int(_) | ty::Uint(_)) +} + +fn check_int_ty_and_feature(cx: &LateContext<'_>, expr_ty: Ty<'_>) -> bool { match expr_ty.peel_refs().kind() { ty::Uint(_) => true, ty::Int(_) => cx.tcx.features().enabled(sym::int_roundings), diff --git a/clippy_lints/src/operators/manual_is_multiple_of.rs b/clippy_lints/src/operators/manual_is_multiple_of.rs index 0b9bd4fb6d32..291d81097b51 100644 --- a/clippy_lints/src/operators/manual_is_multiple_of.rs +++ b/clippy_lints/src/operators/manual_is_multiple_of.rs @@ -35,7 +35,7 @@ pub(super) fn check<'tcx>( { let mut app = Applicability::MachineApplicable; let divisor = deref_sugg( - Sugg::hir_with_applicability(cx, operand_right, "_", &mut app), + Sugg::hir_with_context(cx, operand_right, expr.span.ctxt(), "_", &mut app), cx.typeck_results().expr_ty_adjusted(operand_right), ); span_lint_and_sugg( @@ -47,7 +47,7 @@ pub(super) fn check<'tcx>( format!( "{}{}.is_multiple_of({divisor})", if op == BinOpKind::Eq { "" } else { "!" }, - Sugg::hir_with_applicability(cx, operand_left, "_", &mut app).maybe_paren() + Sugg::hir_with_context(cx, operand_left, expr.span.ctxt(), "_", &mut app).maybe_paren() ), app, ); diff --git a/clippy_lints/src/question_mark.rs b/clippy_lints/src/question_mark.rs index 59d31f782bc3..e5fb3c0fa431 100644 --- a/clippy_lints/src/question_mark.rs +++ b/clippy_lints/src/question_mark.rs @@ -5,7 +5,7 @@ use clippy_config::types::MatchLintBehaviour; use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::msrvs::{self, Msrv}; use clippy_utils::res::{MaybeDef, MaybeQPath, MaybeResPath}; -use clippy_utils::source::snippet_with_applicability; +use clippy_utils::source::{snippet_with_applicability, snippet_with_context}; use clippy_utils::sugg::Sugg; use clippy_utils::ty::{implements_trait, is_copy}; use clippy_utils::usage::local_used_after_expr; @@ -147,7 +147,8 @@ fn check_let_some_else_return_none(cx: &LateContext<'_>, stmt: &Stmt<'_>) { && !span_contains_cfg(cx, els.span) { let mut applicability = Applicability::MaybeIncorrect; - let init_expr_str = Sugg::hir_with_applicability(cx, init_expr, "..", &mut applicability).maybe_paren(); + let init_expr_str = + Sugg::hir_with_context(cx, init_expr, stmt.span.ctxt(), "..", &mut applicability).maybe_paren(); // Take care when binding is `ref` let sugg = if let PatKind::Binding( BindingMode(ByRef::Yes(_, ref_mutability), binding_mutability), @@ -295,7 +296,7 @@ fn check_is_none_or_err_and_early_return<'tcx>(cx: &LateContext<'tcx>, expr: &Ex && (is_early_return(sym::Option, cx, &if_block) || is_early_return(sym::Result, cx, &if_block)) { let mut applicability = Applicability::MachineApplicable; - let receiver_str = snippet_with_applicability(cx, caller.span, "..", &mut applicability); + let receiver_str = snippet_with_context(cx, caller.span, expr.span.ctxt(), "..", &mut applicability).0; let by_ref = !cx.type_is_copy_modulo_regions(caller_ty) && !matches!(caller.kind, ExprKind::Call(..) | ExprKind::MethodCall(..)); let sugg = if let Some(else_inner) = r#else { diff --git a/clippy_lints/src/single_range_in_vec_init.rs b/clippy_lints/src/single_range_in_vec_init.rs index 92d1b112198f..e4906eb0c777 100644 --- a/clippy_lints/src/single_range_in_vec_init.rs +++ b/clippy_lints/src/single_range_in_vec_init.rs @@ -98,7 +98,7 @@ impl LateLintPass<'_> for SingleRangeInVecInit { && snippet.starts_with(suggested_type.starts_with()) && snippet.ends_with(suggested_type.ends_with()) { - let mut applicability = Applicability::MachineApplicable; + let mut applicability = Applicability::MaybeIncorrect; let (start_snippet, _) = snippet_with_context(cx, start.expr.span, span.ctxt(), "..", &mut applicability); let (end_snippet, _) = snippet_with_context(cx, end.expr.span, span.ctxt(), "..", &mut applicability); diff --git a/clippy_lints/src/strings.rs b/clippy_lints/src/strings.rs index 1d0efa46a14c..c0be724bcdee 100644 --- a/clippy_lints/src/strings.rs +++ b/clippy_lints/src/strings.rs @@ -1,6 +1,6 @@ use clippy_utils::diagnostics::{span_lint, span_lint_and_sugg, span_lint_and_then}; use clippy_utils::res::{MaybeDef, MaybeQPath}; -use clippy_utils::source::{snippet, snippet_with_applicability}; +use clippy_utils::source::{snippet, snippet_with_applicability, snippet_with_context}; use clippy_utils::{ SpanlessEq, get_expr_use_or_unification_node, get_parent_expr, is_lint_allowed, method_calls, peel_blocks, sym, }; @@ -273,6 +273,7 @@ impl<'tcx> LateLintPass<'tcx> for StringLitAsBytes { let string_expression = &expressions[0].0; let snippet_app = snippet_with_applicability(cx, string_expression.span, "..", &mut applicability); + let (right_snip, _) = snippet_with_context(cx, right.span, e.span.ctxt(), "..", &mut applicability); span_lint_and_sugg( cx, @@ -280,7 +281,7 @@ impl<'tcx> LateLintPass<'tcx> for StringLitAsBytes { e.span, "calling a slice of `as_bytes()` with `from_utf8` should be not necessary", "try", - format!("Some(&{snippet_app}[{}])", snippet(cx, right.span, "..")), + format!("Some(&{snippet_app}[{right_snip}])"), applicability, ); } @@ -404,7 +405,8 @@ impl<'tcx> LateLintPass<'tcx> for StrToString { "`to_string()` called on a `&str`", |diag| { let mut applicability = Applicability::MachineApplicable; - let snippet = snippet_with_applicability(cx, self_arg.span, "..", &mut applicability); + let (snippet, _) = + snippet_with_context(cx, self_arg.span, expr.span.ctxt(), "..", &mut applicability); diag.span_suggestion(expr.span, "try", format!("{snippet}.to_owned()"), applicability); }, ); diff --git a/clippy_lints/src/strlen_on_c_strings.rs b/clippy_lints/src/strlen_on_c_strings.rs index 58d692db5029..0d50bd547652 100644 --- a/clippy_lints/src/strlen_on_c_strings.rs +++ b/clippy_lints/src/strlen_on_c_strings.rs @@ -14,8 +14,8 @@ declare_clippy_lint! { /// and suggest calling `as_bytes().len()` or `to_bytes().len()` respectively instead. /// /// ### Why is this bad? - /// This avoids calling an unsafe `libc` function. - /// Currently, it also avoids calculating the length. + /// libc::strlen is an unsafe function, which we don't need to call + /// if all we want to know is the length of the c-string. /// /// ### Example /// ```rust, ignore diff --git a/clippy_lints/src/transmute/transmuting_null.rs b/clippy_lints/src/transmute/transmuting_null.rs index 31e770f421e1..3f435f255d91 100644 --- a/clippy_lints/src/transmute/transmuting_null.rs +++ b/clippy_lints/src/transmute/transmuting_null.rs @@ -2,7 +2,7 @@ use clippy_utils::consts::{ConstEvalCtxt, Constant}; use clippy_utils::diagnostics::span_lint; use clippy_utils::is_integer_const; use clippy_utils::res::{MaybeDef, MaybeResPath}; -use rustc_hir::{Expr, ExprKind}; +use rustc_hir::{ConstBlock, Expr, ExprKind}; use rustc_lint::LateContext; use rustc_middle::ty::Ty; use rustc_span::symbol::sym; @@ -42,5 +42,23 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, arg: &'t return true; } + // Catching: + // `std::mem::transmute({ 0 as *const u64 })` and similar const blocks + if let ExprKind::Block(block, _) = arg.kind + && block.stmts.is_empty() + && let Some(inner) = block.expr + { + // Run again with the inner expression + return check(cx, expr, inner, to_ty); + } + + // Catching: + // `std::mem::transmute(const { u64::MIN as *const u64 });` + if let ExprKind::ConstBlock(ConstBlock { body, .. }) = arg.kind { + // Strip out the const and run again + let block = cx.tcx.hir_body(body).value; + return check(cx, expr, block, to_ty); + } + false } diff --git a/clippy_lints/src/useless_conversion.rs b/clippy_lints/src/useless_conversion.rs index c06313d1a4c4..423301edfe83 100644 --- a/clippy_lints/src/useless_conversion.rs +++ b/clippy_lints/src/useless_conversion.rs @@ -456,13 +456,25 @@ fn has_eligible_receiver(cx: &LateContext<'_>, recv: &Expr<'_>, expr: &Expr<'_>) fn adjustments(cx: &LateContext<'_>, expr: &Expr<'_>) -> String { let mut prefix = String::new(); - for adj in cx.typeck_results().expr_adjustments(expr) { + + let adjustments = cx.typeck_results().expr_adjustments(expr); + + let [.., last] = adjustments else { return prefix }; + let target = last.target; + + for adj in adjustments { match adj.kind { Adjust::Deref(_) => prefix = format!("*{prefix}"), Adjust::Borrow(AutoBorrow::Ref(AutoBorrowMutability::Mut { .. })) => prefix = format!("&mut {prefix}"), Adjust::Borrow(AutoBorrow::Ref(AutoBorrowMutability::Not)) => prefix = format!("&{prefix}"), _ => {}, } + + // Stop once we reach the final target type. + // This prevents over-adjusting (e.g. suggesting &**y instead of *y). + if adj.target == target { + break; + } } prefix } diff --git a/clippy_lints/src/utils/author.rs b/clippy_lints/src/utils/author.rs index 7a54ba7a8fe1..58b153f06545 100644 --- a/clippy_lints/src/utils/author.rs +++ b/clippy_lints/src/utils/author.rs @@ -321,7 +321,6 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { }, ConstArgKind::Struct(..) => chain!(self, "let ConstArgKind::Struct(..) = {const_arg}.kind"), ConstArgKind::TupleCall(..) => chain!(self, "let ConstArgKind::TupleCall(..) = {const_arg}.kind"), - ConstArgKind::Tup(..) => chain!(self, "let ConstArgKind::Tup(..) = {const_arg}.kind"), ConstArgKind::Infer(..) => chain!(self, "let ConstArgKind::Infer(..) = {const_arg}.kind"), ConstArgKind::Error(..) => chain!(self, "let ConstArgKind::Error(..) = {const_arg}.kind"), ConstArgKind::Literal(..) => chain!(self, "let ConstArgKind::Literal(..) = {const_arg}.kind") diff --git a/clippy_lints_internal/src/lint_without_lint_pass.rs b/clippy_lints_internal/src/lint_without_lint_pass.rs index fda65bc84eda..f56b6f31550b 100644 --- a/clippy_lints_internal/src/lint_without_lint_pass.rs +++ b/clippy_lints_internal/src/lint_without_lint_pass.rs @@ -250,8 +250,8 @@ pub(super) fn extract_clippy_version_value(cx: &LateContext<'_>, item: &'_ Item< if let hir::Attribute::Unparsed(attr_kind) = &attr // Identify attribute && let [tool_name, attr_name] = &attr_kind.path.segments[..] - && tool_name.name == sym::clippy - && attr_name.name == sym::version + && tool_name == &sym::clippy + && attr_name == &sym::version && let Some(version) = attr.value_str() { Some(version) diff --git a/clippy_utils/README.md b/clippy_utils/README.md index 01257c1a3059..ecd36b157571 100644 --- a/clippy_utils/README.md +++ b/clippy_utils/README.md @@ -8,7 +8,7 @@ This crate is only guaranteed to build with this `nightly` toolchain: ``` -nightly-2025-12-25 +nightly-2026-01-08 ``` @@ -30,7 +30,7 @@ Function signatures can change or be removed without replacement without any pri -Copyright 2014-2025 The Rust Project Developers +Copyright (c) The Rust Project Contributors Licensed under the Apache License, Version 2.0 <[https://www.apache.org/licenses/LICENSE-2.0](https://www.apache.org/licenses/LICENSE-2.0)> or the MIT license diff --git a/clippy_utils/src/ast_utils/mod.rs b/clippy_utils/src/ast_utils/mod.rs index 618719286e8f..cf8716398efb 100644 --- a/clippy_utils/src/ast_utils/mod.rs +++ b/clippy_utils/src/ast_utils/mod.rs @@ -976,7 +976,9 @@ pub fn eq_attr(l: &Attribute, r: &Attribute) -> bool { l.style == r.style && match (&l.kind, &r.kind) { (DocComment(l1, l2), DocComment(r1, r2)) => l1 == r1 && l2 == r2, - (Normal(l), Normal(r)) => eq_path(&l.item.path, &r.item.path) && eq_attr_item_kind(&l.item.args, &r.item.args), + (Normal(l), Normal(r)) => { + eq_path(&l.item.path, &r.item.path) && eq_attr_item_kind(&l.item.args, &r.item.args) + }, _ => false, } } diff --git a/clippy_utils/src/attrs.rs b/clippy_utils/src/attrs.rs index 87fdd755908b..32f6cb4fd5e9 100644 --- a/clippy_utils/src/attrs.rs +++ b/clippy_utils/src/attrs.rs @@ -23,7 +23,9 @@ pub fn get_builtin_attr<'a, A: AttributeExt + 'a>( if let [clippy, segment2] = &*attr.path() && *clippy == sym::clippy { - let path_span = attr.path_span().expect("Clippy attributes are unparsed and have a span"); + let path_span = attr + .path_span() + .expect("Clippy attributes are unparsed and have a span"); let new_name = match *segment2 { sym::cyclomatic_complexity => Some("cognitive_complexity"), sym::author @@ -48,7 +50,7 @@ pub fn get_builtin_attr<'a, A: AttributeExt + 'a>( .with_span_suggestion( path_span, "consider using", - format!("clippy::{}", new_name), + format!("clippy::{new_name}"), Applicability::MachineApplicable, ) .emit(); diff --git a/clippy_utils/src/consts.rs b/clippy_utils/src/consts.rs index 5f4b87590dc1..334cc6bb5d55 100644 --- a/clippy_utils/src/consts.rs +++ b/clippy_utils/src/consts.rs @@ -1140,9 +1140,11 @@ pub fn const_item_rhs_to_expr<'tcx>(tcx: TyCtxt<'tcx>, ct_rhs: ConstItemRhs<'tcx ConstItemRhs::Body(body_id) => Some(tcx.hir_body(body_id).value), ConstItemRhs::TypeConst(const_arg) => match const_arg.kind { ConstArgKind::Anon(anon) => Some(tcx.hir_body(anon.body).value), - ConstArgKind::Struct(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Tup(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) | ConstArgKind::Infer(..) | ConstArgKind::Literal(..) => { - None - }, + ConstArgKind::Struct(..) + | ConstArgKind::TupleCall(..) + | ConstArgKind::Path(_) + | ConstArgKind::Error(..) + | ConstArgKind::Infer(..) => None, }, } } diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs index b4e483ea8072..73b1cbb21548 100644 --- a/clippy_utils/src/hir_utils.rs +++ b/clippy_utils/src/hir_utils.rs @@ -675,7 +675,7 @@ impl HirEqInterExpr<'_, '_, '_> { .iter() .zip(*inits_b) .all(|(init_a, init_b)| self.eq_const_arg(init_a.expr, init_b.expr)) - } + }, (ConstArgKind::TupleCall(path_a, args_a), ConstArgKind::TupleCall(path_b, args_b)) => { self.eq_qpath(path_a, path_b) && args_a @@ -688,7 +688,7 @@ impl HirEqInterExpr<'_, '_, '_> { .iter() .zip(*args_b) .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) - } + }, (ConstArgKind::Literal(kind_l), ConstArgKind::Literal(kind_r)) => { kind_l == kind_r }, diff --git a/clippy_utils/src/lib.rs b/clippy_utils/src/lib.rs index 38e1542cd758..2a620e917228 100644 --- a/clippy_utils/src/lib.rs +++ b/clippy_utils/src/lib.rs @@ -90,13 +90,13 @@ use std::sync::{Mutex, MutexGuard, OnceLock}; use itertools::Itertools; use rustc_abi::Integer; use rustc_ast::ast::{self, LitKind, RangeLimits}; -use rustc_ast::join_path_syms; +use rustc_ast::{LitIntType, join_path_syms}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::indexmap; use rustc_data_structures::packed::Pu128; use rustc_data_structures::unhash::UnindexMap; use rustc_hir::LangItem::{OptionNone, OptionSome, ResultErr, ResultOk}; -use rustc_hir::attrs::AttributeKind; +use rustc_hir::attrs::{AttributeKind, CfgEntry}; use rustc_hir::def::{DefKind, Res}; use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId}; use rustc_hir::definitions::{DefPath, DefPathData}; @@ -121,7 +121,6 @@ use rustc_middle::ty::{ self as rustc_ty, Binder, BorrowKind, ClosureKind, EarlyBinder, GenericArgKind, GenericArgsRef, IntTy, Ty, TyCtxt, TypeFlags, TypeVisitableExt, UintTy, UpvarCapture, }; -use rustc_hir::attrs::CfgEntry; use rustc_span::hygiene::{ExpnKind, MacroKind}; use rustc_span::source_map::SourceMap; use rustc_span::symbol::{Ident, Symbol, kw}; @@ -1386,6 +1385,17 @@ pub fn is_integer_literal(expr: &Expr<'_>, value: u128) -> bool { false } +/// Checks whether the given expression is an untyped integer literal. +pub fn is_integer_literal_untyped(expr: &Expr<'_>) -> bool { + if let ExprKind::Lit(spanned) = expr.kind + && let LitKind::Int(_, suffix) = spanned.node + { + return suffix == LitIntType::Unsuffixed; + } + + false +} + /// Checks whether the given expression is a constant literal of the given value. pub fn is_float_literal(expr: &Expr<'_>, value: f64) -> bool { if let ExprKind::Lit(spanned) = expr.kind @@ -2403,7 +2413,10 @@ pub fn is_test_function(tcx: TyCtxt<'_>, fn_def_id: LocalDefId) -> bool { /// use [`is_in_cfg_test`] pub fn is_cfg_test(tcx: TyCtxt<'_>, id: HirId) -> bool { if let Some(cfgs) = find_attr!(tcx.hir_attrs(id), AttributeKind::CfgTrace(cfgs) => cfgs) - && cfgs.iter().any(|(cfg, _)| { matches!(cfg, CfgEntry::NameValue { name: sym::test, ..})}) { + && cfgs + .iter() + .any(|(cfg, _)| matches!(cfg, CfgEntry::NameValue { name: sym::test, .. })) + { true } else { false @@ -2423,9 +2436,11 @@ pub fn is_in_test(tcx: TyCtxt<'_>, hir_id: HirId) -> bool { /// Checks if the item of any of its parents has `#[cfg(...)]` attribute applied. pub fn inherits_cfg(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { find_attr!(tcx.get_all_attrs(def_id), AttributeKind::CfgTrace(..)) - || find_attr!(tcx - .hir_parent_iter(tcx.local_def_id_to_hir_id(def_id)) - .flat_map(|(parent_id, _)| tcx.hir_attrs(parent_id)), AttributeKind::CfgTrace(..)) + || find_attr!( + tcx.hir_parent_iter(tcx.local_def_id_to_hir_id(def_id)) + .flat_map(|(parent_id, _)| tcx.hir_attrs(parent_id)), + AttributeKind::CfgTrace(..) + ) } /// Walks up the HIR tree from the given expression in an attempt to find where the value is diff --git a/clippy_utils/src/res.rs b/clippy_utils/src/res.rs index a3efece7d224..e890a73620a5 100644 --- a/clippy_utils/src/res.rs +++ b/clippy_utils/src/res.rs @@ -301,7 +301,7 @@ impl<'tcx> MaybeQPath<'tcx> for &'tcx PatExpr<'_> { fn opt_qpath(self) -> Option> { match &self.kind { PatExprKind::Path(qpath) => Some((qpath, self.hir_id)), - _ => None, + PatExprKind::Lit { .. } => None, } } } @@ -419,7 +419,7 @@ impl<'a> MaybeResPath<'a> for &PatExpr<'a> { fn opt_res_path(self) -> OptResPath<'a> { match &self.kind { PatExprKind::Path(qpath) => qpath.opt_res_path(), - _ => (None, None), + PatExprKind::Lit { .. } => (None, None), } } } diff --git a/clippy_utils/src/sugg.rs b/clippy_utils/src/sugg.rs index 2ef2afb45071..3ade38bea8ed 100644 --- a/clippy_utils/src/sugg.rs +++ b/clippy_utils/src/sugg.rs @@ -127,7 +127,7 @@ impl<'a> Sugg<'a> { /// Generate a suggestion for an expression with the given snippet. This is used by the `hir_*` /// function variants of `Sugg`, since these use different snippet functions. - fn hir_from_snippet( + pub fn hir_from_snippet( cx: &LateContext<'_>, expr: &hir::Expr<'_>, mut get_snippet: impl FnMut(Span) -> Cow<'a, str>, diff --git a/clippy_utils/src/sym.rs b/clippy_utils/src/sym.rs index a0d2e8673fe6..5357a0941d32 100644 --- a/clippy_utils/src/sym.rs +++ b/clippy_utils/src/sym.rs @@ -248,6 +248,7 @@ generate! { next_back, next_if, next_if_eq, + next_multiple_of, next_tuple, nth, ok, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index dbec79e111fb..0755e1d29c69 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,6 +1,6 @@ [toolchain] # begin autogenerated nightly -channel = "nightly-2025-12-25" +channel = "nightly-2026-01-08" # end autogenerated nightly components = ["cargo", "llvm-tools", "rust-src", "rust-std", "rustc", "rustc-dev", "rustfmt"] profile = "minimal" diff --git a/rustc_tools_util/README.md b/rustc_tools_util/README.md index f47a4c69c2c3..45d2844ad00b 100644 --- a/rustc_tools_util/README.md +++ b/rustc_tools_util/README.md @@ -51,7 +51,7 @@ The changelog for `rustc_tools_util` is available under: -Copyright 2014-2025 The Rust Project Developers +Copyright (c) The Rust Project Contributors Licensed under the Apache License, Version 2.0 or the MIT license diff --git a/tests/ui-toml/missing_enforced_import_rename/clippy.toml b/tests/ui-toml/missing_enforced_import_rename/clippy.toml index 05ba822874d5..11af5b0a3306 100644 --- a/tests/ui-toml/missing_enforced_import_rename/clippy.toml +++ b/tests/ui-toml/missing_enforced_import_rename/clippy.toml @@ -6,5 +6,6 @@ enforced-import-renames = [ { path = "std::clone", rename = "foo" }, { path = "std::thread::sleep", rename = "thread_sleep" }, { path = "std::any::type_name", rename = "ident" }, - { path = "std::sync::Mutex", rename = "StdMutie" } + { path = "std::sync::Mutex", rename = "StdMutie" }, + { path = "std::io::Write", rename = "to_test_rename_as_underscore" } ] diff --git a/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.fixed b/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.fixed index 3e882f496985..c96df2884b8d 100644 --- a/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.fixed +++ b/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.fixed @@ -15,6 +15,7 @@ use std::{ sync :: Mutex as StdMutie, //~^ missing_enforced_import_renames }; +use std::io::Write as _; fn main() { use std::collections::BTreeMap as Map; diff --git a/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.rs b/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.rs index 32255af5117f..662e89157675 100644 --- a/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.rs +++ b/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.rs @@ -15,6 +15,7 @@ use std::{ sync :: Mutex, //~^ missing_enforced_import_renames }; +use std::io::Write as _; fn main() { use std::collections::BTreeMap as OopsWrongRename; diff --git a/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.stderr b/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.stderr index 982b144eb872..139331d17619 100644 --- a/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.stderr +++ b/tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.stderr @@ -32,7 +32,7 @@ LL | sync :: Mutex, | ^^^^^^^^^^^^^ help: try: `sync :: Mutex as StdMutie` error: this import should be renamed - --> tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.rs:20:5 + --> tests/ui-toml/missing_enforced_import_rename/conf_missing_enforced_import_rename.rs:21:5 | LL | use std::collections::BTreeMap as OopsWrongRename; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `use std::collections::BTreeMap as Map` diff --git a/tests/ui/bool_assert_comparison.fixed b/tests/ui/bool_assert_comparison.fixed index ec76abbef05a..cd390ce0db9d 100644 --- a/tests/ui/bool_assert_comparison.fixed +++ b/tests/ui/bool_assert_comparison.fixed @@ -216,3 +216,16 @@ fn main() { assert!(!(b + b)); //~^ bool_assert_comparison } + +fn issue16279() { + macro_rules! is_empty { + ($x:expr) => { + $x.is_empty() + }; + } + + assert!(!is_empty!("a")); + //~^ bool_assert_comparison + assert!(is_empty!("")); + //~^ bool_assert_comparison +} diff --git a/tests/ui/bool_assert_comparison.rs b/tests/ui/bool_assert_comparison.rs index 40824a23c82e..b2ea5b6ea540 100644 --- a/tests/ui/bool_assert_comparison.rs +++ b/tests/ui/bool_assert_comparison.rs @@ -216,3 +216,16 @@ fn main() { assert_eq!(b + b, false); //~^ bool_assert_comparison } + +fn issue16279() { + macro_rules! is_empty { + ($x:expr) => { + $x.is_empty() + }; + } + + assert_eq!(is_empty!("a"), false); + //~^ bool_assert_comparison + assert_eq!(is_empty!(""), true); + //~^ bool_assert_comparison +} diff --git a/tests/ui/bool_assert_comparison.stderr b/tests/ui/bool_assert_comparison.stderr index 72aa6303a202..b4e8fcf09bb6 100644 --- a/tests/ui/bool_assert_comparison.stderr +++ b/tests/ui/bool_assert_comparison.stderr @@ -444,5 +444,29 @@ LL - assert_eq!(b + b, false); LL + assert!(!(b + b)); | -error: aborting due to 37 previous errors +error: used `assert_eq!` with a literal bool + --> tests/ui/bool_assert_comparison.rs:227:5 + | +LL | assert_eq!(is_empty!("a"), false); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: replace it with `assert!(..)` + | +LL - assert_eq!(is_empty!("a"), false); +LL + assert!(!is_empty!("a")); + | + +error: used `assert_eq!` with a literal bool + --> tests/ui/bool_assert_comparison.rs:229:5 + | +LL | assert_eq!(is_empty!(""), true); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: replace it with `assert!(..)` + | +LL - assert_eq!(is_empty!(""), true); +LL + assert!(is_empty!("")); + | + +error: aborting due to 39 previous errors diff --git a/tests/ui/checked_conversions.fixed b/tests/ui/checked_conversions.fixed index 6175275ef047..2309a053146f 100644 --- a/tests/ui/checked_conversions.fixed +++ b/tests/ui/checked_conversions.fixed @@ -1,6 +1,7 @@ #![allow( clippy::cast_lossless, clippy::legacy_numeric_constants, + clippy::no_effect, unused, // Int::max_value will be deprecated in the future deprecated, @@ -105,4 +106,19 @@ fn msrv_1_34() { //~^ checked_conversions } +fn issue16293() { + struct Outer { + inner: u32, + } + let outer = Outer { inner: 42 }; + macro_rules! dot_inner { + ($obj:expr) => { + $obj.inner + }; + } + + i32::try_from(dot_inner!(outer)).is_ok(); + //~^ checked_conversions +} + fn main() {} diff --git a/tests/ui/checked_conversions.rs b/tests/ui/checked_conversions.rs index 9ed0e8f660d0..dabb552eba27 100644 --- a/tests/ui/checked_conversions.rs +++ b/tests/ui/checked_conversions.rs @@ -1,6 +1,7 @@ #![allow( clippy::cast_lossless, clippy::legacy_numeric_constants, + clippy::no_effect, unused, // Int::max_value will be deprecated in the future deprecated, @@ -105,4 +106,19 @@ fn msrv_1_34() { //~^ checked_conversions } +fn issue16293() { + struct Outer { + inner: u32, + } + let outer = Outer { inner: 42 }; + macro_rules! dot_inner { + ($obj:expr) => { + $obj.inner + }; + } + + dot_inner!(outer) <= i32::MAX as u32; + //~^ checked_conversions +} + fn main() {} diff --git a/tests/ui/checked_conversions.stderr b/tests/ui/checked_conversions.stderr index 624876dacb26..6018dacace39 100644 --- a/tests/ui/checked_conversions.stderr +++ b/tests/ui/checked_conversions.stderr @@ -1,5 +1,5 @@ error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:15:13 + --> tests/ui/checked_conversions.rs:16:13 | LL | let _ = value <= (u32::max_value() as i64) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u32::try_from(value).is_ok()` @@ -8,100 +8,106 @@ LL | let _ = value <= (u32::max_value() as i64) && value >= 0; = help: to override `-D warnings` add `#[allow(clippy::checked_conversions)]` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:17:13 + --> tests/ui/checked_conversions.rs:18:13 | LL | let _ = value <= (u32::MAX as i64) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u32::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:22:13 + --> tests/ui/checked_conversions.rs:23:13 | LL | let _ = value <= i64::from(u16::max_value()) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u16::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:24:13 + --> tests/ui/checked_conversions.rs:25:13 | LL | let _ = value <= i64::from(u16::MAX) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u16::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:29:13 + --> tests/ui/checked_conversions.rs:30:13 | LL | let _ = value <= (u8::max_value() as isize) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u8::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:31:13 + --> tests/ui/checked_conversions.rs:32:13 | LL | let _ = value <= (u8::MAX as isize) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u8::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:38:13 + --> tests/ui/checked_conversions.rs:39:13 | LL | let _ = value <= (i32::max_value() as i64) && value >= (i32::min_value() as i64); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i32::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:40:13 + --> tests/ui/checked_conversions.rs:41:13 | LL | let _ = value <= (i32::MAX as i64) && value >= (i32::MIN as i64); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i32::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:45:13 + --> tests/ui/checked_conversions.rs:46:13 | LL | let _ = value <= i64::from(i16::max_value()) && value >= i64::from(i16::min_value()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i16::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:47:13 + --> tests/ui/checked_conversions.rs:48:13 | LL | let _ = value <= i64::from(i16::MAX) && value >= i64::from(i16::MIN); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i16::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:54:13 + --> tests/ui/checked_conversions.rs:55:13 | LL | let _ = value <= i32::max_value() as u32; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i32::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:56:13 + --> tests/ui/checked_conversions.rs:57:13 | LL | let _ = value <= i32::MAX as u32; | ^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i32::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:61:13 + --> tests/ui/checked_conversions.rs:62:13 | LL | let _ = value <= isize::max_value() as usize && value as i32 == 5; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `isize::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:63:13 + --> tests/ui/checked_conversions.rs:64:13 | LL | let _ = value <= isize::MAX as usize && value as i32 == 5; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `isize::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:68:13 + --> tests/ui/checked_conversions.rs:69:13 | LL | let _ = value <= u16::max_value() as u32 && value as i32 == 5; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u16::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:70:13 + --> tests/ui/checked_conversions.rs:71:13 | LL | let _ = value <= u16::MAX as u32 && value as i32 == 5; | ^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u16::try_from(value).is_ok()` error: checked cast can be simplified - --> tests/ui/checked_conversions.rs:104:13 + --> tests/ui/checked_conversions.rs:105:13 | LL | let _ = value <= (u32::max_value() as i64) && value >= 0; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `u32::try_from(value).is_ok()` -error: aborting due to 17 previous errors +error: checked cast can be simplified + --> tests/ui/checked_conversions.rs:120:5 + | +LL | dot_inner!(outer) <= i32::MAX as u32; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `i32::try_from(dot_inner!(outer)).is_ok()` + +error: aborting due to 18 previous errors diff --git a/tests/ui/cmp_owned/with_suggestion.fixed b/tests/ui/cmp_owned/with_suggestion.fixed index 85d0991bef05..4c3b13b30043 100644 --- a/tests/ui/cmp_owned/with_suggestion.fixed +++ b/tests/ui/cmp_owned/with_suggestion.fixed @@ -83,3 +83,32 @@ fn issue_8103() { let _ = foo1 == foo2; //~^ cmp_owned } + +macro_rules! issue16322_macro_generator { + ($locale:ident) => { + mod $locale { + macro_rules! _make { + ($token:tt) => { + stringify!($token) + }; + } + + pub(crate) use _make; + } + + macro_rules! t { + ($token:tt) => { + crate::$locale::_make!($token) + }; + } + }; +} + +issue16322_macro_generator!(de); + +fn issue16322(item: String) { + if item == t!(frohes_neu_Jahr) { + //~^ cmp_owned + println!("Ja!"); + } +} diff --git a/tests/ui/cmp_owned/with_suggestion.rs b/tests/ui/cmp_owned/with_suggestion.rs index 2393757d76f2..a9d7509feaaf 100644 --- a/tests/ui/cmp_owned/with_suggestion.rs +++ b/tests/ui/cmp_owned/with_suggestion.rs @@ -83,3 +83,32 @@ fn issue_8103() { let _ = foo1 == foo2.to_owned(); //~^ cmp_owned } + +macro_rules! issue16322_macro_generator { + ($locale:ident) => { + mod $locale { + macro_rules! _make { + ($token:tt) => { + stringify!($token) + }; + } + + pub(crate) use _make; + } + + macro_rules! t { + ($token:tt) => { + crate::$locale::_make!($token) + }; + } + }; +} + +issue16322_macro_generator!(de); + +fn issue16322(item: String) { + if item == t!(frohes_neu_Jahr).to_string() { + //~^ cmp_owned + println!("Ja!"); + } +} diff --git a/tests/ui/cmp_owned/with_suggestion.stderr b/tests/ui/cmp_owned/with_suggestion.stderr index dd9ffa70897a..66544ce0c217 100644 --- a/tests/ui/cmp_owned/with_suggestion.stderr +++ b/tests/ui/cmp_owned/with_suggestion.stderr @@ -49,5 +49,11 @@ error: this creates an owned instance just for comparison LL | let _ = foo1 == foo2.to_owned(); | ^^^^^^^^^^^^^^^ help: try: `foo2` -error: aborting due to 8 previous errors +error: this creates an owned instance just for comparison + --> tests/ui/cmp_owned/with_suggestion.rs:110:16 + | +LL | if item == t!(frohes_neu_Jahr).to_string() { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `t!(frohes_neu_Jahr)` + +error: aborting due to 9 previous errors diff --git a/tests/ui/derive_ord_xor_partial_ord.rs b/tests/ui/derive_ord_xor_partial_ord.rs index b4bb24b0d2fe..386ab39401c5 100644 --- a/tests/ui/derive_ord_xor_partial_ord.rs +++ b/tests/ui/derive_ord_xor_partial_ord.rs @@ -91,3 +91,17 @@ mod issue15708 { } } } + +mod issue16298 { + #[derive(Clone, Copy, Debug, Default, PartialEq, PartialOrd)] + struct Normalized(S); + + impl Eq for Normalized {} + + #[expect(clippy::derive_ord_xor_partial_ord)] + impl Ord for Normalized { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.partial_cmp(other).unwrap() + } + } +} diff --git a/tests/ui/double_parens.fixed b/tests/ui/double_parens.fixed index 024af6840132..ef7838491f8f 100644 --- a/tests/ui/double_parens.fixed +++ b/tests/ui/double_parens.fixed @@ -161,4 +161,20 @@ fn issue15940() { pub struct Person; } +fn issue16224() { + fn test() -> i32 { 42 } + + macro_rules! call { + ($matcher:pat $(=> $result:expr)?) => { + match test() { + $matcher => Result::Ok(($($result),*)), + _ => Result::Err("No match".to_string()), + } + }; + } + + let _: Result<(), String> = call!(_); + let _: Result = call!(_ => 42); +} + fn main() {} diff --git a/tests/ui/double_parens.rs b/tests/ui/double_parens.rs index 8a76f2837f35..07eafdf69575 100644 --- a/tests/ui/double_parens.rs +++ b/tests/ui/double_parens.rs @@ -161,4 +161,20 @@ fn issue15940() { pub struct Person; } +fn issue16224() { + fn test() -> i32 { 42 } + + macro_rules! call { + ($matcher:pat $(=> $result:expr)?) => { + match test() { + $matcher => Result::Ok(($($result),*)), + _ => Result::Err("No match".to_string()), + } + }; + } + + let _: Result<(), String> = call!(_); + let _: Result = call!(_ => 42); +} + fn main() {} diff --git a/tests/ui/for_kv_map.fixed b/tests/ui/for_kv_map.fixed index 2a68b7443fbf..6ec4cb01ffd1 100644 --- a/tests/ui/for_kv_map.fixed +++ b/tests/ui/for_kv_map.fixed @@ -69,3 +69,20 @@ fn main() { let _v = v; } } + +fn wrongly_unmangled_macros() { + use std::collections::HashMap; + + macro_rules! test_map { + ($val:expr) => { + &*$val + }; + } + + let m: HashMap = HashMap::new(); + let wrapped = Rc::new(m); + for v in test_map!(wrapped).values() { + //~^ for_kv_map + let _v = v; + } +} diff --git a/tests/ui/for_kv_map.rs b/tests/ui/for_kv_map.rs index 485a97815e3c..19e907ff10a6 100644 --- a/tests/ui/for_kv_map.rs +++ b/tests/ui/for_kv_map.rs @@ -69,3 +69,20 @@ fn main() { let _v = v; } } + +fn wrongly_unmangled_macros() { + use std::collections::HashMap; + + macro_rules! test_map { + ($val:expr) => { + &*$val + }; + } + + let m: HashMap = HashMap::new(); + let wrapped = Rc::new(m); + for (_, v) in test_map!(wrapped) { + //~^ for_kv_map + let _v = v; + } +} diff --git a/tests/ui/for_kv_map.stderr b/tests/ui/for_kv_map.stderr index 0bd474a10682..5436592f2ab6 100644 --- a/tests/ui/for_kv_map.stderr +++ b/tests/ui/for_kv_map.stderr @@ -72,5 +72,17 @@ LL - 'label: for (k, _value) in rm { LL + 'label: for k in rm.keys() { | -error: aborting due to 6 previous errors +error: you seem to want to iterate on a map's values + --> tests/ui/for_kv_map.rs:84:19 + | +LL | for (_, v) in test_map!(wrapped) { + | ^^^^^^^^^^^^^^^^^^ + | +help: use the corresponding method + | +LL - for (_, v) in test_map!(wrapped) { +LL + for v in test_map!(wrapped).values() { + | + +error: aborting due to 7 previous errors diff --git a/tests/ui/impl.rs b/tests/ui/impl.rs index e6044cc50781..75761a34c86e 100644 --- a/tests/ui/impl.rs +++ b/tests/ui/impl.rs @@ -14,6 +14,7 @@ impl MyStruct { } impl<'a> MyStruct { + //~^ multiple_inherent_impl fn lifetimed() {} } @@ -90,10 +91,12 @@ struct Lifetime<'s> { } impl Lifetime<'_> {} -impl Lifetime<'_> {} // false negative +impl Lifetime<'_> {} +//~^ multiple_inherent_impl impl<'a> Lifetime<'a> {} -impl<'a> Lifetime<'a> {} // false negative +impl<'a> Lifetime<'a> {} +//~^ multiple_inherent_impl impl<'b> Lifetime<'b> {} // false negative? @@ -104,6 +107,39 @@ struct Generic { } impl Generic {} -impl Generic {} // false negative +impl Generic {} +//~^ multiple_inherent_impl + +use std::fmt::Debug; + +#[derive(Debug)] +struct GenericWithBounds(T); + +impl GenericWithBounds { + fn make_one(_one: T) -> Self { + todo!() + } +} + +impl GenericWithBounds { + //~^ multiple_inherent_impl + fn make_two(_two: T) -> Self { + todo!() + } +} + +struct MultipleTraitBounds(T); + +impl MultipleTraitBounds { + fn debug_fn() {} +} + +impl MultipleTraitBounds { + fn clone_fn() {} +} + +impl MultipleTraitBounds { + fn debug_clone_fn() {} +} fn main() {} diff --git a/tests/ui/impl.stderr b/tests/ui/impl.stderr index 93d4b3998f90..9c4aaf183d70 100644 --- a/tests/ui/impl.stderr +++ b/tests/ui/impl.stderr @@ -19,7 +19,24 @@ LL | | } = help: to override `-D warnings` add `#[allow(clippy::multiple_inherent_impl)]` error: multiple implementations of this structure - --> tests/ui/impl.rs:26:5 + --> tests/ui/impl.rs:16:1 + | +LL | / impl<'a> MyStruct { +LL | | +LL | | fn lifetimed() {} +LL | | } + | |_^ + | +note: first implementation here + --> tests/ui/impl.rs:6:1 + | +LL | / impl MyStruct { +LL | | fn first() {} +LL | | } + | |_^ + +error: multiple implementations of this structure + --> tests/ui/impl.rs:27:5 | LL | / impl super::MyStruct { LL | | @@ -37,7 +54,7 @@ LL | | } | |_^ error: multiple implementations of this structure - --> tests/ui/impl.rs:48:1 + --> tests/ui/impl.rs:49:1 | LL | / impl WithArgs { LL | | @@ -47,7 +64,7 @@ LL | | } | |_^ | note: first implementation here - --> tests/ui/impl.rs:45:1 + --> tests/ui/impl.rs:46:1 | LL | / impl WithArgs { LL | | fn f2() {} @@ -55,28 +72,85 @@ LL | | } | |_^ error: multiple implementations of this structure - --> tests/ui/impl.rs:71:1 + --> tests/ui/impl.rs:72:1 | LL | impl OneAllowedImpl {} | ^^^^^^^^^^^^^^^^^^^^^^ | note: first implementation here - --> tests/ui/impl.rs:68:1 + --> tests/ui/impl.rs:69:1 | LL | impl OneAllowedImpl {} | ^^^^^^^^^^^^^^^^^^^^^^ error: multiple implementations of this structure - --> tests/ui/impl.rs:84:1 + --> tests/ui/impl.rs:85:1 | LL | impl OneExpected {} | ^^^^^^^^^^^^^^^^^^^ | note: first implementation here - --> tests/ui/impl.rs:81:1 + --> tests/ui/impl.rs:82:1 | LL | impl OneExpected {} | ^^^^^^^^^^^^^^^^^^^ -error: aborting due to 5 previous errors +error: multiple implementations of this structure + --> tests/ui/impl.rs:94:1 + | +LL | impl Lifetime<'_> {} + | ^^^^^^^^^^^^^^^^^^^^ + | +note: first implementation here + --> tests/ui/impl.rs:93:1 + | +LL | impl Lifetime<'_> {} + | ^^^^^^^^^^^^^^^^^^^^ + +error: multiple implementations of this structure + --> tests/ui/impl.rs:98:1 + | +LL | impl<'a> Lifetime<'a> {} + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +note: first implementation here + --> tests/ui/impl.rs:97:1 + | +LL | impl<'a> Lifetime<'a> {} + | ^^^^^^^^^^^^^^^^^^^^^^^^ + +error: multiple implementations of this structure + --> tests/ui/impl.rs:110:1 + | +LL | impl Generic {} + | ^^^^^^^^^^^^^^^^^^^^^ + | +note: first implementation here + --> tests/ui/impl.rs:109:1 + | +LL | impl Generic {} + | ^^^^^^^^^^^^^^^^^^^^^ + +error: multiple implementations of this structure + --> tests/ui/impl.rs:124:1 + | +LL | / impl GenericWithBounds { +LL | | +LL | | fn make_two(_two: T) -> Self { +LL | | todo!() +LL | | } +LL | | } + | |_^ + | +note: first implementation here + --> tests/ui/impl.rs:118:1 + | +LL | / impl GenericWithBounds { +LL | | fn make_one(_one: T) -> Self { +LL | | todo!() +LL | | } +LL | | } + | |_^ + +error: aborting due to 10 previous errors diff --git a/tests/ui/implicit_saturating_sub.fixed b/tests/ui/implicit_saturating_sub.fixed index 1aab6c54407e..22e59bbd2705 100644 --- a/tests/ui/implicit_saturating_sub.fixed +++ b/tests/ui/implicit_saturating_sub.fixed @@ -252,3 +252,11 @@ fn arbitrary_expression() { 0 }; } + +fn issue16307() { + let x: u8 = 100; + let y = 100_u8.saturating_sub(x); + //~^ implicit_saturating_sub + + println!("{y}"); +} diff --git a/tests/ui/implicit_saturating_sub.rs b/tests/ui/implicit_saturating_sub.rs index 7ca57a2902db..7fa19f0c8ad2 100644 --- a/tests/ui/implicit_saturating_sub.rs +++ b/tests/ui/implicit_saturating_sub.rs @@ -326,3 +326,11 @@ fn arbitrary_expression() { 0 }; } + +fn issue16307() { + let x: u8 = 100; + let y = if x >= 100 { 0 } else { 100 - x }; + //~^ implicit_saturating_sub + + println!("{y}"); +} diff --git a/tests/ui/implicit_saturating_sub.stderr b/tests/ui/implicit_saturating_sub.stderr index 0c225856fd07..2f3d2ba787e8 100644 --- a/tests/ui/implicit_saturating_sub.stderr +++ b/tests/ui/implicit_saturating_sub.stderr @@ -238,5 +238,11 @@ error: manual arithmetic check found LL | let _ = if a < b * 2 { 0 } else { a - b * 2 }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace it with: `a.saturating_sub(b * 2)` -error: aborting due to 27 previous errors +error: manual arithmetic check found + --> tests/ui/implicit_saturating_sub.rs:332:13 + | +LL | let y = if x >= 100 { 0 } else { 100 - x }; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace it with: `100_u8.saturating_sub(x)` + +error: aborting due to 28 previous errors diff --git a/tests/ui/iter_kv_map.fixed b/tests/ui/iter_kv_map.fixed index b18dda358877..189d76bc9431 100644 --- a/tests/ui/iter_kv_map.fixed +++ b/tests/ui/iter_kv_map.fixed @@ -189,3 +189,9 @@ fn issue14595() { let _ = map.as_ref().values().copied().collect::>(); //~^ iter_kv_map } + +fn issue16340() { + let hm: HashMap<&str, &str> = HashMap::new(); + let _ = hm.keys().map(|key| vec![key]); + //~^ iter_kv_map +} diff --git a/tests/ui/iter_kv_map.rs b/tests/ui/iter_kv_map.rs index 729e4e8a266c..cfc303447004 100644 --- a/tests/ui/iter_kv_map.rs +++ b/tests/ui/iter_kv_map.rs @@ -193,3 +193,9 @@ fn issue14595() { let _ = map.as_ref().iter().map(|(_, v)| v).copied().collect::>(); //~^ iter_kv_map } + +fn issue16340() { + let hm: HashMap<&str, &str> = HashMap::new(); + let _ = hm.iter().map(|(key, _)| vec![key]); + //~^ iter_kv_map +} diff --git a/tests/ui/iter_kv_map.stderr b/tests/ui/iter_kv_map.stderr index 8f73541f5033..866e69ea1922 100644 --- a/tests/ui/iter_kv_map.stderr +++ b/tests/ui/iter_kv_map.stderr @@ -269,5 +269,11 @@ error: iterating on a map's values LL | let _ = map.as_ref().iter().map(|(_, v)| v).copied().collect::>(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `map.as_ref().values()` -error: aborting due to 39 previous errors +error: iterating on a map's keys + --> tests/ui/iter_kv_map.rs:199:13 + | +LL | let _ = hm.iter().map(|(key, _)| vec![key]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `hm.keys().map(|key| vec![key])` + +error: aborting due to 40 previous errors diff --git a/tests/ui/manual_div_ceil.fixed b/tests/ui/manual_div_ceil.fixed index cd91be87ec17..8ffd107dd42e 100644 --- a/tests/ui/manual_div_ceil.fixed +++ b/tests/ui/manual_div_ceil.fixed @@ -105,3 +105,32 @@ fn issue_15705(size: u64, c: &u64) { let _ = size.div_ceil(*c); //~^ manual_div_ceil } + +struct MyStruct(u32); +impl MyStruct { + // Method matching name on different type should not trigger lint + fn next_multiple_of(self, y: u32) -> u32 { + self.0.next_multiple_of(y) + } +} + +fn issue_16219() { + let x = 33u32; + + // Lint. + let _ = x.div_ceil(8); + //~^ manual_div_ceil + let _ = x.div_ceil(8); + //~^ manual_div_ceil + + let y = &x; + let _ = y.div_ceil(8); + //~^ manual_div_ceil + + // No lint. + let _ = x.next_multiple_of(8) / 7; + let _ = x.next_multiple_of(7) / 8; + + let z = MyStruct(x); + let _ = z.next_multiple_of(8) / 8; +} diff --git a/tests/ui/manual_div_ceil.rs b/tests/ui/manual_div_ceil.rs index 9899c7d775c2..859fb5a13c44 100644 --- a/tests/ui/manual_div_ceil.rs +++ b/tests/ui/manual_div_ceil.rs @@ -105,3 +105,32 @@ fn issue_15705(size: u64, c: &u64) { let _ = (size + c - 1) / c; //~^ manual_div_ceil } + +struct MyStruct(u32); +impl MyStruct { + // Method matching name on different type should not trigger lint + fn next_multiple_of(self, y: u32) -> u32 { + self.0.next_multiple_of(y) + } +} + +fn issue_16219() { + let x = 33u32; + + // Lint. + let _ = x.next_multiple_of(8) / 8; + //~^ manual_div_ceil + let _ = u32::next_multiple_of(x, 8) / 8; + //~^ manual_div_ceil + + let y = &x; + let _ = y.next_multiple_of(8) / 8; + //~^ manual_div_ceil + + // No lint. + let _ = x.next_multiple_of(8) / 7; + let _ = x.next_multiple_of(7) / 8; + + let z = MyStruct(x); + let _ = z.next_multiple_of(8) / 8; +} diff --git a/tests/ui/manual_div_ceil.stderr b/tests/ui/manual_div_ceil.stderr index 44de3ba99be7..0efc114c7078 100644 --- a/tests/ui/manual_div_ceil.stderr +++ b/tests/ui/manual_div_ceil.stderr @@ -131,5 +131,23 @@ error: manually reimplementing `div_ceil` LL | let _ = (size + c - 1) / c; | ^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `size.div_ceil(*c)` -error: aborting due to 20 previous errors +error: manually reimplementing `div_ceil` + --> tests/ui/manual_div_ceil.rs:121:13 + | +LL | let _ = x.next_multiple_of(8) / 8; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `x.div_ceil(8)` + +error: manually reimplementing `div_ceil` + --> tests/ui/manual_div_ceil.rs:123:13 + | +LL | let _ = u32::next_multiple_of(x, 8) / 8; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `x.div_ceil(8)` + +error: manually reimplementing `div_ceil` + --> tests/ui/manual_div_ceil.rs:127:13 + | +LL | let _ = y.next_multiple_of(8) / 8; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `y.div_ceil(8)` + +error: aborting due to 23 previous errors diff --git a/tests/ui/manual_div_ceil_with_feature.fixed b/tests/ui/manual_div_ceil_with_feature.fixed index f55999c5ad08..d77b8bc28986 100644 --- a/tests/ui/manual_div_ceil_with_feature.fixed +++ b/tests/ui/manual_div_ceil_with_feature.fixed @@ -84,3 +84,32 @@ fn issue_13950() { let _ = y.div_ceil(-7); //~^ manual_div_ceil } + +struct MyStruct(i32); +impl MyStruct { + // Method matching name on different type should not trigger lint + fn next_multiple_of(self, y: i32) -> i32 { + self.0.next_multiple_of(y) + } +} + +fn issue_16219() { + let x = 33i32; + + // Lint. + let _ = x.div_ceil(8); + //~^ manual_div_ceil + let _ = x.div_ceil(8); + //~^ manual_div_ceil + + let y = &x; + let _ = y.div_ceil(8); + //~^ manual_div_ceil + + // No lint. + let _ = x.next_multiple_of(8) / 7; + let _ = x.next_multiple_of(7) / 8; + + let z = MyStruct(x); + let _ = z.next_multiple_of(8) / 8; +} diff --git a/tests/ui/manual_div_ceil_with_feature.rs b/tests/ui/manual_div_ceil_with_feature.rs index 8a895f634cb4..5b9a4d9156a7 100644 --- a/tests/ui/manual_div_ceil_with_feature.rs +++ b/tests/ui/manual_div_ceil_with_feature.rs @@ -84,3 +84,32 @@ fn issue_13950() { let _ = (y - 8) / -7; //~^ manual_div_ceil } + +struct MyStruct(i32); +impl MyStruct { + // Method matching name on different type should not trigger lint + fn next_multiple_of(self, y: i32) -> i32 { + self.0.next_multiple_of(y) + } +} + +fn issue_16219() { + let x = 33i32; + + // Lint. + let _ = x.next_multiple_of(8) / 8; + //~^ manual_div_ceil + let _ = i32::next_multiple_of(x, 8) / 8; + //~^ manual_div_ceil + + let y = &x; + let _ = y.next_multiple_of(8) / 8; + //~^ manual_div_ceil + + // No lint. + let _ = x.next_multiple_of(8) / 7; + let _ = x.next_multiple_of(7) / 8; + + let z = MyStruct(x); + let _ = z.next_multiple_of(8) / 8; +} diff --git a/tests/ui/manual_div_ceil_with_feature.stderr b/tests/ui/manual_div_ceil_with_feature.stderr index e1160d962996..c5fa15112a87 100644 --- a/tests/ui/manual_div_ceil_with_feature.stderr +++ b/tests/ui/manual_div_ceil_with_feature.stderr @@ -139,5 +139,23 @@ error: manually reimplementing `div_ceil` LL | let _ = (y - 8) / -7; | ^^^^^^^^^^^^ help: consider using `.div_ceil()`: `y.div_ceil(-7)` -error: aborting due to 23 previous errors +error: manually reimplementing `div_ceil` + --> tests/ui/manual_div_ceil_with_feature.rs:100:13 + | +LL | let _ = x.next_multiple_of(8) / 8; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `x.div_ceil(8)` + +error: manually reimplementing `div_ceil` + --> tests/ui/manual_div_ceil_with_feature.rs:102:13 + | +LL | let _ = i32::next_multiple_of(x, 8) / 8; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `x.div_ceil(8)` + +error: manually reimplementing `div_ceil` + --> tests/ui/manual_div_ceil_with_feature.rs:106:13 + | +LL | let _ = y.next_multiple_of(8) / 8; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `y.div_ceil(8)` + +error: aborting due to 26 previous errors diff --git a/tests/ui/manual_ignore_case_cmp.fixed b/tests/ui/manual_ignore_case_cmp.fixed index cd7adc20b127..f0e413aaec0d 100644 --- a/tests/ui/manual_ignore_case_cmp.fixed +++ b/tests/ui/manual_ignore_case_cmp.fixed @@ -160,3 +160,23 @@ fn ref_osstring(a: OsString, b: &OsString) { b.eq_ignore_ascii_case(a); //~^ manual_ignore_case_cmp } + +fn wrongly_unmangled_macros(a: &str, b: &str) -> bool { + struct S<'a> { + inner: &'a str, + } + + let a = S { inner: a }; + let b = S { inner: b }; + + macro_rules! dot_inner { + ($s:expr) => { + $s.inner + }; + } + + dot_inner!(a).eq_ignore_ascii_case(dot_inner!(b)) + //~^ manual_ignore_case_cmp + || dot_inner!(a).eq_ignore_ascii_case("abc") + //~^ manual_ignore_case_cmp +} diff --git a/tests/ui/manual_ignore_case_cmp.rs b/tests/ui/manual_ignore_case_cmp.rs index 85f6719827c9..9802e87cd233 100644 --- a/tests/ui/manual_ignore_case_cmp.rs +++ b/tests/ui/manual_ignore_case_cmp.rs @@ -160,3 +160,23 @@ fn ref_osstring(a: OsString, b: &OsString) { b.to_ascii_lowercase() == a.to_ascii_lowercase(); //~^ manual_ignore_case_cmp } + +fn wrongly_unmangled_macros(a: &str, b: &str) -> bool { + struct S<'a> { + inner: &'a str, + } + + let a = S { inner: a }; + let b = S { inner: b }; + + macro_rules! dot_inner { + ($s:expr) => { + $s.inner + }; + } + + dot_inner!(a).to_ascii_lowercase() == dot_inner!(b).to_ascii_lowercase() + //~^ manual_ignore_case_cmp + || dot_inner!(a).to_ascii_lowercase() == "abc" + //~^ manual_ignore_case_cmp +} diff --git a/tests/ui/manual_ignore_case_cmp.stderr b/tests/ui/manual_ignore_case_cmp.stderr index fa7fadd91076..2f698e076ed3 100644 --- a/tests/ui/manual_ignore_case_cmp.stderr +++ b/tests/ui/manual_ignore_case_cmp.stderr @@ -588,5 +588,29 @@ LL - b.to_ascii_lowercase() == a.to_ascii_lowercase(); LL + b.eq_ignore_ascii_case(a); | -error: aborting due to 49 previous errors +error: manual case-insensitive ASCII comparison + --> tests/ui/manual_ignore_case_cmp.rs:178:5 + | +LL | dot_inner!(a).to_ascii_lowercase() == dot_inner!(b).to_ascii_lowercase() + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: consider using `.eq_ignore_ascii_case()` instead + | +LL - dot_inner!(a).to_ascii_lowercase() == dot_inner!(b).to_ascii_lowercase() +LL + dot_inner!(a).eq_ignore_ascii_case(dot_inner!(b)) + | + +error: manual case-insensitive ASCII comparison + --> tests/ui/manual_ignore_case_cmp.rs:180:12 + | +LL | || dot_inner!(a).to_ascii_lowercase() == "abc" + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: consider using `.eq_ignore_ascii_case()` instead + | +LL - || dot_inner!(a).to_ascii_lowercase() == "abc" +LL + || dot_inner!(a).eq_ignore_ascii_case("abc") + | + +error: aborting due to 51 previous errors diff --git a/tests/ui/manual_ilog2.fixed b/tests/ui/manual_ilog2.fixed index a0f6d9392c30..ea86fc927c7c 100644 --- a/tests/ui/manual_ilog2.fixed +++ b/tests/ui/manual_ilog2.fixed @@ -30,3 +30,20 @@ fn foo(a: u32, b: u64) { external!($a.ilog(2)); with_span!(span; a.ilog(2)); } + +fn wrongly_unmangled_macros() { + struct S { + inner: u32, + } + + let x = S { inner: 42 }; + macro_rules! access { + ($s:expr) => { + $s.inner + }; + } + let log = access!(x).ilog2(); + //~^ manual_ilog2 + let log = access!(x).ilog2(); + //~^ manual_ilog2 +} diff --git a/tests/ui/manual_ilog2.rs b/tests/ui/manual_ilog2.rs index bd4b5d9d3c0d..8cb0e12d7361 100644 --- a/tests/ui/manual_ilog2.rs +++ b/tests/ui/manual_ilog2.rs @@ -30,3 +30,20 @@ fn foo(a: u32, b: u64) { external!($a.ilog(2)); with_span!(span; a.ilog(2)); } + +fn wrongly_unmangled_macros() { + struct S { + inner: u32, + } + + let x = S { inner: 42 }; + macro_rules! access { + ($s:expr) => { + $s.inner + }; + } + let log = 31 - access!(x).leading_zeros(); + //~^ manual_ilog2 + let log = access!(x).ilog(2); + //~^ manual_ilog2 +} diff --git a/tests/ui/manual_ilog2.stderr b/tests/ui/manual_ilog2.stderr index 7c9694f35330..d0ef8378081a 100644 --- a/tests/ui/manual_ilog2.stderr +++ b/tests/ui/manual_ilog2.stderr @@ -19,5 +19,17 @@ error: manually reimplementing `ilog2` LL | 63 - b.leading_zeros(); | ^^^^^^^^^^^^^^^^^^^^^^ help: try: `b.ilog2()` -error: aborting due to 3 previous errors +error: manually reimplementing `ilog2` + --> tests/ui/manual_ilog2.rs:45:15 + | +LL | let log = 31 - access!(x).leading_zeros(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `access!(x).ilog2()` + +error: manually reimplementing `ilog2` + --> tests/ui/manual_ilog2.rs:47:15 + | +LL | let log = access!(x).ilog(2); + | ^^^^^^^^^^^^^^^^^^ help: try: `access!(x).ilog2()` + +error: aborting due to 5 previous errors diff --git a/tests/ui/manual_is_multiple_of.fixed b/tests/ui/manual_is_multiple_of.fixed index 03f75e725ed5..82e0684e5e57 100644 --- a/tests/ui/manual_is_multiple_of.fixed +++ b/tests/ui/manual_is_multiple_of.fixed @@ -101,3 +101,19 @@ mod issue15103 { (1..1_000).filter(|&i| i == d(d(i)) && i != d(i)).sum() } } + +fn wrongly_unmangled_macros(a: u32, b: u32) { + struct Wrapper(u32); + let a = Wrapper(a); + let b = Wrapper(b); + macro_rules! dot_0 { + ($x:expr) => { + $x.0 + }; + } + + if dot_0!(a).is_multiple_of(dot_0!(b)) { + //~^ manual_is_multiple_of + todo!() + } +} diff --git a/tests/ui/manual_is_multiple_of.rs b/tests/ui/manual_is_multiple_of.rs index 7b6fa64c843d..82a492e24092 100644 --- a/tests/ui/manual_is_multiple_of.rs +++ b/tests/ui/manual_is_multiple_of.rs @@ -101,3 +101,19 @@ mod issue15103 { (1..1_000).filter(|&i| i == d(d(i)) && i != d(i)).sum() } } + +fn wrongly_unmangled_macros(a: u32, b: u32) { + struct Wrapper(u32); + let a = Wrapper(a); + let b = Wrapper(b); + macro_rules! dot_0 { + ($x:expr) => { + $x.0 + }; + } + + if dot_0!(a) % dot_0!(b) == 0 { + //~^ manual_is_multiple_of + todo!() + } +} diff --git a/tests/ui/manual_is_multiple_of.stderr b/tests/ui/manual_is_multiple_of.stderr index 8523599ec402..3aba869c9111 100644 --- a/tests/ui/manual_is_multiple_of.stderr +++ b/tests/ui/manual_is_multiple_of.stderr @@ -67,5 +67,11 @@ error: manual implementation of `.is_multiple_of()` LL | let d = |n: u32| -> u32 { (1..=n / 2).filter(|i| n % i == 0).sum() }; | ^^^^^^^^^^ help: replace with: `n.is_multiple_of(*i)` -error: aborting due to 11 previous errors +error: manual implementation of `.is_multiple_of()` + --> tests/ui/manual_is_multiple_of.rs:115:8 + | +LL | if dot_0!(a) % dot_0!(b) == 0 { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ help: replace with: `dot_0!(a).is_multiple_of(dot_0!(b))` + +error: aborting due to 12 previous errors diff --git a/tests/ui/manual_ok_err.fixed b/tests/ui/manual_ok_err.fixed index 9b70ce0df43a..e22f91a0155f 100644 --- a/tests/ui/manual_ok_err.fixed +++ b/tests/ui/manual_ok_err.fixed @@ -127,3 +127,13 @@ mod issue15051 { result_with_ref_mut(x).as_mut().ok() } } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + Ok::($val) + }; + } + + let _ = test_expr!(42).ok(); +} diff --git a/tests/ui/manual_ok_err.rs b/tests/ui/manual_ok_err.rs index dee904638245..c1355f0d4096 100644 --- a/tests/ui/manual_ok_err.rs +++ b/tests/ui/manual_ok_err.rs @@ -177,3 +177,17 @@ mod issue15051 { } } } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + Ok::($val) + }; + } + + let _ = match test_expr!(42) { + //~^ manual_ok_err + Ok(v) => Some(v), + Err(_) => None, + }; +} diff --git a/tests/ui/manual_ok_err.stderr b/tests/ui/manual_ok_err.stderr index 448fbffc0509..0c2ed368eb97 100644 --- a/tests/ui/manual_ok_err.stderr +++ b/tests/ui/manual_ok_err.stderr @@ -141,5 +141,16 @@ LL | | Err(_) => None, LL | | } | |_________^ help: replace with: `result_with_ref_mut(x).as_mut().ok()` -error: aborting due to 12 previous errors +error: manual implementation of `ok` + --> tests/ui/manual_ok_err.rs:188:13 + | +LL | let _ = match test_expr!(42) { + | _____________^ +LL | | +LL | | Ok(v) => Some(v), +LL | | Err(_) => None, +LL | | }; + | |_____^ help: replace with: `test_expr!(42).ok()` + +error: aborting due to 13 previous errors diff --git a/tests/ui/match_as_ref.fixed b/tests/ui/match_as_ref.fixed index 09a6ed169390..b1b8ffb885f5 100644 --- a/tests/ui/match_as_ref.fixed +++ b/tests/ui/match_as_ref.fixed @@ -90,3 +90,13 @@ fn issue15932() { let _: Option<&dyn std::fmt::Debug> = Some(0).as_ref().map(|x| x as _); } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + Some($val) + }; + } + + let _: Option<&u32> = test_expr!(42).as_ref(); +} diff --git a/tests/ui/match_as_ref.rs b/tests/ui/match_as_ref.rs index 347b6d186887..3113167957d4 100644 --- a/tests/ui/match_as_ref.rs +++ b/tests/ui/match_as_ref.rs @@ -114,3 +114,17 @@ fn issue15932() { Some(ref mut v) => Some(v), }; } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + Some($val) + }; + } + + let _: Option<&u32> = match test_expr!(42) { + //~^ match_as_ref + None => None, + Some(ref v) => Some(v), + }; +} diff --git a/tests/ui/match_as_ref.stderr b/tests/ui/match_as_ref.stderr index df06e358f296..3eab499fe409 100644 --- a/tests/ui/match_as_ref.stderr +++ b/tests/ui/match_as_ref.stderr @@ -127,5 +127,26 @@ LL - }; LL + let _: Option<&dyn std::fmt::Debug> = Some(0).as_ref().map(|x| x as _); | -error: aborting due to 6 previous errors +error: manual implementation of `Option::as_ref` + --> tests/ui/match_as_ref.rs:125:27 + | +LL | let _: Option<&u32> = match test_expr!(42) { + | ___________________________^ +LL | | +LL | | None => None, +LL | | Some(ref v) => Some(v), +LL | | }; + | |_____^ + | +help: use `Option::as_ref()` directly + | +LL - let _: Option<&u32> = match test_expr!(42) { +LL - +LL - None => None, +LL - Some(ref v) => Some(v), +LL - }; +LL + let _: Option<&u32> = test_expr!(42).as_ref(); + | + +error: aborting due to 7 previous errors diff --git a/tests/ui/match_bool.fixed b/tests/ui/match_bool.fixed index 876ae935afde..3d5d0a0d532c 100644 --- a/tests/ui/match_bool.fixed +++ b/tests/ui/match_bool.fixed @@ -74,4 +74,15 @@ fn issue15351() { } } +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + ($val + 1) > 0 + }; + } + + let x = 5; + if test_expr!(x) { 1 } else { 0 }; +} + fn main() {} diff --git a/tests/ui/match_bool.rs b/tests/ui/match_bool.rs index a134ad8346e2..4db0aedf3260 100644 --- a/tests/ui/match_bool.rs +++ b/tests/ui/match_bool.rs @@ -126,4 +126,19 @@ fn issue15351() { } } +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + ($val + 1) > 0 + }; + } + + let x = 5; + match test_expr!(x) { + //~^ match_bool + true => 1, + false => 0, + }; +} + fn main() {} diff --git a/tests/ui/match_bool.stderr b/tests/ui/match_bool.stderr index c05742e56339..223acd17aead 100644 --- a/tests/ui/match_bool.stderr +++ b/tests/ui/match_bool.stderr @@ -183,5 +183,15 @@ LL + break 'a; LL + } } | -error: aborting due to 13 previous errors +error: `match` on a boolean expression + --> tests/ui/match_bool.rs:137:5 + | +LL | / match test_expr!(x) { +LL | | +LL | | true => 1, +LL | | false => 0, +LL | | }; + | |_____^ help: consider using an `if`/`else` expression: `if test_expr!(x) { 1 } else { 0 }` + +error: aborting due to 14 previous errors diff --git a/tests/ui/mutex_atomic.fixed b/tests/ui/mutex_atomic.fixed index e4218726019f..dc05d8a2c61f 100644 --- a/tests/ui/mutex_atomic.fixed +++ b/tests/ui/mutex_atomic.fixed @@ -65,3 +65,15 @@ fn issue13378() { let (funky_mtx) = std::sync::atomic::AtomicU64::new(0); //~^ mutex_integer } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + ($val > 0 && true) + }; + } + + let _ = std::sync::atomic::AtomicBool::new(test_expr!(1)); + //~^ mutex_atomic + // The suggestion should preserve the macro call: `AtomicBool::new(test_expr!(true))` +} diff --git a/tests/ui/mutex_atomic.rs b/tests/ui/mutex_atomic.rs index 95f2b135903f..33745f8fc5e1 100644 --- a/tests/ui/mutex_atomic.rs +++ b/tests/ui/mutex_atomic.rs @@ -65,3 +65,15 @@ fn issue13378() { let (funky_mtx): Mutex = Mutex::new(0); //~^ mutex_integer } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + ($val > 0 && true) + }; + } + + let _ = Mutex::new(test_expr!(1)); + //~^ mutex_atomic + // The suggestion should preserve the macro call: `AtomicBool::new(test_expr!(true))` +} diff --git a/tests/ui/mutex_atomic.stderr b/tests/ui/mutex_atomic.stderr index 0afc6d541dea..56d94035583c 100644 --- a/tests/ui/mutex_atomic.stderr +++ b/tests/ui/mutex_atomic.stderr @@ -130,5 +130,13 @@ LL - let (funky_mtx): Mutex = Mutex::new(0); LL + let (funky_mtx) = std::sync::atomic::AtomicU64::new(0); | -error: aborting due to 14 previous errors +error: using a `Mutex` where an atomic would do + --> tests/ui/mutex_atomic.rs:76:13 + | +LL | let _ = Mutex::new(test_expr!(1)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `std::sync::atomic::AtomicBool::new(test_expr!(1))` + | + = help: if you just want the locking behavior and not the internal type, consider using `Mutex<()>` + +error: aborting due to 15 previous errors diff --git a/tests/ui/needless_bool_assign.fixed b/tests/ui/needless_bool_assign.fixed index d6fab4c51b53..8fd572038140 100644 --- a/tests/ui/needless_bool_assign.fixed +++ b/tests/ui/needless_bool_assign.fixed @@ -42,3 +42,22 @@ fn issue15063(x: bool, y: bool) { } else { z = x || y; } //~^^^^^ needless_bool_assign } + +fn wrongly_unmangled_macros(must_keep: fn(usize, usize) -> bool, x: usize, y: usize) { + struct Wrapper(T); + let mut skip = Wrapper(false); + + macro_rules! invoke { + ($func:expr, $a:expr, $b:expr) => { + $func($a, $b) + }; + } + macro_rules! dot_0 { + ($w:expr) => { + $w.0 + }; + } + + dot_0!(skip) = !invoke!(must_keep, x, y); + //~^^^^^ needless_bool_assign +} diff --git a/tests/ui/needless_bool_assign.rs b/tests/ui/needless_bool_assign.rs index c504f61f4dd1..4721ab433b32 100644 --- a/tests/ui/needless_bool_assign.rs +++ b/tests/ui/needless_bool_assign.rs @@ -58,3 +58,26 @@ fn issue15063(x: bool, y: bool) { } //~^^^^^ needless_bool_assign } + +fn wrongly_unmangled_macros(must_keep: fn(usize, usize) -> bool, x: usize, y: usize) { + struct Wrapper(T); + let mut skip = Wrapper(false); + + macro_rules! invoke { + ($func:expr, $a:expr, $b:expr) => { + $func($a, $b) + }; + } + macro_rules! dot_0 { + ($w:expr) => { + $w.0 + }; + } + + if invoke!(must_keep, x, y) { + dot_0!(skip) = false; + } else { + dot_0!(skip) = true; + } + //~^^^^^ needless_bool_assign +} diff --git a/tests/ui/needless_bool_assign.stderr b/tests/ui/needless_bool_assign.stderr index 1d09b8b25a09..34ff782f34a3 100644 --- a/tests/ui/needless_bool_assign.stderr +++ b/tests/ui/needless_bool_assign.stderr @@ -62,5 +62,15 @@ LL | | z = false; LL | | } | |_____^ help: you can reduce it to: `{ z = x || y; }` -error: aborting due to 5 previous errors +error: this if-then-else expression assigns a bool literal + --> tests/ui/needless_bool_assign.rs:77:5 + | +LL | / if invoke!(must_keep, x, y) { +LL | | dot_0!(skip) = false; +LL | | } else { +LL | | dot_0!(skip) = true; +LL | | } + | |_____^ help: you can reduce it to: `dot_0!(skip) = !invoke!(must_keep, x, y);` + +error: aborting due to 6 previous errors diff --git a/tests/ui/needless_for_each_fixable.fixed b/tests/ui/needless_for_each_fixable.fixed index a6d64d9afc1a..19b34f42af24 100644 --- a/tests/ui/needless_for_each_fixable.fixed +++ b/tests/ui/needless_for_each_fixable.fixed @@ -149,3 +149,11 @@ fn issue15256() { for v in vec.iter() { println!("{v}"); } //~^ needless_for_each } + +fn issue16294() { + let vec: Vec = Vec::new(); + for elem in vec.iter() { + //~^ needless_for_each + println!("{elem}"); + } +} diff --git a/tests/ui/needless_for_each_fixable.rs b/tests/ui/needless_for_each_fixable.rs index 7e74d2b428fd..f04e2555a370 100644 --- a/tests/ui/needless_for_each_fixable.rs +++ b/tests/ui/needless_for_each_fixable.rs @@ -149,3 +149,11 @@ fn issue15256() { vec.iter().for_each(|v| println!("{v}")); //~^ needless_for_each } + +fn issue16294() { + let vec: Vec = Vec::new(); + vec.iter().for_each(|elem| { + //~^ needless_for_each + println!("{elem}"); + }) +} diff --git a/tests/ui/needless_for_each_fixable.stderr b/tests/ui/needless_for_each_fixable.stderr index 204cfa36b022..121669d15072 100644 --- a/tests/ui/needless_for_each_fixable.stderr +++ b/tests/ui/needless_for_each_fixable.stderr @@ -154,5 +154,22 @@ error: needless use of `for_each` LL | vec.iter().for_each(|v| println!("{v}")); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `for v in vec.iter() { println!("{v}"); }` -error: aborting due to 11 previous errors +error: needless use of `for_each` + --> tests/ui/needless_for_each_fixable.rs:155:5 + | +LL | / vec.iter().for_each(|elem| { +LL | | +LL | | println!("{elem}"); +LL | | }) + | |______^ + | +help: try + | +LL ~ for elem in vec.iter() { +LL + +LL + println!("{elem}"); +LL + } + | + +error: aborting due to 12 previous errors diff --git a/tests/ui/never_loop_iterator_reduction.rs b/tests/ui/never_loop_iterator_reduction.rs index 6b07b91db29a..27f1766b841d 100644 --- a/tests/ui/never_loop_iterator_reduction.rs +++ b/tests/ui/never_loop_iterator_reduction.rs @@ -1,8 +1,9 @@ //@no-rustfix #![warn(clippy::never_loop)] +#![expect(clippy::needless_return)] fn main() { - // diverging closure: should trigger + // diverging closure with no `return`: should trigger [0, 1].into_iter().for_each(|x| { //~^ never_loop @@ -14,4 +15,10 @@ fn main() { [0, 1].into_iter().for_each(|x| { let _ = x + 1; }); + + // `return` should NOT trigger even though it is diverging + [0, 1].into_iter().for_each(|x| { + println!("x = {x}"); + return; + }); } diff --git a/tests/ui/never_loop_iterator_reduction.stderr b/tests/ui/never_loop_iterator_reduction.stderr index b76ee283146c..92483c85432d 100644 --- a/tests/ui/never_loop_iterator_reduction.stderr +++ b/tests/ui/never_loop_iterator_reduction.stderr @@ -1,5 +1,5 @@ error: this iterator reduction never loops (closure always diverges) - --> tests/ui/never_loop_iterator_reduction.rs:6:5 + --> tests/ui/never_loop_iterator_reduction.rs:7:5 | LL | / [0, 1].into_iter().for_each(|x| { LL | | diff --git a/tests/ui/new_without_default.fixed b/tests/ui/new_without_default.fixed index 9a5e90b48065..f6591820feeb 100644 --- a/tests/ui/new_without_default.fixed +++ b/tests/ui/new_without_default.fixed @@ -409,3 +409,58 @@ mod issue15778 { } } } + +pub mod issue16255 { + use std::fmt::Display; + use std::marker::PhantomData; + + pub struct Foo { + marker: PhantomData, + } + + impl Default for Foo + where + T: Display, + T: Clone, + { + fn default() -> Self { + Self::new() + } + } + + impl Foo + where + T: Display, + { + pub fn new() -> Self + //~^ new_without_default + where + T: Clone, + { + Self { marker: PhantomData } + } + } + + pub struct Bar { + marker: PhantomData, + } + + impl Default for Bar + where + T: Clone, + { + fn default() -> Self { + Self::new() + } + } + + impl Bar { + pub fn new() -> Self + //~^ new_without_default + where + T: Clone, + { + Self { marker: PhantomData } + } + } +} diff --git a/tests/ui/new_without_default.rs b/tests/ui/new_without_default.rs index f7466aa32189..d3447f2e16b2 100644 --- a/tests/ui/new_without_default.rs +++ b/tests/ui/new_without_default.rs @@ -324,3 +324,39 @@ mod issue15778 { } } } + +pub mod issue16255 { + use std::fmt::Display; + use std::marker::PhantomData; + + pub struct Foo { + marker: PhantomData, + } + + impl Foo + where + T: Display, + { + pub fn new() -> Self + //~^ new_without_default + where + T: Clone, + { + Self { marker: PhantomData } + } + } + + pub struct Bar { + marker: PhantomData, + } + + impl Bar { + pub fn new() -> Self + //~^ new_without_default + where + T: Clone, + { + Self { marker: PhantomData } + } + } +} diff --git a/tests/ui/new_without_default.stderr b/tests/ui/new_without_default.stderr index 0593dbb00fb6..6c0f73d13185 100644 --- a/tests/ui/new_without_default.stderr +++ b/tests/ui/new_without_default.stderr @@ -250,5 +250,58 @@ LL + } LL + } | -error: aborting due to 13 previous errors +error: you should consider adding a `Default` implementation for `Foo` + --> tests/ui/new_without_default.rs:340:9 + | +LL | / pub fn new() -> Self +LL | | +LL | | where +LL | | T: Clone, +LL | | { +LL | | Self { marker: PhantomData } +LL | | } + | |_________^ + | +help: try adding this + | +LL ~ impl Default for Foo +LL + where +LL + T: Display, +LL + T: Clone, +LL + { +LL + fn default() -> Self { +LL + Self::new() +LL + } +LL + } +LL + +LL ~ impl Foo + | + +error: you should consider adding a `Default` implementation for `Bar` + --> tests/ui/new_without_default.rs:354:9 + | +LL | / pub fn new() -> Self +LL | | +LL | | where +LL | | T: Clone, +LL | | { +LL | | Self { marker: PhantomData } +LL | | } + | |_________^ + | +help: try adding this + | +LL ~ impl Default for Bar +LL + where +LL + T: Clone, +LL + { +LL + fn default() -> Self { +LL + Self::new() +LL + } +LL + } +LL + +LL ~ impl Bar { + | + +error: aborting due to 15 previous errors diff --git a/tests/ui/question_mark.fixed b/tests/ui/question_mark.fixed index 2c5ee0245038..b8072932c4ea 100644 --- a/tests/ui/question_mark.fixed +++ b/tests/ui/question_mark.fixed @@ -1,5 +1,5 @@ #![feature(try_blocks)] -#![allow(clippy::unnecessary_wraps)] +#![allow(clippy::unnecessary_wraps, clippy::no_effect)] use std::sync::MutexGuard; @@ -500,3 +500,18 @@ mod issue14894 { Ok(()) } } + +fn wrongly_unmangled_macros() -> Option { + macro_rules! test_expr { + ($val:expr) => { + Some($val) + }; + } + + let x = test_expr!(42)?; + //~^^^ question_mark + Some(x); + + test_expr!(42)?; + test_expr!(42) +} diff --git a/tests/ui/question_mark.rs b/tests/ui/question_mark.rs index b9ff9d1565b2..b320dcd4b0bc 100644 --- a/tests/ui/question_mark.rs +++ b/tests/ui/question_mark.rs @@ -1,5 +1,5 @@ #![feature(try_blocks)] -#![allow(clippy::unnecessary_wraps)] +#![allow(clippy::unnecessary_wraps, clippy::no_effect)] use std::sync::MutexGuard; @@ -615,3 +615,23 @@ mod issue14894 { Ok(()) } } + +fn wrongly_unmangled_macros() -> Option { + macro_rules! test_expr { + ($val:expr) => { + Some($val) + }; + } + + let Some(x) = test_expr!(42) else { + return None; + }; + //~^^^ question_mark + Some(x); + + if test_expr!(42).is_none() { + //~^ question_mark + return None; + } + test_expr!(42) +} diff --git a/tests/ui/question_mark.stderr b/tests/ui/question_mark.stderr index 9b2896328e66..d645c8830adc 100644 --- a/tests/ui/question_mark.stderr +++ b/tests/ui/question_mark.stderr @@ -333,5 +333,22 @@ LL | | return Err(reason); LL | | } | |_________^ help: replace it with: `result?;` -error: aborting due to 35 previous errors +error: this `let...else` may be rewritten with the `?` operator + --> tests/ui/question_mark.rs:626:5 + | +LL | / let Some(x) = test_expr!(42) else { +LL | | return None; +LL | | }; + | |______^ help: replace it with: `let x = test_expr!(42)?;` + +error: this block may be rewritten with the `?` operator + --> tests/ui/question_mark.rs:632:5 + | +LL | / if test_expr!(42).is_none() { +LL | | +LL | | return None; +LL | | } + | |_____^ help: replace it with: `test_expr!(42)?;` + +error: aborting due to 37 previous errors diff --git a/tests/ui/redundant_pattern_matching_option.fixed b/tests/ui/redundant_pattern_matching_option.fixed index 08903ef7fdda..b44009446640 100644 --- a/tests/ui/redundant_pattern_matching_option.fixed +++ b/tests/ui/redundant_pattern_matching_option.fixed @@ -195,3 +195,16 @@ fn issue16045() { } } } + +fn issue14989() { + macro_rules! x { + () => { + None:: + }; + } + + if x! {}.is_some() {}; + //~^ redundant_pattern_matching + while x! {}.is_some() {} + //~^ redundant_pattern_matching +} diff --git a/tests/ui/redundant_pattern_matching_option.rs b/tests/ui/redundant_pattern_matching_option.rs index 95eff3f9ebf9..c13cf993e786 100644 --- a/tests/ui/redundant_pattern_matching_option.rs +++ b/tests/ui/redundant_pattern_matching_option.rs @@ -231,3 +231,16 @@ fn issue16045() { } } } + +fn issue14989() { + macro_rules! x { + () => { + None:: + }; + } + + if let Some(_) = (x! {}) {}; + //~^ redundant_pattern_matching + while let Some(_) = (x! {}) {} + //~^ redundant_pattern_matching +} diff --git a/tests/ui/redundant_pattern_matching_option.stderr b/tests/ui/redundant_pattern_matching_option.stderr index 6fd0c5a6f859..5c9edfd4c50a 100644 --- a/tests/ui/redundant_pattern_matching_option.stderr +++ b/tests/ui/redundant_pattern_matching_option.stderr @@ -236,5 +236,17 @@ error: redundant pattern matching, consider using `is_some()` LL | if let Some(_) = x.await { | -------^^^^^^^---------- help: try: `if x.await.is_some()` -error: aborting due to 33 previous errors +error: redundant pattern matching, consider using `is_some()` + --> tests/ui/redundant_pattern_matching_option.rs:242:12 + | +LL | if let Some(_) = (x! {}) {}; + | -------^^^^^^^---------- help: try: `if x! {}.is_some()` + +error: redundant pattern matching, consider using `is_some()` + --> tests/ui/redundant_pattern_matching_option.rs:244:15 + | +LL | while let Some(_) = (x! {}) {} + | ----------^^^^^^^---------- help: try: `while x! {}.is_some()` + +error: aborting due to 35 previous errors diff --git a/tests/ui/redundant_pattern_matching_result.fixed b/tests/ui/redundant_pattern_matching_result.fixed index 261d82fc35c8..8754d71aa629 100644 --- a/tests/ui/redundant_pattern_matching_result.fixed +++ b/tests/ui/redundant_pattern_matching_result.fixed @@ -165,3 +165,23 @@ fn issue10803() { // Don't lint let _ = matches!(x, Err(16)); } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + Ok::($val) + }; + } + + let _ = test_expr!(42).is_ok(); + + macro_rules! test_guard { + ($val:expr) => { + ($val + 1) > 0 + }; + } + + let x: Result = Ok(42); + let _ = x.is_ok() && test_guard!(42); + //~^ redundant_pattern_matching +} diff --git a/tests/ui/redundant_pattern_matching_result.rs b/tests/ui/redundant_pattern_matching_result.rs index 6cae4cc4b6b0..b83b02588fb6 100644 --- a/tests/ui/redundant_pattern_matching_result.rs +++ b/tests/ui/redundant_pattern_matching_result.rs @@ -205,3 +205,27 @@ fn issue10803() { // Don't lint let _ = matches!(x, Err(16)); } + +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($val:expr) => { + Ok::($val) + }; + } + + let _ = match test_expr!(42) { + //~^ redundant_pattern_matching + Ok(_) => true, + Err(_) => false, + }; + + macro_rules! test_guard { + ($val:expr) => { + ($val + 1) > 0 + }; + } + + let x: Result = Ok(42); + let _ = matches!(x, Ok(_) if test_guard!(42)); + //~^ redundant_pattern_matching +} diff --git a/tests/ui/redundant_pattern_matching_result.stderr b/tests/ui/redundant_pattern_matching_result.stderr index 7e7d27d07a7f..dda203b753c3 100644 --- a/tests/ui/redundant_pattern_matching_result.stderr +++ b/tests/ui/redundant_pattern_matching_result.stderr @@ -209,5 +209,22 @@ error: redundant pattern matching, consider using `is_err()` LL | let _ = matches!(x, Err(_)); | ^^^^^^^^^^^^^^^^^^^ help: try: `x.is_err()` -error: aborting due to 28 previous errors +error: redundant pattern matching, consider using `is_ok()` + --> tests/ui/redundant_pattern_matching_result.rs:216:13 + | +LL | let _ = match test_expr!(42) { + | _____________^ +LL | | +LL | | Ok(_) => true, +LL | | Err(_) => false, +LL | | }; + | |_____^ help: try: `test_expr!(42).is_ok()` + +error: redundant pattern matching, consider using `is_ok()` + --> tests/ui/redundant_pattern_matching_result.rs:229:13 + | +LL | let _ = matches!(x, Ok(_) if test_guard!(42)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `x.is_ok() && test_guard!(42)` + +error: aborting due to 30 previous errors diff --git a/tests/ui/single_range_in_vec_init_unfixable.rs b/tests/ui/single_range_in_vec_init_unfixable.rs new file mode 100644 index 000000000000..33378b386f34 --- /dev/null +++ b/tests/ui/single_range_in_vec_init_unfixable.rs @@ -0,0 +1,12 @@ +//@no-rustfix +#![warn(clippy::single_range_in_vec_init)] + +use std::ops::Range; + +fn issue16306(v: &[i32]) { + fn takes_range_slice(_: &[Range]) {} + + let len = v.len(); + takes_range_slice(&[0..len as i64]); + //~^ single_range_in_vec_init +} diff --git a/tests/ui/single_range_in_vec_init_unfixable.stderr b/tests/ui/single_range_in_vec_init_unfixable.stderr new file mode 100644 index 000000000000..b10af21ed21c --- /dev/null +++ b/tests/ui/single_range_in_vec_init_unfixable.stderr @@ -0,0 +1,16 @@ +error: an array of `Range` that is only one element + --> tests/ui/single_range_in_vec_init_unfixable.rs:10:24 + | +LL | takes_range_slice(&[0..len as i64]); + | ^^^^^^^^^^^^^^^ + | + = note: `-D clippy::single-range-in-vec-init` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::single_range_in_vec_init)]` +help: if you wanted a `Vec` that contains the entire range, try + | +LL - takes_range_slice(&[0..len as i64]); +LL + takes_range_slice(&(0..len as i64).collect::>()); + | + +error: aborting due to 1 previous error + diff --git a/tests/ui/str_to_string.fixed b/tests/ui/str_to_string.fixed index 2941c4dbd33d..8713c4f9bc86 100644 --- a/tests/ui/str_to_string.fixed +++ b/tests/ui/str_to_string.fixed @@ -8,3 +8,17 @@ fn main() { msg.to_owned(); //~^ str_to_string } + +fn issue16271(key: &[u8]) { + macro_rules! t { + ($e:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + } + }; + } + + let _value = t!(str::from_utf8(key)).to_owned(); + //~^ str_to_string +} diff --git a/tests/ui/str_to_string.rs b/tests/ui/str_to_string.rs index 4c4d2bb18062..b81759e1037b 100644 --- a/tests/ui/str_to_string.rs +++ b/tests/ui/str_to_string.rs @@ -8,3 +8,17 @@ fn main() { msg.to_string(); //~^ str_to_string } + +fn issue16271(key: &[u8]) { + macro_rules! t { + ($e:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + } + }; + } + + let _value = t!(str::from_utf8(key)).to_string(); + //~^ str_to_string +} diff --git a/tests/ui/str_to_string.stderr b/tests/ui/str_to_string.stderr index cb7b6b48843a..c0a38c8ebe46 100644 --- a/tests/ui/str_to_string.stderr +++ b/tests/ui/str_to_string.stderr @@ -13,5 +13,11 @@ error: `to_string()` called on a `&str` LL | msg.to_string(); | ^^^^^^^^^^^^^^^ help: try: `msg.to_owned()` -error: aborting due to 2 previous errors +error: `to_string()` called on a `&str` + --> tests/ui/str_to_string.rs:22:18 + | +LL | let _value = t!(str::from_utf8(key)).to_string(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `t!(str::from_utf8(key)).to_owned()` + +error: aborting due to 3 previous errors diff --git a/tests/ui/string_from_utf8_as_bytes.fixed b/tests/ui/string_from_utf8_as_bytes.fixed index 193217114d88..98fa3a4fcf70 100644 --- a/tests/ui/string_from_utf8_as_bytes.fixed +++ b/tests/ui/string_from_utf8_as_bytes.fixed @@ -1,6 +1,16 @@ #![warn(clippy::string_from_utf8_as_bytes)] +macro_rules! test_range { + ($start:expr, $end:expr) => { + $start..$end + }; +} + fn main() { let _ = Some(&"Hello World!"[6..11]); //~^ string_from_utf8_as_bytes + + let s = "Hello World!"; + let _ = Some(&s[test_range!(6, 11)]); + //~^ string_from_utf8_as_bytes } diff --git a/tests/ui/string_from_utf8_as_bytes.rs b/tests/ui/string_from_utf8_as_bytes.rs index 49beb19ee40f..6354d5376ad6 100644 --- a/tests/ui/string_from_utf8_as_bytes.rs +++ b/tests/ui/string_from_utf8_as_bytes.rs @@ -1,6 +1,16 @@ #![warn(clippy::string_from_utf8_as_bytes)] +macro_rules! test_range { + ($start:expr, $end:expr) => { + $start..$end + }; +} + fn main() { let _ = std::str::from_utf8(&"Hello World!".as_bytes()[6..11]); //~^ string_from_utf8_as_bytes + + let s = "Hello World!"; + let _ = std::str::from_utf8(&s.as_bytes()[test_range!(6, 11)]); + //~^ string_from_utf8_as_bytes } diff --git a/tests/ui/string_from_utf8_as_bytes.stderr b/tests/ui/string_from_utf8_as_bytes.stderr index 99c8d8ae4eab..bba9ec0caf19 100644 --- a/tests/ui/string_from_utf8_as_bytes.stderr +++ b/tests/ui/string_from_utf8_as_bytes.stderr @@ -1,5 +1,5 @@ error: calling a slice of `as_bytes()` with `from_utf8` should be not necessary - --> tests/ui/string_from_utf8_as_bytes.rs:4:13 + --> tests/ui/string_from_utf8_as_bytes.rs:10:13 | LL | let _ = std::str::from_utf8(&"Hello World!".as_bytes()[6..11]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `Some(&"Hello World!"[6..11])` @@ -7,5 +7,11 @@ LL | let _ = std::str::from_utf8(&"Hello World!".as_bytes()[6..11]); = note: `-D clippy::string-from-utf8-as-bytes` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::string_from_utf8_as_bytes)]` -error: aborting due to 1 previous error +error: calling a slice of `as_bytes()` with `from_utf8` should be not necessary + --> tests/ui/string_from_utf8_as_bytes.rs:14:13 + | +LL | let _ = std::str::from_utf8(&s.as_bytes()[test_range!(6, 11)]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `Some(&s[test_range!(6, 11)])` + +error: aborting due to 2 previous errors diff --git a/tests/ui/transmuting_null.rs b/tests/ui/transmuting_null.rs index 0d3b26673452..00aa35dff803 100644 --- a/tests/ui/transmuting_null.rs +++ b/tests/ui/transmuting_null.rs @@ -37,8 +37,19 @@ fn transmute_const_int() { } } +fn transumute_single_expr_blocks() { + unsafe { + let _: &u64 = std::mem::transmute({ 0 as *const u64 }); + //~^ transmuting_null + + let _: &u64 = std::mem::transmute(const { u64::MIN as *const u64 }); + //~^ transmuting_null + } +} + fn main() { one_liners(); transmute_const(); transmute_const_int(); + transumute_single_expr_blocks(); } diff --git a/tests/ui/transmuting_null.stderr b/tests/ui/transmuting_null.stderr index ed7c3396a243..e1de391813bd 100644 --- a/tests/ui/transmuting_null.stderr +++ b/tests/ui/transmuting_null.stderr @@ -25,5 +25,17 @@ error: transmuting a known null pointer into a reference LL | let _: &u64 = std::mem::transmute(u64::MIN as *const u64); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error: aborting due to 4 previous errors +error: transmuting a known null pointer into a reference + --> tests/ui/transmuting_null.rs:42:23 + | +LL | let _: &u64 = std::mem::transmute({ 0 as *const u64 }); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: transmuting a known null pointer into a reference + --> tests/ui/transmuting_null.rs:45:23 + | +LL | let _: &u64 = std::mem::transmute(const { u64::MIN as *const u64 }); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 6 previous errors diff --git a/tests/ui/unnecessary_fold.fixed b/tests/ui/unnecessary_fold.fixed index c3eeafbc39cd..d51359349cb9 100644 --- a/tests/ui/unnecessary_fold.fixed +++ b/tests/ui/unnecessary_fold.fixed @@ -178,4 +178,15 @@ fn issue10000() { } } +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($e:expr) => { + ($e + 1) > 2 + }; + } + + let _ = (0..3).any(|x| test_expr!(x)); + //~^ unnecessary_fold +} + fn main() {} diff --git a/tests/ui/unnecessary_fold.rs b/tests/ui/unnecessary_fold.rs index 6ab41a942625..c6eb7157ab12 100644 --- a/tests/ui/unnecessary_fold.rs +++ b/tests/ui/unnecessary_fold.rs @@ -178,4 +178,15 @@ fn issue10000() { } } +fn wrongly_unmangled_macros() { + macro_rules! test_expr { + ($e:expr) => { + ($e + 1) > 2 + }; + } + + let _ = (0..3).fold(false, |acc: bool, x| acc || test_expr!(x)); + //~^ unnecessary_fold +} + fn main() {} diff --git a/tests/ui/unnecessary_fold.stderr b/tests/ui/unnecessary_fold.stderr index bb8aa7e18d34..560427a681a9 100644 --- a/tests/ui/unnecessary_fold.stderr +++ b/tests/ui/unnecessary_fold.stderr @@ -4,6 +4,7 @@ error: this `.fold` can be written more succinctly using another method LL | let _ = (0..3).fold(false, |acc, x| acc || x > 2); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `any(|x| x > 2)` | + = note: the `any` method is short circuiting and may change the program semantics if the iterator has side effects = note: `-D clippy::unnecessary-fold` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::unnecessary_fold)]` @@ -21,6 +22,8 @@ error: this `.fold` can be written more succinctly using another method | LL | let _ = (0..3).fold(true, |acc, x| acc && x > 2); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `all(|x| x > 2)` + | + = note: the `all` method is short circuiting and may change the program semantics if the iterator has side effects error: this `.fold` can be written more succinctly using another method --> tests/ui/unnecessary_fold.rs:24:25 @@ -63,12 +66,16 @@ error: this `.fold` can be written more succinctly using another method | LL | let _: bool = (0..3).map(|x| 2 * x).fold(false, |acc, x| acc || x > 2); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `any(|x| x > 2)` + | + = note: the `any` method is short circuiting and may change the program semantics if the iterator has side effects error: this `.fold` can be written more succinctly using another method --> tests/ui/unnecessary_fold.rs:110:10 | LL | .fold(false, |acc, x| acc || x > 2); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `any(|x| x > 2)` + | + = note: the `any` method is short circuiting and may change the program semantics if the iterator has side effects error: this `.fold` can be written more succinctly using another method --> tests/ui/unnecessary_fold.rs:123:33 @@ -196,5 +203,13 @@ error: this `.fold` can be written more succinctly using another method LL | (0..3).fold(1, |acc, x| acc * x) | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `product::()` -error: aborting due to 32 previous errors +error: this `.fold` can be written more succinctly using another method + --> tests/ui/unnecessary_fold.rs:188:20 + | +LL | let _ = (0..3).fold(false, |acc: bool, x| acc || test_expr!(x)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `any(|x| test_expr!(x))` + | + = note: the `any` method is short circuiting and may change the program semantics if the iterator has side effects + +error: aborting due to 33 previous errors diff --git a/tests/ui/unnecessary_to_owned.fixed b/tests/ui/unnecessary_to_owned.fixed index 316eac0b58b7..590359bc1ad2 100644 --- a/tests/ui/unnecessary_to_owned.fixed +++ b/tests/ui/unnecessary_to_owned.fixed @@ -681,3 +681,12 @@ fn issue14833() { let mut s = HashSet::<&String>::new(); s.remove(&"hello".to_owned()); } + +#[allow(clippy::redundant_clone)] +fn issue16351() { + fn take(_: impl AsRef) {} + + let dot = "."; + take(&format!("ouch{dot}")); + //~^ unnecessary_to_owned +} diff --git a/tests/ui/unnecessary_to_owned.rs b/tests/ui/unnecessary_to_owned.rs index f2dbd1db3c9f..d1e3e6497c0a 100644 --- a/tests/ui/unnecessary_to_owned.rs +++ b/tests/ui/unnecessary_to_owned.rs @@ -681,3 +681,12 @@ fn issue14833() { let mut s = HashSet::<&String>::new(); s.remove(&"hello".to_owned()); } + +#[allow(clippy::redundant_clone)] +fn issue16351() { + fn take(_: impl AsRef) {} + + let dot = "."; + take(format!("ouch{dot}").to_string()); + //~^ unnecessary_to_owned +} diff --git a/tests/ui/unnecessary_to_owned.stderr b/tests/ui/unnecessary_to_owned.stderr index 6c52be839301..50e3d5eb2195 100644 --- a/tests/ui/unnecessary_to_owned.stderr +++ b/tests/ui/unnecessary_to_owned.stderr @@ -550,5 +550,11 @@ error: unnecessary use of `to_vec` LL | s.remove(&(&["b"]).to_vec()); | ^^^^^^^^^^^^^^^^^^ help: replace it with: `(&["b"]).as_slice()` -error: aborting due to 82 previous errors +error: unnecessary use of `to_string` + --> tests/ui/unnecessary_to_owned.rs:690:10 + | +LL | take(format!("ouch{dot}").to_string()); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `&format!("ouch{dot}")` + +error: aborting due to 83 previous errors diff --git a/tests/ui/useless_conversion.fixed b/tests/ui/useless_conversion.fixed index adf5e58d9a1a..4832e922fa8e 100644 --- a/tests/ui/useless_conversion.fixed +++ b/tests/ui/useless_conversion.fixed @@ -1,4 +1,5 @@ #![deny(clippy::useless_conversion)] +#![allow(clippy::into_iter_on_ref)] #![allow(clippy::needless_ifs, clippy::unnecessary_wraps, unused)] // FIXME(static_mut_refs): Do not allow `static_mut_refs` lint #![allow(static_mut_refs)] @@ -131,6 +132,15 @@ fn main() { dont_lint_into_iter_on_copy_iter(); dont_lint_into_iter_on_static_copy_iter(); + { + // triggers the IntoIterator trait + fn consume(_: impl IntoIterator) {} + + // Should suggest `*items` instead of `&**items` + let items = &&[1, 2, 3]; + consume(*items); //~ useless_conversion + } + let _: String = "foo".into(); let _: String = From::from("foo"); let _ = String::from("foo"); diff --git a/tests/ui/useless_conversion.rs b/tests/ui/useless_conversion.rs index d95fe49e2e2b..6ef1f93a5606 100644 --- a/tests/ui/useless_conversion.rs +++ b/tests/ui/useless_conversion.rs @@ -1,4 +1,5 @@ #![deny(clippy::useless_conversion)] +#![allow(clippy::into_iter_on_ref)] #![allow(clippy::needless_ifs, clippy::unnecessary_wraps, unused)] // FIXME(static_mut_refs): Do not allow `static_mut_refs` lint #![allow(static_mut_refs)] @@ -131,6 +132,15 @@ fn main() { dont_lint_into_iter_on_copy_iter(); dont_lint_into_iter_on_static_copy_iter(); + { + // triggers the IntoIterator trait + fn consume(_: impl IntoIterator) {} + + // Should suggest `*items` instead of `&**items` + let items = &&[1, 2, 3]; + consume(items.into_iter()); //~ useless_conversion + } + let _: String = "foo".into(); let _: String = From::from("foo"); let _ = String::from("foo"); diff --git a/tests/ui/useless_conversion.stderr b/tests/ui/useless_conversion.stderr index 052c664f6f2e..d28b7a5cbfb6 100644 --- a/tests/ui/useless_conversion.stderr +++ b/tests/ui/useless_conversion.stderr @@ -1,5 +1,5 @@ error: useless conversion to the same type: `T` - --> tests/ui/useless_conversion.rs:9:13 + --> tests/ui/useless_conversion.rs:10:13 | LL | let _ = T::from(val); | ^^^^^^^^^^^^ help: consider removing `T::from()`: `val` @@ -11,115 +11,132 @@ LL | #![deny(clippy::useless_conversion)] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ error: useless conversion to the same type: `T` - --> tests/ui/useless_conversion.rs:11:5 + --> tests/ui/useless_conversion.rs:12:5 | LL | val.into() | ^^^^^^^^^^ help: consider removing `.into()`: `val` error: useless conversion to the same type: `i32` - --> tests/ui/useless_conversion.rs:24:22 + --> tests/ui/useless_conversion.rs:25:22 | LL | let _: i32 = 0i32.into(); | ^^^^^^^^^^^ help: consider removing `.into()`: `0i32` error: useless conversion to the same type: `std::str::Lines<'_>` - --> tests/ui/useless_conversion.rs:55:22 + --> tests/ui/useless_conversion.rs:56:22 | LL | if Some("ok") == lines.into_iter().next() {} | ^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `lines` error: useless conversion to the same type: `std::str::Lines<'_>` - --> tests/ui/useless_conversion.rs:61:21 + --> tests/ui/useless_conversion.rs:62:21 | LL | let mut lines = text.lines().into_iter(); | ^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `text.lines()` error: useless conversion to the same type: `std::str::Lines<'_>` - --> tests/ui/useless_conversion.rs:68:22 + --> tests/ui/useless_conversion.rs:69:22 | LL | if Some("ok") == text.lines().into_iter().next() {} | ^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `text.lines()` error: useless conversion to the same type: `std::ops::Range` - --> tests/ui/useless_conversion.rs:75:13 + --> tests/ui/useless_conversion.rs:76:13 | LL | let _ = NUMBERS.into_iter().next(); | ^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `NUMBERS` error: useless conversion to the same type: `std::ops::Range` - --> tests/ui/useless_conversion.rs:81:17 + --> tests/ui/useless_conversion.rs:82:17 | LL | let mut n = NUMBERS.into_iter(); | ^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `NUMBERS` +error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` + --> tests/ui/useless_conversion.rs:141:17 + | +LL | consume(items.into_iter()); + | ^^^^^^^^^^^^^^^^^ + | +note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` + --> tests/ui/useless_conversion.rs:137:28 + | +LL | fn consume(_: impl IntoIterator) {} + | ^^^^^^^^^^^^ +help: consider removing the `.into_iter()` + | +LL - consume(items.into_iter()); +LL + consume(*items); + | + error: useless conversion to the same type: `std::string::String` - --> tests/ui/useless_conversion.rs:144:21 + --> tests/ui/useless_conversion.rs:154:21 | LL | let _: String = "foo".to_string().into(); | ^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into()`: `"foo".to_string()` error: useless conversion to the same type: `std::string::String` - --> tests/ui/useless_conversion.rs:146:21 + --> tests/ui/useless_conversion.rs:156:21 | LL | let _: String = From::from("foo".to_string()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `From::from()`: `"foo".to_string()` error: useless conversion to the same type: `std::string::String` - --> tests/ui/useless_conversion.rs:148:13 + --> tests/ui/useless_conversion.rs:158:13 | LL | let _ = String::from("foo".to_string()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `String::from()`: `"foo".to_string()` error: useless conversion to the same type: `std::string::String` - --> tests/ui/useless_conversion.rs:150:13 + --> tests/ui/useless_conversion.rs:160:13 | LL | let _ = String::from(format!("A: {:04}", 123)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `String::from()`: `format!("A: {:04}", 123)` error: useless conversion to the same type: `std::str::Lines<'_>` - --> tests/ui/useless_conversion.rs:152:13 + --> tests/ui/useless_conversion.rs:162:13 | LL | let _ = "".lines().into_iter(); | ^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `"".lines()` error: useless conversion to the same type: `std::vec::IntoIter` - --> tests/ui/useless_conversion.rs:154:13 + --> tests/ui/useless_conversion.rs:164:13 | LL | let _ = vec![1, 2, 3].into_iter().into_iter(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `vec![1, 2, 3].into_iter()` error: useless conversion to the same type: `std::string::String` - --> tests/ui/useless_conversion.rs:156:21 + --> tests/ui/useless_conversion.rs:166:21 | LL | let _: String = format!("Hello {}", "world").into(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into()`: `format!("Hello {}", "world")` error: useless conversion to the same type: `i32` - --> tests/ui/useless_conversion.rs:162:13 + --> tests/ui/useless_conversion.rs:172:13 | LL | let _ = i32::from(a + b) * 3; | ^^^^^^^^^^^^^^^^ help: consider removing `i32::from()`: `(a + b)` error: useless conversion to the same type: `Foo<'a'>` - --> tests/ui/useless_conversion.rs:169:23 + --> tests/ui/useless_conversion.rs:179:23 | LL | let _: Foo<'a'> = s2.into(); | ^^^^^^^^^ help: consider removing `.into()`: `s2` error: useless conversion to the same type: `Foo<'a'>` - --> tests/ui/useless_conversion.rs:172:13 + --> tests/ui/useless_conversion.rs:182:13 | LL | let _ = Foo::<'a'>::from(s3); | ^^^^^^^^^^^^^^^^^^^^ help: consider removing `Foo::<'a'>::from()`: `s3` error: useless conversion to the same type: `std::vec::IntoIter>` - --> tests/ui/useless_conversion.rs:175:13 + --> tests/ui/useless_conversion.rs:185:13 | LL | let _ = vec![s4, s4, s4].into_iter().into_iter(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `vec![s4, s4, s4].into_iter()` error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:208:7 + --> tests/ui/useless_conversion.rs:218:7 | LL | b(vec![1, 2].into_iter()); | ^^^^^^^^^^------------ @@ -127,13 +144,13 @@ LL | b(vec![1, 2].into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:198:13 + --> tests/ui/useless_conversion.rs:208:13 | LL | fn b>(_: T) {} | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:210:7 + --> tests/ui/useless_conversion.rs:220:7 | LL | c(vec![1, 2].into_iter()); | ^^^^^^^^^^------------ @@ -141,13 +158,13 @@ LL | c(vec![1, 2].into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:199:18 + --> tests/ui/useless_conversion.rs:209:18 | LL | fn c(_: impl IntoIterator) {} | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:212:7 + --> tests/ui/useless_conversion.rs:222:7 | LL | d(vec![1, 2].into_iter()); | ^^^^^^^^^^------------ @@ -155,13 +172,13 @@ LL | d(vec![1, 2].into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:202:12 + --> tests/ui/useless_conversion.rs:212:12 | LL | T: IntoIterator, | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:216:7 + --> tests/ui/useless_conversion.rs:226:7 | LL | b(vec![1, 2].into_iter().into_iter()); | ^^^^^^^^^^------------------------ @@ -169,13 +186,13 @@ LL | b(vec![1, 2].into_iter().into_iter()); | help: consider removing the `.into_iter()`s | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:198:13 + --> tests/ui/useless_conversion.rs:208:13 | LL | fn b>(_: T) {} | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:218:7 + --> tests/ui/useless_conversion.rs:228:7 | LL | b(vec![1, 2].into_iter().into_iter().into_iter()); | ^^^^^^^^^^------------------------------------ @@ -183,13 +200,13 @@ LL | b(vec![1, 2].into_iter().into_iter().into_iter()); | help: consider removing the `.into_iter()`s | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:198:13 + --> tests/ui/useless_conversion.rs:208:13 | LL | fn b>(_: T) {} | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:265:24 + --> tests/ui/useless_conversion.rs:275:24 | LL | foo2::([1, 2, 3].into_iter()); | ^^^^^^^^^------------ @@ -197,13 +214,13 @@ LL | foo2::([1, 2, 3].into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:244:12 + --> tests/ui/useless_conversion.rs:254:12 | LL | I: IntoIterator + Helper, | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:274:14 + --> tests/ui/useless_conversion.rs:284:14 | LL | foo3([1, 2, 3].into_iter()); | ^^^^^^^^^------------ @@ -211,13 +228,13 @@ LL | foo3([1, 2, 3].into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:253:12 + --> tests/ui/useless_conversion.rs:263:12 | LL | I: IntoIterator, | ^^^^^^^^^^^^^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:284:16 + --> tests/ui/useless_conversion.rs:294:16 | LL | S1.foo([1, 2].into_iter()); | ^^^^^^------------ @@ -225,13 +242,13 @@ LL | S1.foo([1, 2].into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:281:27 + --> tests/ui/useless_conversion.rs:291:27 | LL | pub fn foo(&self, _: I) {} | ^^^^^^^^^^^^ error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:304:44 + --> tests/ui/useless_conversion.rs:314:44 | LL | v0.into_iter().interleave_shortest(v1.into_iter()); | ^^------------ @@ -239,67 +256,67 @@ LL | v0.into_iter().interleave_shortest(v1.into_iter()); | help: consider removing the `.into_iter()` | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:291:20 + --> tests/ui/useless_conversion.rs:301:20 | LL | J: IntoIterator, | ^^^^^^^^^^^^ error: useless conversion to the same type: `()` - --> tests/ui/useless_conversion.rs:332:58 + --> tests/ui/useless_conversion.rs:342:58 | LL | let _: Result<(), std::io::Error> = test_issue_3913().map(Into::into); | ^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `std::io::Error` - --> tests/ui/useless_conversion.rs:335:58 + --> tests/ui/useless_conversion.rs:345:58 | LL | let _: Result<(), std::io::Error> = test_issue_3913().map_err(Into::into); | ^^^^^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `()` - --> tests/ui/useless_conversion.rs:338:58 + --> tests/ui/useless_conversion.rs:348:58 | LL | let _: Result<(), std::io::Error> = test_issue_3913().map(From::from); | ^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `std::io::Error` - --> tests/ui/useless_conversion.rs:341:58 + --> tests/ui/useless_conversion.rs:351:58 | LL | let _: Result<(), std::io::Error> = test_issue_3913().map_err(From::from); | ^^^^^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `()` - --> tests/ui/useless_conversion.rs:345:31 + --> tests/ui/useless_conversion.rs:355:31 | LL | let _: ControlFlow<()> = c.map_break(Into::into); | ^^^^^^^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `()` - --> tests/ui/useless_conversion.rs:349:31 + --> tests/ui/useless_conversion.rs:359:31 | LL | let _: ControlFlow<()> = c.map_continue(Into::into); | ^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `u32` - --> tests/ui/useless_conversion.rs:363:41 + --> tests/ui/useless_conversion.rs:373:41 | LL | let _: Vec = [1u32].into_iter().map(Into::into).collect(); | ^^^^^^^^^^^^^^^^ help: consider removing error: useless conversion to the same type: `T` - --> tests/ui/useless_conversion.rs:374:18 + --> tests/ui/useless_conversion.rs:384:18 | LL | x.into_iter().map(Into::into).collect() | ^^^^^^^^^^^^^^^^ help: consider removing error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:390:29 + --> tests/ui/useless_conversion.rs:400:29 | LL | takes_into_iter(self.my_field.into_iter()); | ^^^^^^^^^^^^^^^^^^^^^^^^^ | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:379:32 + --> tests/ui/useless_conversion.rs:389:32 | LL | fn takes_into_iter(_: impl IntoIterator) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -310,13 +327,13 @@ LL + takes_into_iter(&self.my_field); | error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:398:29 + --> tests/ui/useless_conversion.rs:408:29 | LL | takes_into_iter(self.my_field.into_iter()); | ^^^^^^^^^^^^^^^^^^^^^^^^^ | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:379:32 + --> tests/ui/useless_conversion.rs:389:32 | LL | fn takes_into_iter(_: impl IntoIterator) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -327,13 +344,13 @@ LL + takes_into_iter(&mut self.my_field); | error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:407:29 + --> tests/ui/useless_conversion.rs:417:29 | LL | takes_into_iter(self.my_field.into_iter()); | ^^^^^^^^^^^^^^^^^^^^^^^^^ | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:379:32 + --> tests/ui/useless_conversion.rs:389:32 | LL | fn takes_into_iter(_: impl IntoIterator) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -344,13 +361,13 @@ LL + takes_into_iter(*self.my_field); | error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:416:29 + --> tests/ui/useless_conversion.rs:426:29 | LL | takes_into_iter(self.my_field.into_iter()); | ^^^^^^^^^^^^^^^^^^^^^^^^^ | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:379:32 + --> tests/ui/useless_conversion.rs:389:32 | LL | fn takes_into_iter(_: impl IntoIterator) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -361,13 +378,13 @@ LL + takes_into_iter(&*self.my_field); | error: explicit call to `.into_iter()` in function argument accepting `IntoIterator` - --> tests/ui/useless_conversion.rs:425:29 + --> tests/ui/useless_conversion.rs:435:29 | LL | takes_into_iter(self.my_field.into_iter()); | ^^^^^^^^^^^^^^^^^^^^^^^^^ | note: this parameter accepts any `IntoIterator`, so you don't need to call `.into_iter()` - --> tests/ui/useless_conversion.rs:379:32 + --> tests/ui/useless_conversion.rs:389:32 | LL | fn takes_into_iter(_: impl IntoIterator) {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -378,22 +395,22 @@ LL + takes_into_iter(&mut *self.my_field); | error: useless conversion to the same type: `std::ops::Range` - --> tests/ui/useless_conversion.rs:440:5 + --> tests/ui/useless_conversion.rs:450:5 | LL | R.into_iter().for_each(|_x| {}); | ^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `R` error: useless conversion to the same type: `std::ops::Range` - --> tests/ui/useless_conversion.rs:442:13 + --> tests/ui/useless_conversion.rs:452:13 | LL | let _ = R.into_iter().map(|_x| 0); | ^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `R` error: useless conversion to the same type: `std::slice::Iter<'_, i32>` - --> tests/ui/useless_conversion.rs:453:14 + --> tests/ui/useless_conversion.rs:463:14 | LL | for _ in mac!(iter [1, 2]).into_iter() {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider removing `.into_iter()`: `mac!(iter [1, 2])` -error: aborting due to 44 previous errors +error: aborting due to 45 previous errors diff --git a/triagebot.toml b/triagebot.toml index 5f637205fa65..09dec7675e7e 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -63,7 +63,6 @@ users_on_vacation = [ "Alexendoo", "y21", "blyxyas", - "samueltardieu", ] [assign.owners] From 7c3cc4f3ef6d936a11f19857984b06baa1a04d10 Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Thu, 8 Jan 2026 19:08:01 +0100 Subject: [PATCH 0150/1843] Make Clippy compile with `ConstArgKind::Tup()` --- clippy_lints/src/large_include_file.rs | 1 - clippy_lints/src/utils/author.rs | 1 + clippy_utils/src/consts.rs | 1 + clippy_utils/src/hir_utils.rs | 8 ++++++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/clippy_lints/src/large_include_file.rs b/clippy_lints/src/large_include_file.rs index d77e0beeaf4c..54599499515e 100644 --- a/clippy_lints/src/large_include_file.rs +++ b/clippy_lints/src/large_include_file.rs @@ -1,4 +1,3 @@ -use rustc_ast::AttrItemKind; use clippy_config::Conf; use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::macros::root_macro_call_first_node; diff --git a/clippy_lints/src/utils/author.rs b/clippy_lints/src/utils/author.rs index 58b153f06545..acea701b2e83 100644 --- a/clippy_lints/src/utils/author.rs +++ b/clippy_lints/src/utils/author.rs @@ -323,6 +323,7 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { ConstArgKind::TupleCall(..) => chain!(self, "let ConstArgKind::TupleCall(..) = {const_arg}.kind"), ConstArgKind::Infer(..) => chain!(self, "let ConstArgKind::Infer(..) = {const_arg}.kind"), ConstArgKind::Error(..) => chain!(self, "let ConstArgKind::Error(..) = {const_arg}.kind"), + ConstArgKind::Tup(..) => chain!(self, "let ConstArgKind::Tup(..) = {const_arg}.kind"), ConstArgKind::Literal(..) => chain!(self, "let ConstArgKind::Literal(..) = {const_arg}.kind") } } diff --git a/clippy_utils/src/consts.rs b/clippy_utils/src/consts.rs index 334cc6bb5d55..e4f76cf4ed57 100644 --- a/clippy_utils/src/consts.rs +++ b/clippy_utils/src/consts.rs @@ -1141,6 +1141,7 @@ pub fn const_item_rhs_to_expr<'tcx>(tcx: TyCtxt<'tcx>, ct_rhs: ConstItemRhs<'tcx ConstItemRhs::TypeConst(const_arg) => match const_arg.kind { ConstArgKind::Anon(anon) => Some(tcx.hir_body(anon.body).value), ConstArgKind::Struct(..) + | ConstArgKind::Tup(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs index 73b1cbb21548..82a12fc51c9a 100644 --- a/clippy_utils/src/hir_utils.rs +++ b/clippy_utils/src/hir_utils.rs @@ -666,6 +666,8 @@ impl HirEqInterExpr<'_, '_, '_> { } match (&left.kind, &right.kind) { + (ConstArgKind::Tup(l_t), ConstArgKind::Tup(r_t)) => + l_t.len() == r_t.len() && l_t.iter().zip(*r_t).all(|(l_c, r_c)| self.eq_const_arg(*l_c, *r_c)), (ConstArgKind::Path(l_p), ConstArgKind::Path(r_p)) => self.eq_qpath(l_p, r_p), (ConstArgKind::Anon(l_an), ConstArgKind::Anon(r_an)) => self.eq_body(l_an.body, r_an.body), (ConstArgKind::Infer(..), ConstArgKind::Infer(..)) => true, @@ -695,6 +697,7 @@ impl HirEqInterExpr<'_, '_, '_> { // Use explicit match for now since ConstArg is undergoing flux. ( ConstArgKind::Path(..) + | ConstArgKind::Tup(..) | ConstArgKind::Anon(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Tup(..) @@ -1561,6 +1564,11 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> { fn hash_const_arg(&mut self, const_arg: &ConstArg<'_>) { match &const_arg.kind { + ConstArgKind::Tup(tup) => { + for arg in *tup { + self.hash_const_arg(*arg); + } + }, ConstArgKind::Path(path) => self.hash_qpath(path), ConstArgKind::Anon(anon) => self.hash_body(anon.body), ConstArgKind::Struct(path, inits) => { From 697e9e2aacecaa6a610ab310b9af54a732903d21 Mon Sep 17 00:00:00 2001 From: Philipp Krones Date: Fri, 9 Jan 2026 11:59:23 +0100 Subject: [PATCH 0151/1843] Further Clippy fixes for Tup/Literal ConstArgKind --- clippy_utils/src/consts.rs | 1 + clippy_utils/src/hir_utils.rs | 23 +++++------------------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/clippy_utils/src/consts.rs b/clippy_utils/src/consts.rs index e4f76cf4ed57..538f1fd2628c 100644 --- a/clippy_utils/src/consts.rs +++ b/clippy_utils/src/consts.rs @@ -1142,6 +1142,7 @@ pub fn const_item_rhs_to_expr<'tcx>(tcx: TyCtxt<'tcx>, ct_rhs: ConstItemRhs<'tcx ConstArgKind::Anon(anon) => Some(tcx.hir_body(anon.body).value), ConstArgKind::Struct(..) | ConstArgKind::Tup(..) + | ConstArgKind::Literal(..) | ConstArgKind::TupleCall(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs index 82a12fc51c9a..0bb641568879 100644 --- a/clippy_utils/src/hir_utils.rs +++ b/clippy_utils/src/hir_utils.rs @@ -666,8 +666,9 @@ impl HirEqInterExpr<'_, '_, '_> { } match (&left.kind, &right.kind) { - (ConstArgKind::Tup(l_t), ConstArgKind::Tup(r_t)) => - l_t.len() == r_t.len() && l_t.iter().zip(*r_t).all(|(l_c, r_c)| self.eq_const_arg(*l_c, *r_c)), + (ConstArgKind::Tup(l_t), ConstArgKind::Tup(r_t)) => { + l_t.len() == r_t.len() && l_t.iter().zip(*r_t).all(|(l_c, r_c)| self.eq_const_arg(*l_c, *r_c)) + }, (ConstArgKind::Path(l_p), ConstArgKind::Path(r_p)) => self.eq_qpath(l_p, r_p), (ConstArgKind::Anon(l_an), ConstArgKind::Anon(r_an)) => self.eq_body(l_an.body, r_an.body), (ConstArgKind::Infer(..), ConstArgKind::Infer(..)) => true, @@ -684,23 +685,14 @@ impl HirEqInterExpr<'_, '_, '_> { .iter() .zip(*args_b) .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) - } - (ConstArgKind::Tup(args_a), ConstArgKind::Tup(args_b)) => { - args_a - .iter() - .zip(*args_b) - .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) - }, - (ConstArgKind::Literal(kind_l), ConstArgKind::Literal(kind_r)) => { - kind_l == kind_r }, + (ConstArgKind::Literal(kind_l), ConstArgKind::Literal(kind_r)) => kind_l == kind_r, // Use explicit match for now since ConstArg is undergoing flux. ( ConstArgKind::Path(..) | ConstArgKind::Tup(..) | ConstArgKind::Anon(..) | ConstArgKind::TupleCall(..) - | ConstArgKind::Tup(..) | ConstArgKind::Infer(..) | ConstArgKind::Struct(..) | ConstArgKind::Literal(..) @@ -1583,13 +1575,8 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> { self.hash_const_arg(arg); } }, - ConstArgKind::Tup(args) => { - for arg in *args { - self.hash_const_arg(arg); - } - }, ConstArgKind::Infer(..) | ConstArgKind::Error(..) => {}, - ConstArgKind::Literal(lit) => lit.hash(&mut self.s) + ConstArgKind::Literal(lit) => lit.hash(&mut self.s), } } From e80fbd4bca604211f810fc207f33089730a3e9e1 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Fri, 9 Jan 2026 13:47:13 +0200 Subject: [PATCH 0152/1843] Fix lifetimes len diagnostics for fn pointers --- .../rust-analyzer/crates/hir-ty/src/lower.rs | 51 ++++++++++--------- .../crates/hir-ty/src/lower/path.rs | 8 +-- .../src/handlers/missing_lifetime.rs | 15 ++++++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs b/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs index a97d7687162e..9befca11b3e5 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs @@ -27,8 +27,8 @@ use hir_def::{ resolver::{HasResolver, LifetimeNs, Resolver, TypeNs, ValueNs}, signatures::{FunctionSignature, TraitFlags, TypeAliasFlags}, type_ref::{ - ConstRef, LifetimeRefId, PathId, TraitBoundModifier, TraitRef as HirTraitRef, TypeBound, - TypeRef, TypeRefId, + ConstRef, FnType, LifetimeRefId, PathId, TraitBoundModifier, TraitRef as HirTraitRef, + TypeBound, TypeRef, TypeRefId, }, }; use hir_expand::name::Name; @@ -98,7 +98,7 @@ impl ImplTraitLoweringState { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub enum LifetimeElisionKind<'db> { /// Create a new anonymous lifetime parameter and reference it. /// @@ -437,26 +437,7 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { Ty::new_ref(interner, lifetime, inner_ty, lower_mutability(ref_.mutability)) } TypeRef::Placeholder => Ty::new_error(interner, ErrorGuaranteed), - TypeRef::Fn(fn_) => { - let substs = self.with_shifted_in( - DebruijnIndex::from_u32(1), - |ctx: &mut TyLoweringContext<'_, '_>| { - Tys::new_from_iter( - interner, - fn_.params.iter().map(|&(_, tr)| ctx.lower_ty(tr)), - ) - }, - ); - Ty::new_fn_ptr( - interner, - Binder::dummy(FnSig { - abi: fn_.abi.as_ref().map_or(FnAbi::Rust, FnAbi::from_symbol), - safety: if fn_.is_unsafe { Safety::Unsafe } else { Safety::Safe }, - c_variadic: fn_.is_varargs, - inputs_and_output: substs, - }), - ) - } + TypeRef::Fn(fn_) => self.lower_fn_ptr(fn_), TypeRef::DynTrait(bounds) => self.lower_dyn_trait(bounds), TypeRef::ImplTrait(bounds) => { match self.impl_trait_mode.mode { @@ -517,6 +498,30 @@ impl<'db, 'a> TyLoweringContext<'db, 'a> { (ty, res) } + fn lower_fn_ptr(&mut self, fn_: &FnType) -> Ty<'db> { + let interner = self.interner; + let (params, ret_ty) = fn_.split_params_and_ret(); + let old_lifetime_elision = self.lifetime_elision; + let mut args = Vec::with_capacity(fn_.params.len()); + self.with_shifted_in(DebruijnIndex::from_u32(1), |ctx: &mut TyLoweringContext<'_, '_>| { + ctx.lifetime_elision = + LifetimeElisionKind::AnonymousCreateParameter { report_in_path: false }; + args.extend(params.iter().map(|&(_, tr)| ctx.lower_ty(tr))); + ctx.lifetime_elision = LifetimeElisionKind::for_fn_ret(interner); + args.push(ctx.lower_ty(ret_ty)); + }); + self.lifetime_elision = old_lifetime_elision; + Ty::new_fn_ptr( + interner, + Binder::dummy(FnSig { + abi: fn_.abi.as_ref().map_or(FnAbi::Rust, FnAbi::from_symbol), + safety: if fn_.is_unsafe { Safety::Unsafe } else { Safety::Safe }, + c_variadic: fn_.is_varargs, + inputs_and_output: Tys::new_from_slice(&args), + }), + ) + } + /// This is only for `generic_predicates_for_param`, where we can't just /// lower the self types of the predicates since that could lead to cycles. /// So we just check here if the `type_ref` resolves to a generic param, and which. diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs b/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs index b77aeab62d15..f3d0de12275e 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/lower/path.rs @@ -599,7 +599,7 @@ impl<'a, 'b, 'db> PathLoweringContext<'a, 'b, 'db> { explicit_self_ty: Option>, lowering_assoc_type_generics: bool, ) -> GenericArgs<'db> { - let old_lifetime_elision = self.ctx.lifetime_elision.clone(); + let old_lifetime_elision = self.ctx.lifetime_elision; if let Some(args) = self.current_or_prev_segment.args_and_bindings && args.parenthesized != GenericArgsParentheses::No @@ -640,7 +640,7 @@ impl<'a, 'b, 'db> PathLoweringContext<'a, 'b, 'db> { explicit_self_ty, PathGenericsSource::Segment(self.current_segment_u32()), lowering_assoc_type_generics, - self.ctx.lifetime_elision.clone(), + self.ctx.lifetime_elision, ); self.ctx.lifetime_elision = old_lifetime_elision; result @@ -884,7 +884,7 @@ impl<'a, 'b, 'db> PathLoweringContext<'a, 'b, 'db> { assoc_type: binding_idx as u32, }, false, - this.ctx.lifetime_elision.clone(), + this.ctx.lifetime_elision, ) }); let args = GenericArgs::new_from_iter( @@ -902,7 +902,7 @@ impl<'a, 'b, 'db> PathLoweringContext<'a, 'b, 'db> { // `Fn()`-style generics are elided like functions. This is `Output` (we lower to it in hir-def). LifetimeElisionKind::for_fn_ret(self.ctx.interner) } else { - self.ctx.lifetime_elision.clone() + self.ctx.lifetime_elision }; self.with_lifetime_elision(lifetime_elision, |this| { match (&this.ctx.store[type_ref], this.ctx.impl_trait_mode.mode) { diff --git a/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/missing_lifetime.rs b/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/missing_lifetime.rs index b07f9e68f634..5cb710b66b5f 100644 --- a/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/missing_lifetime.rs +++ b/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/missing_lifetime.rs @@ -100,4 +100,19 @@ fn foo WithLifetime>() {} "#, ); } + + #[test] + fn regression_21430() { + check_diagnostics( + r#" +struct S { + f: fn(A<()>), +} + +struct A<'a, T> { + a: &'a T, +} + "#, + ); + } } From e52695c3fca6f9b973c98103407d83d1c963cbdf Mon Sep 17 00:00:00 2001 From: Wilfred Hughes Date: Fri, 9 Jan 2026 11:58:16 +0000 Subject: [PATCH 0153/1843] internal: Include private definitions in generated rustdoc rust-analyzer has handy prebuilt `cargo doc` output at https://rust-lang.github.io/rust-analyzer/ide/ However, it doesn't include private definitions, which makes it less useful when trying to learn unfamiliar parts of the codebase. Instead, pass `--document-private-items` so the HTML includes information on private types and modules too. rustdoc renders these with a padlock icon, so it's still clear that they're private. This change also exposes some more rustdoc warnings, which I've fixed. --- src/tools/rust-analyzer/.github/workflows/rustdoc.yaml | 2 +- src/tools/rust-analyzer/crates/hir-def/src/nameres.rs | 2 +- .../crates/hir-expand/src/builtin/attr_macro.rs | 2 +- .../rust-analyzer/crates/hir-expand/src/cfg_process.rs | 2 +- .../rust-analyzer/crates/hir-ty/src/infer/closure.rs | 2 +- .../rust-analyzer/crates/hir-ty/src/method_resolution.rs | 4 ++-- .../crates/hir-ty/src/method_resolution/probe.rs | 2 +- .../crates/hir-ty/src/next_solver/infer/mod.rs | 2 +- src/tools/rust-analyzer/crates/hir/src/term_search.rs | 2 +- .../crates/ide-assists/src/handlers/inline_type_alias.rs | 6 +++++- .../rust-analyzer/crates/ide-completion/src/context.rs | 2 +- src/tools/rust-analyzer/crates/parser/src/grammar.rs | 2 +- .../crates/project-model/src/cargo_workspace.rs | 2 +- .../crates/rust-analyzer/src/config/patch_old_style.rs | 2 +- .../rust-analyzer/crates/rust-analyzer/src/discover.rs | 4 ++-- .../crates/rust-analyzer/src/global_state.rs | 2 +- .../crates/rust-analyzer/src/handlers/dispatch.rs | 6 +++--- .../rust-analyzer/crates/rust-analyzer/src/task_pool.rs | 2 +- src/tools/rust-analyzer/crates/span/src/hygiene.rs | 8 ++++---- .../crates/syntax/src/syntax_editor/mapping.rs | 2 +- src/tools/rust-analyzer/lib/line-index/src/lib.rs | 2 +- 21 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/tools/rust-analyzer/.github/workflows/rustdoc.yaml b/src/tools/rust-analyzer/.github/workflows/rustdoc.yaml index 9cc18fc69ede..0cc7ce77ddb6 100644 --- a/src/tools/rust-analyzer/.github/workflows/rustdoc.yaml +++ b/src/tools/rust-analyzer/.github/workflows/rustdoc.yaml @@ -24,7 +24,7 @@ jobs: run: rustup update --no-self-update stable - name: Build Documentation - run: cargo doc --all --no-deps + run: cargo doc --all --no-deps --document-private-items - name: Deploy Docs uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs index 150372f1a0d9..1e3ea50c5a0f 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres.rs @@ -216,7 +216,7 @@ struct DefMapCrateData { registered_tools: Vec, /// Unstable features of Rust enabled with `#![feature(A, B)]`. unstable_features: FxHashSet, - /// #[rustc_coherence_is_core] + /// `#[rustc_coherence_is_core]` rustc_coherence_is_core: bool, no_core: bool, no_std: bool, diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/attr_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/attr_macro.rs index 06b9b5418e37..c94663ca0cbc 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/attr_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/attr_macro.rs @@ -115,7 +115,7 @@ fn dummy_gate_test_expand( /// wasting a lot of memory, and it would also require some way to use a path in a way that makes it /// always resolve as a derive without nameres recollecting them. /// So this hacky approach is a lot more friendly for us, though it does require a bit of support in -/// [`hir::Semantics`] to make this work. +/// hir::Semantics to make this work. fn derive_expand( db: &dyn ExpandDatabase, id: MacroCallId, diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs b/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs index a0de36548e9f..ccef9168ac3a 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/cfg_process.rs @@ -1,4 +1,4 @@ -//! Processes out #[cfg] and #[cfg_attr] attributes from the input for the derive macro +//! Processes out `#[cfg]` and `#[cfg_attr]` attributes from the input for the derive macro use std::{cell::OnceCell, ops::ControlFlow}; use ::tt::TextRange; diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer/closure.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer/closure.rs index d1391ad24e4d..ce99016470c1 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/infer/closure.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer/closure.rs @@ -466,7 +466,7 @@ impl<'db> InferenceContext<'_, 'db> { } /// Given an `FnOnce::Output` or `AsyncFn::Output` projection, extract the args - /// and return type to infer a [`ty::PolyFnSig`] for the closure. + /// and return type to infer a `PolyFnSig` for the closure. fn extract_sig_from_projection( &self, projection: PolyProjectionPredicate<'db>, diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution.rs b/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution.rs index e4681b464fec..ad4d79e68a9f 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution.rs @@ -206,11 +206,11 @@ impl<'a, 'db> InferenceContext<'a, 'db> { } } -/// Used by [FnCtxt::lookup_method_for_operator] with `-Znext-solver`. +/// Used by `FnCtxt::lookup_method_for_operator` with `-Znext-solver`. /// /// With `AsRigid` we error on `impl Opaque: NotInItemBounds` while /// `AsInfer` just treats it as ambiguous and succeeds. This is necessary -/// as we want [FnCtxt::check_expr_call] to treat not-yet-defined opaque +/// as we want `FnCtxt::check_expr_call` to treat not-yet-defined opaque /// types as rigid to support `impl Deref` and /// `Box`. /// diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution/probe.rs b/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution/probe.rs index 4a7c7d93539e..42a590e8b4cb 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution/probe.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/method_resolution/probe.rs @@ -1740,7 +1740,7 @@ impl<'a, 'db, Choice: ProbeChoice<'db>> ProbeContext<'a, 'db, Choice> { /// We want to only accept trait methods if they were hold even if the /// opaque types were rigid. To handle this, we both check that for trait /// candidates the goal were to hold even when treating opaques as rigid, - /// see [OpaqueTypesJank](rustc_trait_selection::solve::OpaqueTypesJank). + /// see `rustc_trait_selection::solve::OpaqueTypesJank`. /// /// We also check that all opaque types encountered as self types in the /// autoderef chain don't get constrained when applying the candidate. diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/mod.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/mod.rs index 7d291f7ddbed..21baacb11693 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/mod.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/infer/mod.rs @@ -140,7 +140,7 @@ pub struct InferCtxtInner<'db> { /// /// Before running `resolve_regions_and_report_errors`, the creator /// of the inference context is expected to invoke - /// [`InferCtxt::process_registered_region_obligations`] + /// `InferCtxt::process_registered_region_obligations` /// for each body-id in this map, which will process the /// obligations within. This is expected to be done 'late enough' /// that all type inference variables have been bound and so forth. diff --git a/src/tools/rust-analyzer/crates/hir/src/term_search.rs b/src/tools/rust-analyzer/crates/hir/src/term_search.rs index e4089218305c..f2dc1ce798ad 100644 --- a/src/tools/rust-analyzer/crates/hir/src/term_search.rs +++ b/src/tools/rust-analyzer/crates/hir/src/term_search.rs @@ -172,7 +172,7 @@ impl<'db> LookupTable<'db> { /// Insert new type trees for type /// /// Note that the types have to be the same, unification is not enough as unification is not - /// transitive. For example Vec and FxHashSet both unify with Iterator, + /// transitive. For example `Vec` and `FxHashSet` both unify with `Iterator`, /// but they clearly do not unify themselves. fn insert(&mut self, ty: Type<'db>, exprs: impl Iterator>) { match self.data.get_mut(&ty) { diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_type_alias.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_type_alias.rs index ae8d130df23c..c7a48f3261a9 100644 --- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_type_alias.rs +++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/inline_type_alias.rs @@ -290,19 +290,23 @@ impl ConstAndTypeMap { /// ^ alias generic params /// let a: A<100>; /// ^ instance generic args -/// ``` /// /// generic['a] = '_ due to omission /// generic[N] = 100 due to the instance arg /// generic[T] = u64 due to the default param +/// ``` /// /// 2. Copy the concrete type and substitute in each found mapping: /// +/// ```ignore /// &'_ [u64; 100] +/// ``` /// /// 3. Remove wildcard lifetimes entirely: /// +/// ```ignore /// &[u64; 100] +/// ``` fn create_replacement( lifetime_map: &LifetimeMap, const_and_type_map: &ConstAndTypeMap, diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/context.rs b/src/tools/rust-analyzer/crates/ide-completion/src/context.rs index d116f665adbd..cab8bced88df 100644 --- a/src/tools/rust-analyzer/crates/ide-completion/src/context.rs +++ b/src/tools/rust-analyzer/crates/ide-completion/src/context.rs @@ -628,7 +628,7 @@ impl CompletionContext<'_> { } /// A version of [`SemanticsScope::process_all_names`] that filters out `#[doc(hidden)]` items and - /// passes all doc-aliases along, to funnel it into [`Completions::add_path_resolution`]. + /// passes all doc-aliases along, to funnel it into `Completions::add_path_resolution`. pub(crate) fn process_all_names(&self, f: &mut dyn FnMut(Name, ScopeDef, Vec)) { let _p = tracing::info_span!("CompletionContext::process_all_names").entered(); self.scope.process_all_names(&mut |name, def| { diff --git a/src/tools/rust-analyzer/crates/parser/src/grammar.rs b/src/tools/rust-analyzer/crates/parser/src/grammar.rs index bf8430294110..e481bbe9bc4a 100644 --- a/src/tools/rust-analyzer/crates/parser/src/grammar.rs +++ b/src/tools/rust-analyzer/crates/parser/src/grammar.rs @@ -6,7 +6,7 @@ //! each submodule starts with `use super::*` import and exports //! "public" productions via `pub(super)`. //! -//! See docs for [`Parser`](super::parser::Parser) to learn about API, +//! See docs for [`Parser`] to learn about API, //! available to the grammar, and see docs for [`Event`](super::event::Event) //! to learn how this actually manages to produce parse trees. //! diff --git a/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs b/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs index 6e1a3f37ff1c..483ab2845045 100644 --- a/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs +++ b/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs @@ -640,7 +640,7 @@ impl FetchMetadata { /// Builds a command to fetch metadata for the given `cargo_toml` manifest. /// /// Performs a lightweight pre-fetch using the `--no-deps` option, - /// available via [`FetchMetadata::no_deps_metadata`], to gather basic + /// available via `FetchMetadata::no_deps_metadata`, to gather basic /// information such as the `target-dir`. /// /// The provided sysroot is used to set the `RUSTUP_TOOLCHAIN` diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs index 389bb7848c01..5dc463eccce4 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs @@ -3,7 +3,7 @@ use serde_json::{Value, json}; /// This function patches the json config to the new expected keys. /// That is we try to load old known config keys here and convert them to the new ones. -/// See https://github.com/rust-lang/rust-analyzer/pull/12010 +/// See /// /// We already have an alias system for simple cases, but if we make structural changes /// the alias infra fails down. diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs index 4aef5b0b7f3d..f129f156a030 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs @@ -42,7 +42,7 @@ impl DiscoverCommand { Self { sender, command } } - /// Spawn the command inside [Discover] and report progress, if any. + /// Spawn the command inside `DiscoverCommand` and report progress, if any. pub(crate) fn spawn( &self, discover_arg: DiscoverArgument, @@ -73,7 +73,7 @@ impl DiscoverCommand { } } -/// A handle to a spawned [Discover]. +/// A handle to a spawned `DiscoverCommand`. #[derive(Debug)] pub(crate) struct DiscoverHandle { pub(crate) handle: CommandHandle, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs index 39b4aaa64738..afd4162de622 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs @@ -188,7 +188,7 @@ pub(crate) struct GlobalState { /// been called. pub(crate) deferred_task_queue: DeferredTaskQueue, - /// HACK: Workaround for https://github.com/rust-lang/rust-analyzer/issues/19709 + /// HACK: Workaround for /// This is marked true if we failed to load a crate root file at crate graph creation, /// which will usually end up causing a bunch of incorrect diagnostics on startup. pub(crate) incomplete_crate_graph: bool, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs index 10bbb0bb31d9..90deae2d902e 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs @@ -101,7 +101,7 @@ impl RequestDispatcher<'_> { } /// Dispatches a non-latency-sensitive request onto the thread pool. When the VFS is marked not - /// ready this will return a default constructed [`R::Result`]. + /// ready this will return a default constructed `R::Result`. pub(crate) fn on( &mut self, f: fn(GlobalStateSnapshot, R::Params) -> anyhow::Result, @@ -128,7 +128,7 @@ impl RequestDispatcher<'_> { } /// Dispatches a non-latency-sensitive request onto the thread pool. When the VFS is marked not - /// ready this will return a `default` constructed [`R::Result`]. + /// ready this will return a `default` constructed `R::Result`. pub(crate) fn on_with_vfs_default( &mut self, f: fn(GlobalStateSnapshot, R::Params) -> anyhow::Result, @@ -176,7 +176,7 @@ impl RequestDispatcher<'_> { } /// Dispatches a latency-sensitive request onto the thread pool. When the VFS is marked not - /// ready this will return a default constructed [`R::Result`]. + /// ready this will return a default constructed `R::Result`. pub(crate) fn on_latency_sensitive( &mut self, f: fn(GlobalStateSnapshot, R::Params) -> anyhow::Result, diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/task_pool.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/task_pool.rs index 8b8876b801cf..104cd3d2eae9 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/task_pool.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/task_pool.rs @@ -52,7 +52,7 @@ impl TaskPool { /// `DeferredTaskQueue` holds deferred tasks. /// /// These are tasks that must be run after -/// [`GlobalState::process_changes`] has been called. +/// `GlobalState::process_changes` has been called. pub(crate) struct DeferredTaskQueue { pub(crate) sender: crossbeam_channel::Sender, pub(crate) receiver: crossbeam_channel::Receiver, diff --git a/src/tools/rust-analyzer/crates/span/src/hygiene.rs b/src/tools/rust-analyzer/crates/span/src/hygiene.rs index ea4f4c5efb42..92bf892ea529 100644 --- a/src/tools/rust-analyzer/crates/span/src/hygiene.rs +++ b/src/tools/rust-analyzer/crates/span/src/hygiene.rs @@ -8,9 +8,9 @@ //! //! # The Expansion Order Hierarchy //! -//! `ExpnData` in rustc, rust-analyzer's version is [`MacroCallLoc`]. Traversing the hierarchy -//! upwards can be achieved by walking up [`MacroCallLoc::kind`]'s contained file id, as -//! [`MacroFile`]s are interned [`MacroCallLoc`]s. +//! `ExpnData` in rustc, rust-analyzer's version is `MacroCallLoc`. Traversing the hierarchy +//! upwards can be achieved by walking up `MacroCallLoc::kind`'s contained file id, as +//! `MacroFile`s are interned `MacroCallLoc`s. //! //! # The Macro Definition Hierarchy //! @@ -18,7 +18,7 @@ //! //! # The Call-site Hierarchy //! -//! `ExpnData::call_site` in rustc, [`MacroCallLoc::call_site`] in rust-analyzer. +//! `ExpnData::call_site` in rustc, `MacroCallLoc::call_site` in rust-analyzer. use crate::Edition; use std::fmt; diff --git a/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/mapping.rs b/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/mapping.rs index 1eaef03197c5..6257bf4e572e 100644 --- a/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/mapping.rs +++ b/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/mapping.rs @@ -1,6 +1,6 @@ //! Maps syntax elements through disjoint syntax nodes. //! -//! [`SyntaxMappingBuilder`] should be used to create mappings to add to a [`SyntaxEditor`] +//! [`SyntaxMappingBuilder`] should be used to create mappings to add to a `SyntaxEditor` use itertools::Itertools; use rustc_hash::FxHashMap; diff --git a/src/tools/rust-analyzer/lib/line-index/src/lib.rs b/src/tools/rust-analyzer/lib/line-index/src/lib.rs index 905da330e64b..d5f0584d988f 100644 --- a/src/tools/rust-analyzer/lib/line-index/src/lib.rs +++ b/src/tools/rust-analyzer/lib/line-index/src/lib.rs @@ -207,7 +207,7 @@ impl LineIndex { } } -/// This is adapted from the rustc_span crate, https://github.com/rust-lang/rust/blob/de59844c98f7925242a798a72c59dc3610dd0e2c/compiler/rustc_span/src/analyze_source_file.rs +/// This is adapted from the rustc_span crate, fn analyze_source_file(src: &str) -> (Vec, IntMap>) { assert!(src.len() < !0u32 as usize); let mut lines = vec![]; From 998a5ac623d9879cfba667d6220a8f7d228845e8 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Fri, 9 Jan 2026 15:55:45 +0200 Subject: [PATCH 0154/1843] Remove code made redundant by method resolution rewrite Its job is now done elsewhere, and it's also wrong (not accounting for autoderef) --- .../crates/hir-ty/src/infer/expr.rs | 12 ++-- .../crates/hir-ty/src/tests/regression.rs | 55 +++++++++++++--- .../crates/hir-ty/src/tests/traits.rs | 66 +++++++++++-------- 3 files changed, 88 insertions(+), 45 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs index 226e9f5cd667..62339779a562 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs @@ -1704,7 +1704,7 @@ impl<'db> InferenceContext<'_, 'db> { }); match resolved { Ok((func, _is_visible)) => { - self.check_method_call(tgt_expr, &[], func.sig, receiver_ty, expected) + self.check_method_call(tgt_expr, &[], func.sig, expected) } Err(_) => self.err_ty(), } @@ -1844,7 +1844,7 @@ impl<'db> InferenceContext<'_, 'db> { item: func.def_id.into(), }) } - self.check_method_call(tgt_expr, args, func.sig, receiver_ty, expected) + self.check_method_call(tgt_expr, args, func.sig, expected) } // Failed to resolve, report diagnostic and try to resolve as call to field access or // assoc function @@ -1934,16 +1934,14 @@ impl<'db> InferenceContext<'_, 'db> { tgt_expr: ExprId, args: &[ExprId], sig: FnSig<'db>, - receiver_ty: Ty<'db>, expected: &Expectation<'db>, ) -> Ty<'db> { - let (formal_receiver_ty, param_tys) = if !sig.inputs_and_output.inputs().is_empty() { - (sig.inputs_and_output.as_slice()[0], &sig.inputs_and_output.inputs()[1..]) + let param_tys = if !sig.inputs_and_output.inputs().is_empty() { + &sig.inputs_and_output.inputs()[1..] } else { - (self.types.types.error, &[] as _) + &[] }; let ret_ty = sig.output(); - self.table.unify(formal_receiver_ty, receiver_ty); self.check_call_arguments(tgt_expr, param_tys, ret_ty, expected, args, &[], sig.c_variadic); ret_ty diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs index c805f030446c..df49d7999fee 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs @@ -891,13 +891,14 @@ use core::ops::Deref; struct BufWriter {} -struct Mutex {} -struct MutexGuard<'a, T> {} +struct Mutex(T); +struct MutexGuard<'a, T>(&'a T); impl Mutex { fn lock(&self) -> MutexGuard<'_, T> {} } impl<'a, T: 'a> Deref for MutexGuard<'a, T> { type Target = T; + fn deref(&self) -> &Self::Target { loop {} } } fn flush(&self) { let w: &Mutex; @@ -905,14 +906,18 @@ fn flush(&self) { } "#, expect![[r#" - 123..127 'self': &'? Mutex - 150..152 '{}': MutexGuard<'?, T> - 234..238 'self': &'? {unknown} - 240..290 '{ ...()); }': () - 250..251 'w': &'? Mutex - 276..287 '*(w.lock())': BufWriter - 278..279 'w': &'? Mutex - 278..286 'w.lock()': MutexGuard<'?, BufWriter> + 129..133 'self': &'? Mutex + 156..158 '{}': MutexGuard<'?, T> + 242..246 'self': &'? MutexGuard<'a, T> + 265..276 '{ loop {} }': &'? T + 267..274 'loop {}': ! + 272..274 '{}': () + 289..293 'self': &'? {unknown} + 295..345 '{ ...()); }': () + 305..306 'w': &'? Mutex + 331..342 '*(w.lock())': BufWriter + 333..334 'w': &'? Mutex + 333..341 'w.lock()': MutexGuard<'?, BufWriter> "#]], ); } @@ -2563,3 +2568,33 @@ fn main() { "#, ); } + +#[test] +fn regression_21429() { + check_no_mismatches( + r#" +trait DatabaseLike { + type ForeignKey: ForeignKeyLike; +} + +trait ForeignKeyLike { + type DB: DatabaseLike; + + fn host_columns(&self, database: &Self::DB); +} + +trait ColumnLike { + type DB: DatabaseLike; + + fn foo() -> &&<::DB as DatabaseLike>::ForeignKey { + loop {} + } + + fn foreign_keys(&self, database: &Self::DB) { + let fk = Self::foo(); + fk.host_columns(database); + } +} + "#, + ); +} diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs index 38591f486e97..b825a0a8f0e5 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs @@ -429,7 +429,7 @@ fn associated_type_shorthand_from_method_bound() { trait Iterable { type Item; } -struct S; +struct S(T); impl S { fn foo(self) -> T::Item where T: Iterable { loop {} } } @@ -1103,40 +1103,50 @@ fn test() { fn argument_impl_trait_type_args_2() { check_infer_with_mismatches( r#" -//- minicore: sized +//- minicore: sized, phantom_data +use core::marker::PhantomData; + trait Trait {} struct S; impl Trait for S {} -struct F; +struct F(PhantomData); impl F { fn foo(self, x: impl Trait) -> (T, U) { loop {} } } fn test() { - F.foo(S); - F::.foo(S); - F::.foo::(S); - F::.foo::(S); // extraneous argument should be ignored + F(PhantomData).foo(S); + F::(PhantomData).foo(S); + F::(PhantomData).foo::(S); + F::(PhantomData).foo::(S); // extraneous argument should be ignored }"#, expect![[r#" - 87..91 'self': F - 93..94 'x': impl Trait - 118..129 '{ loop {} }': (T, U) - 120..127 'loop {}': ! - 125..127 '{}': () - 143..283 '{ ...ored }': () - 149..150 'F': F<{unknown}> - 149..157 'F.foo(S)': ({unknown}, {unknown}) - 155..156 'S': S - 163..171 'F::': F - 163..178 'F::.foo(S)': (u32, {unknown}) - 176..177 'S': S - 184..192 'F::': F - 184..206 'F::(S)': (u32, i32) - 204..205 'S': S - 212..220 'F::': F - 212..239 'F::(S)': (u32, i32) - 237..238 'S': S + 135..139 'self': F + 141..142 'x': impl Trait + 166..177 '{ loop {} }': (T, U) + 168..175 'loop {}': ! + 173..175 '{}': () + 191..383 '{ ...ored }': () + 197..198 'F': fn F<{unknown}>(PhantomData<{unknown}>) -> F<{unknown}> + 197..211 'F(PhantomData)': F<{unknown}> + 197..218 'F(Phan...foo(S)': ({unknown}, {unknown}) + 199..210 'PhantomData': PhantomData<{unknown}> + 216..217 'S': S + 224..232 'F::': fn F(PhantomData) -> F + 224..245 'F:: + 224..252 'F:: + 250..251 'S': S + 258..266 'F::': fn F(PhantomData) -> F + 258..279 'F:: + 258..293 'F::(S)': (u32, i32) + 267..278 'PhantomData': PhantomData + 291..292 'S': S + 299..307 'F::': fn F(PhantomData) -> F + 299..320 'F:: + 299..339 'F::(S)': (u32, i32) + 308..319 'PhantomData': PhantomData + 337..338 'S': S "#]], ); } @@ -4012,7 +4022,7 @@ fn f() { fn dyn_map() { check_types( r#" -pub struct Key {} +pub struct Key(K, V, P); pub trait Policy { type K; @@ -4024,7 +4034,7 @@ impl Policy for (K, V) { type V = V; } -pub struct KeyMap {} +pub struct KeyMap(KEY); impl KeyMap> { pub fn get(&self, key: &P::K) -> P::V { @@ -5023,7 +5033,7 @@ fn main() { 278..280 '{}': () 290..291 '_': Box + '?> 294..298 'iter': Box + 'static> - 294..310 'iter.i...iter()': Box + 'static> + 294..310 'iter.i...iter()': Box + '?> 152..156 'self': &'? mut Box 177..208 '{ ... }': Option<::Item> 191..198 'loop {}': ! From b022edce934b2fb939f876ccb78c1fb8b5bd6349 Mon Sep 17 00:00:00 2001 From: Wonko Date: Fri, 9 Jan 2026 15:17:06 +0100 Subject: [PATCH 0155/1843] add test for unstacking cognitive_complexity= attribute --- tests/ui/cognitive_complexity.rs | 25 +++++++++++++++++++++++++ tests/ui/cognitive_complexity.stderr | 18 +++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/ui/cognitive_complexity.rs b/tests/ui/cognitive_complexity.rs index 8080c6775e0b..3945028f8271 100644 --- a/tests/ui/cognitive_complexity.rs +++ b/tests/ui/cognitive_complexity.rs @@ -472,3 +472,28 @@ mod issue14422 { return; } } + +#[clippy::cognitive_complexity = "1"] +mod attribute_stacking { + fn bad() { + //~^ cognitive_complexity + if true { + println!("a"); + } + } + + #[clippy::cognitive_complexity = "2"] + fn ok() { + if true { + println!("a"); + } + } + + // should revert to cognitive_complexity = "1" + fn bad_again() { + //~^ cognitive_complexity + if true { + println!("a"); + } + } +} diff --git a/tests/ui/cognitive_complexity.stderr b/tests/ui/cognitive_complexity.stderr index 67ef4e5655bd..e5f54a37229d 100644 --- a/tests/ui/cognitive_complexity.stderr +++ b/tests/ui/cognitive_complexity.stderr @@ -176,5 +176,21 @@ LL | fn bar() { | = help: you could split it up into multiple smaller functions -error: aborting due to 22 previous errors +error: the function has a cognitive complexity of (2/1) + --> tests/ui/cognitive_complexity.rs:478:8 + | +LL | fn bad() { + | ^^^ + | + = help: you could split it up into multiple smaller functions + +error: the function has a cognitive complexity of (2/1) + --> tests/ui/cognitive_complexity.rs:493:8 + | +LL | fn bad_again() { + | ^^^^^^^^^ + | + = help: you could split it up into multiple smaller functions + +error: aborting due to 24 previous errors From 13b82eb615c68c71baf85ea8dcfbcab51a1dce1a Mon Sep 17 00:00:00 2001 From: Wonko Date: Fri, 9 Jan 2026 13:43:55 +0100 Subject: [PATCH 0156/1843] Fix LimitStack::pop_attrs in release mode The `LimitStack::pop_attrs` function used to pop from the stack in `debug_assert_eq!` and check the value. The `pop` operation was therefore only executed in debug builds, leading to an unbalanced stack in release builds when attributes were present. This change ensures the `pop` operation is always executed, by moving it out of the debug-only assertion. The assertion is kept for debug builds. changelog: fix unbalanced stack in attributes --- clippy_utils/src/attrs.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clippy_utils/src/attrs.rs b/clippy_utils/src/attrs.rs index 32f6cb4fd5e9..56490cfc8b65 100644 --- a/clippy_utils/src/attrs.rs +++ b/clippy_utils/src/attrs.rs @@ -159,7 +159,10 @@ impl LimitStack { } pub fn pop_attrs(&mut self, sess: &Session, attrs: &[impl AttributeExt], name: Symbol) { let stack = &mut self.stack; - parse_attrs(sess, attrs, name, |val| debug_assert_eq!(stack.pop(), Some(val))); + parse_attrs(sess, attrs, name, |val| { + let popped = stack.pop(); + debug_assert_eq!(popped, Some(val)); + }); } } From 0946c867a86cf3c830dbbd7073d8df7f71b3149b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Dec 2025 16:12:22 +0100 Subject: [PATCH 0157/1843] Add new `duration_suboptimal_units` lint --- CHANGELOG.md | 1 + clippy_dev/src/serve.rs | 2 +- clippy_lints/src/declared_lints.rs | 1 + clippy_lints/src/duration_suboptimal_units.rs | 204 ++++++++++++++++++ clippy_lints/src/lib.rs | 6 +- clippy_utils/src/msrvs.rs | 6 +- clippy_utils/src/sym.rs | 9 + tests/ui/duration_suboptimal_units.fixed | 91 ++++++++ tests/ui/duration_suboptimal_units.rs | 91 ++++++++ tests/ui/duration_suboptimal_units.stderr | 152 +++++++++++++ ...duration_suboptimal_units_days_weeks.fixed | 17 ++ .../duration_suboptimal_units_days_weeks.rs | 17 ++ ...uration_suboptimal_units_days_weeks.stderr | 40 ++++ 13 files changed, 633 insertions(+), 4 deletions(-) create mode 100644 clippy_lints/src/duration_suboptimal_units.rs create mode 100644 tests/ui/duration_suboptimal_units.fixed create mode 100644 tests/ui/duration_suboptimal_units.rs create mode 100644 tests/ui/duration_suboptimal_units.stderr create mode 100644 tests/ui/duration_suboptimal_units_days_weeks.fixed create mode 100644 tests/ui/duration_suboptimal_units_days_weeks.rs create mode 100644 tests/ui/duration_suboptimal_units_days_weeks.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d793489be2..4cda8003b05c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6406,6 +6406,7 @@ Released 2018-09-13 [`duplicate_mod`]: https://rust-lang.github.io/rust-clippy/master/index.html#duplicate_mod [`duplicate_underscore_argument`]: https://rust-lang.github.io/rust-clippy/master/index.html#duplicate_underscore_argument [`duplicated_attributes`]: https://rust-lang.github.io/rust-clippy/master/index.html#duplicated_attributes +[`duration_suboptimal_units`]: https://rust-lang.github.io/rust-clippy/master/index.html#duration_suboptimal_units [`duration_subsec`]: https://rust-lang.github.io/rust-clippy/master/index.html#duration_subsec [`eager_transmute`]: https://rust-lang.github.io/rust-clippy/master/index.html#eager_transmute [`elidable_lifetime_names`]: https://rust-lang.github.io/rust-clippy/master/index.html#elidable_lifetime_names diff --git a/clippy_dev/src/serve.rs b/clippy_dev/src/serve.rs index d9e018133813..b99289672420 100644 --- a/clippy_dev/src/serve.rs +++ b/clippy_dev/src/serve.rs @@ -54,7 +54,7 @@ pub fn run(port: u16, lint: Option) -> ! { } // Delay to avoid updating the metadata too aggressively. - thread::sleep(Duration::from_millis(1000)); + thread::sleep(Duration::from_secs(1)); } } diff --git a/clippy_lints/src/declared_lints.rs b/clippy_lints/src/declared_lints.rs index 6b68940c6423..ef9da84c4407 100644 --- a/clippy_lints/src/declared_lints.rs +++ b/clippy_lints/src/declared_lints.rs @@ -135,6 +135,7 @@ pub static LINTS: &[&::declare_clippy_lint::LintInfo] = &[ crate::drop_forget_ref::FORGET_NON_DROP_INFO, crate::drop_forget_ref::MEM_FORGET_INFO, crate::duplicate_mod::DUPLICATE_MOD_INFO, + crate::duration_suboptimal_units::DURATION_SUBOPTIMAL_UNITS_INFO, crate::else_if_without_else::ELSE_IF_WITHOUT_ELSE_INFO, crate::empty_drop::EMPTY_DROP_INFO, crate::empty_enums::EMPTY_ENUMS_INFO, diff --git a/clippy_lints/src/duration_suboptimal_units.rs b/clippy_lints/src/duration_suboptimal_units.rs new file mode 100644 index 000000000000..8140585b70d3 --- /dev/null +++ b/clippy_lints/src/duration_suboptimal_units.rs @@ -0,0 +1,204 @@ +use std::ops::ControlFlow; + +use clippy_config::Conf; +use clippy_utils::consts::{ConstEvalCtxt, Constant}; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::msrvs::{self, Msrv}; +use clippy_utils::res::MaybeDef; +use clippy_utils::sym; +use rustc_errors::Applicability; +use rustc_hir::{Expr, ExprKind, QPath, RustcVersion}; +use rustc_lint::{LateContext, LateLintPass, LintContext}; +use rustc_middle::ty::TyCtxt; +use rustc_session::impl_lint_pass; +use rustc_span::Symbol; + +declare_clippy_lint! { + /// ### What it does + /// + /// Checks for instances where a `std::time::Duration` is constructed using a smaller time unit + /// when the value could be expressed more clearly using a larger unit. + /// + /// ### Why is this bad? + /// + /// Using a smaller unit for a duration that is evenly divisible by a larger unit reduces + /// readability. Readers have to mentally convert values, which can be error-prone and makes + /// the code less clear. + /// + /// ### Example + /// ``` + /// use std::time::Duration; + /// + /// let dur = Duration::from_millis(5_000); + /// let dur = Duration::from_secs(180); + /// let dur = Duration::from_mins(10 * 60); + /// ``` + /// + /// Use instead: + /// ``` + /// use std::time::Duration; + /// + /// let dur = Duration::from_secs(5); + /// let dur = Duration::from_mins(3); + /// let dur = Duration::from_hours(10); + /// ``` + #[clippy::version = "1.95.0"] + pub DURATION_SUBOPTIMAL_UNITS, + pedantic, + "constructing a `Duration` using a smaller unit when a larger unit would be more readable" +} + +impl_lint_pass!(DurationSuboptimalUnits => [DURATION_SUBOPTIMAL_UNITS]); + +pub struct DurationSuboptimalUnits { + msrv: Msrv, + units: Vec, +} + +impl DurationSuboptimalUnits { + pub fn new(tcx: TyCtxt<'_>, conf: &'static Conf) -> Self { + // The order of the units matters, as they are walked top to bottom + let mut units = UNITS.to_vec(); + if tcx.features().enabled(sym::duration_constructors) { + units.extend(EXTENDED_UNITS); + } + Self { msrv: conf.msrv, units } + } +} + +impl LateLintPass<'_> for DurationSuboptimalUnits { + fn check_expr(&mut self, cx: &LateContext<'_>, expr: &'_ Expr<'_>) { + if !expr.span.in_external_macro(cx.sess().source_map()) + // Check if a function on std::time::Duration is called + && let ExprKind::Call(func, [arg]) = expr.kind + && let ExprKind::Path(QPath::TypeRelative(func_ty, func_name)) = func.kind + && cx + .typeck_results() + .node_type(func_ty.hir_id) + .is_diag_item(cx, sym::Duration) + // We intentionally don't want to evaluate referenced constants, as we don't want to + // recommend a literal value over using constants: + // + // let dur = Duration::from_secs(SIXTY); + // ^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `Duration::from_mins(1)` + && let Some(Constant::Int(value)) = ConstEvalCtxt::new(cx).eval_local(arg, expr.span.ctxt()) + && let value = u64::try_from(value).expect("All Duration::from_ constructors take a u64") + // There is no need to promote e.g. 0 seconds to 0 hours + && value != 0 + && let Some((promoted_constructor, promoted_value)) = self.promote(cx, func_name.ident.name, value) + { + span_lint_and_then( + cx, + DURATION_SUBOPTIMAL_UNITS, + expr.span, + "constructing a `Duration` using a smaller unit when a larger unit would be more readable", + |diag| { + let suggestions = vec![ + (func_name.ident.span, promoted_constructor.to_string()), + (arg.span, promoted_value.to_string()), + ]; + diag.multipart_suggestion_verbose( + format!("try using {promoted_constructor}"), + suggestions, + Applicability::MachineApplicable, + ); + }, + ); + } + } +} + +impl DurationSuboptimalUnits { + /// Tries to promote the given constructor and value to a bigger time unit and returns the + /// promoted constructor name and value. + /// + /// Returns [`None`] in case no promotion could be done. + fn promote(&self, cx: &LateContext<'_>, constructor_name: Symbol, value: u64) -> Option<(Symbol, u64)> { + let (best_unit, best_value) = self + .units + .iter() + .skip_while(|unit| unit.constructor_name != constructor_name) + .skip(1) + .try_fold( + (constructor_name, value), + |(current_unit, current_value), bigger_unit| { + if let Some(bigger_value) = current_value.div_exact(u64::from(bigger_unit.factor)) + && bigger_unit.stable_since.is_none_or(|v| self.msrv.meets(cx, v)) + { + ControlFlow::Continue((bigger_unit.constructor_name, bigger_value)) + } else { + // We have to break early, as we can't skip versions, as they are needed to + // correctly calculate the promoted value. + ControlFlow::Break((current_unit, current_value)) + } + }, + ) + .into_value(); + (best_unit != constructor_name).then_some((best_unit, best_value)) + } +} + +#[derive(Clone, Copy)] +struct Unit { + /// Name of the constructor on [`Duration`](std::time::Duration) to construct it from the given + /// unit, e.g. [`Duration::from_secs`](std::time::Duration::from_secs) + constructor_name: Symbol, + + /// The increase factor over the previous (smaller) unit + factor: u16, + + /// In what rustc version stable support for this constructor was added. + /// We do not need to track the version stable support in const contexts was added, as the const + /// stabilization was done in an ascending order of the time unites, so it's always valid to + /// promote a const constructor. + stable_since: Option, +} + +/// Time unit constructors available on stable. The order matters! +const UNITS: [Unit; 6] = [ + Unit { + constructor_name: sym::from_nanos, + // The value doesn't matter, as there is no previous unit + factor: 0, + stable_since: Some(msrvs::DURATION_FROM_NANOS_MICROS), + }, + Unit { + constructor_name: sym::from_micros, + factor: 1_000, + stable_since: Some(msrvs::DURATION_FROM_NANOS_MICROS), + }, + Unit { + constructor_name: sym::from_millis, + factor: 1_000, + stable_since: Some(msrvs::DURATION_FROM_MILLIS_SECS), + }, + Unit { + constructor_name: sym::from_secs, + factor: 1_000, + stable_since: Some(msrvs::DURATION_FROM_MILLIS_SECS), + }, + Unit { + constructor_name: sym::from_mins, + factor: 60, + stable_since: Some(msrvs::DURATION_FROM_MINUTES_HOURS), + }, + Unit { + constructor_name: sym::from_hours, + factor: 60, + stable_since: Some(msrvs::DURATION_FROM_MINUTES_HOURS), + }, +]; + +/// Time unit constructors behind the `duration_constructors` feature. The order matters! +const EXTENDED_UNITS: [Unit; 2] = [ + Unit { + constructor_name: sym::from_days, + factor: 24, + stable_since: None, + }, + Unit { + constructor_name: sym::from_weeks, + factor: 7, + stable_since: None, + }, +]; diff --git a/clippy_lints/src/lib.rs b/clippy_lints/src/lib.rs index a957afdb1910..ef2461f8b097 100644 --- a/clippy_lints/src/lib.rs +++ b/clippy_lints/src/lib.rs @@ -1,11 +1,13 @@ #![cfg_attr(bootstrap, feature(array_windows))] #![feature(box_patterns)] -#![feature(macro_metavar_expr_concat)] +#![feature(control_flow_into_value)] +#![feature(exact_div)] #![feature(f128)] #![feature(f16)] #![feature(if_let_guard)] #![feature(iter_intersperse)] #![feature(iter_partition_in_place)] +#![feature(macro_metavar_expr_concat)] #![feature(never_type)] #![feature(rustc_private)] #![feature(stmt_expr_attributes)] @@ -113,6 +115,7 @@ mod doc; mod double_parens; mod drop_forget_ref; mod duplicate_mod; +mod duration_suboptimal_units; mod else_if_without_else; mod empty_drop; mod empty_enums; @@ -857,6 +860,7 @@ pub fn register_lint_passes(store: &mut rustc_lint::LintStore, conf: &'static Co Box::new(|_| Box::::default()), Box::new(move |_| Box::new(manual_ilog2::ManualIlog2::new(conf))), Box::new(|_| Box::new(same_length_and_capacity::SameLengthAndCapacity)), + Box::new(move |tcx| Box::new(duration_suboptimal_units::DurationSuboptimalUnits::new(tcx, conf))), // add late passes here, used by `cargo dev new_lint` ]; store.late_passes.extend(late_lints); diff --git a/clippy_utils/src/msrvs.rs b/clippy_utils/src/msrvs.rs index 4a7fa3472cae..14cfb5a88283 100644 --- a/clippy_utils/src/msrvs.rs +++ b/clippy_utils/src/msrvs.rs @@ -23,6 +23,7 @@ macro_rules! msrv_aliases { // names may refer to stabilized feature flags or library items msrv_aliases! { + 1,91,0 { DURATION_FROM_MINUTES_HOURS } 1,88,0 { LET_CHAINS } 1,87,0 { OS_STR_DISPLAY, INT_MIDPOINT, CONST_CHAR_IS_DIGIT, UNSIGNED_IS_MULTIPLE_OF, INTEGER_SIGN_CAST } 1,85,0 { UINT_FLOAT_MIDPOINT, CONST_SIZE_OF_VAL } @@ -69,12 +70,12 @@ msrv_aliases! { 1,35,0 { OPTION_COPIED, RANGE_CONTAINS } 1,34,0 { TRY_FROM } 1,33,0 { UNDERSCORE_IMPORTS } - 1,32,0 { CONST_IS_POWER_OF_TWO } + 1,32,0 { CONST_IS_POWER_OF_TWO, CONST_DURATION_FROM_NANOS_MICROS_MILLIS_SECS } 1,31,0 { OPTION_REPLACE } 1,30,0 { ITERATOR_FIND_MAP, TOOL_ATTRIBUTES } 1,29,0 { ITER_FLATTEN } 1,28,0 { FROM_BOOL, REPEAT_WITH, SLICE_FROM_REF } - 1,27,0 { ITERATOR_TRY_FOLD, DOUBLE_ENDED_ITERATOR_RFIND } + 1,27,0 { ITERATOR_TRY_FOLD, DOUBLE_ENDED_ITERATOR_RFIND, DURATION_FROM_NANOS_MICROS } 1,26,0 { RANGE_INCLUSIVE, STRING_RETAIN, POINTER_ADD_SUB_METHODS } 1,24,0 { IS_ASCII_DIGIT, PTR_NULL } 1,18,0 { HASH_MAP_RETAIN, HASH_SET_RETAIN } @@ -82,6 +83,7 @@ msrv_aliases! { 1,16,0 { STR_REPEAT, RESULT_UNWRAP_OR_DEFAULT } 1,15,0 { MAYBE_BOUND_IN_WHERE } 1,13,0 { QUESTION_MARK_OPERATOR } + 1,3,0 { DURATION_FROM_MILLIS_SECS } } /// `#[clippy::msrv]` attributes are rarely used outside of Clippy's test suite, as a basic diff --git a/clippy_utils/src/sym.rs b/clippy_utils/src/sym.rs index a0d2e8673fe6..f11af159bc2e 100644 --- a/clippy_utils/src/sym.rs +++ b/clippy_utils/src/sym.rs @@ -140,6 +140,7 @@ generate! { disallowed_types, drain, dump, + duration_constructors, ends_with, enum_glob_use, enumerate, @@ -164,12 +165,20 @@ generate! { from_be_bytes, from_bytes_with_nul, from_bytes_with_nul_unchecked, + from_days, + from_hours, from_le_bytes, + from_micros, + from_millis, + from_mins, + from_nanos, from_ne_bytes, from_ptr, from_raw, from_raw_parts, + from_secs, from_str_radix, + from_weeks, fs, fuse, futures_util, diff --git a/tests/ui/duration_suboptimal_units.fixed b/tests/ui/duration_suboptimal_units.fixed new file mode 100644 index 000000000000..98c4b6e965ba --- /dev/null +++ b/tests/ui/duration_suboptimal_units.fixed @@ -0,0 +1,91 @@ +//@aux-build:proc_macros.rs +#![warn(clippy::duration_suboptimal_units)] + +use std::time::Duration; + +const SIXTY: u64 = 60; + +macro_rules! mac { + (slow_rythm) => { + 3600 + }; + (duration) => { + Duration::from_mins(5) + //~^ duration_suboptimal_units + }; + (arg => $e:expr) => { + Duration::from_secs($e) + }; +} + +fn main() { + let dur = Duration::from_secs(0); + let dur = Duration::from_secs(42); + let dur = Duration::from_hours(3); + + let dur = Duration::from_mins(1); + //~^ duration_suboptimal_units + let dur = Duration::from_mins(3); + //~^ duration_suboptimal_units + let dur = Duration::from_mins(10); + //~^ duration_suboptimal_units + let dur = Duration::from_hours(24); + //~^ duration_suboptimal_units + let dur = Duration::from_secs(5); + //~^ duration_suboptimal_units + let dur = Duration::from_hours(13); + //~^ duration_suboptimal_units + + // Constants are intentionally not resolved, as we don't want to recommend a literal value over + // using constants. + let dur = Duration::from_secs(SIXTY); + // Technically it would be nice to use Duration::from_mins(SIXTY) here, but that is a follow-up + let dur = Duration::from_secs(SIXTY * 60); + + const { + let dur = Duration::from_secs(0); + let dur = Duration::from_secs(5); + //~^ duration_suboptimal_units + let dur = Duration::from_mins(3); + //~^ duration_suboptimal_units + let dur = Duration::from_hours(24); + //~^ duration_suboptimal_units + + let dur = Duration::from_secs(SIXTY); + } + + // Qualified Durations must be kept + std::time::Duration::from_mins(1); + //~^ duration_suboptimal_units + + // We lint in normal macros + assert_eq!(Duration::from_hours(1), Duration::from_mins(6)); + //~^ duration_suboptimal_units + + // We lint in normal macros (marker is in macro itself) + let dur = mac!(duration); + + // We don't lint in macros if duration comes from outside + let dur = mac!(arg => 3600); + + // We don't lint in external macros + let dur = proc_macros::external! { Duration::from_secs(3_600) }; + + // We don't lint values coming from macros + let dur = Duration::from_secs(mac!(slow_rythm)); +} + +mod my_duration { + struct Duration {} + + impl Duration { + pub const fn from_secs(_secs: u64) -> Self { + Self {} + } + } + + fn test() { + // Only suggest the change for std::time::Duration, not for other Duration structs + let dur = Duration::from_secs(60); + } +} diff --git a/tests/ui/duration_suboptimal_units.rs b/tests/ui/duration_suboptimal_units.rs new file mode 100644 index 000000000000..c4f33a9f92e0 --- /dev/null +++ b/tests/ui/duration_suboptimal_units.rs @@ -0,0 +1,91 @@ +//@aux-build:proc_macros.rs +#![warn(clippy::duration_suboptimal_units)] + +use std::time::Duration; + +const SIXTY: u64 = 60; + +macro_rules! mac { + (slow_rythm) => { + 3600 + }; + (duration) => { + Duration::from_secs(300) + //~^ duration_suboptimal_units + }; + (arg => $e:expr) => { + Duration::from_secs($e) + }; +} + +fn main() { + let dur = Duration::from_secs(0); + let dur = Duration::from_secs(42); + let dur = Duration::from_hours(3); + + let dur = Duration::from_secs(60); + //~^ duration_suboptimal_units + let dur = Duration::from_secs(180); + //~^ duration_suboptimal_units + let dur = Duration::from_secs(10 * 60); + //~^ duration_suboptimal_units + let dur = Duration::from_mins(24 * 60); + //~^ duration_suboptimal_units + let dur = Duration::from_millis(5_000); + //~^ duration_suboptimal_units + let dur = Duration::from_nanos(13 * 60 * 60 * 1_000 * 1_000 * 1_000); + //~^ duration_suboptimal_units + + // Constants are intentionally not resolved, as we don't want to recommend a literal value over + // using constants. + let dur = Duration::from_secs(SIXTY); + // Technically it would be nice to use Duration::from_mins(SIXTY) here, but that is a follow-up + let dur = Duration::from_secs(SIXTY * 60); + + const { + let dur = Duration::from_secs(0); + let dur = Duration::from_millis(5_000); + //~^ duration_suboptimal_units + let dur = Duration::from_secs(180); + //~^ duration_suboptimal_units + let dur = Duration::from_mins(24 * 60); + //~^ duration_suboptimal_units + + let dur = Duration::from_secs(SIXTY); + } + + // Qualified Durations must be kept + std::time::Duration::from_secs(60); + //~^ duration_suboptimal_units + + // We lint in normal macros + assert_eq!(Duration::from_secs(3_600), Duration::from_mins(6)); + //~^ duration_suboptimal_units + + // We lint in normal macros (marker is in macro itself) + let dur = mac!(duration); + + // We don't lint in macros if duration comes from outside + let dur = mac!(arg => 3600); + + // We don't lint in external macros + let dur = proc_macros::external! { Duration::from_secs(3_600) }; + + // We don't lint values coming from macros + let dur = Duration::from_secs(mac!(slow_rythm)); +} + +mod my_duration { + struct Duration {} + + impl Duration { + pub const fn from_secs(_secs: u64) -> Self { + Self {} + } + } + + fn test() { + // Only suggest the change for std::time::Duration, not for other Duration structs + let dur = Duration::from_secs(60); + } +} diff --git a/tests/ui/duration_suboptimal_units.stderr b/tests/ui/duration_suboptimal_units.stderr new file mode 100644 index 000000000000..f129dbade8dc --- /dev/null +++ b/tests/ui/duration_suboptimal_units.stderr @@ -0,0 +1,152 @@ +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:26:15 + | +LL | let dur = Duration::from_secs(60); + | ^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `-D clippy::duration-suboptimal-units` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::duration_suboptimal_units)]` +help: try using from_mins + | +LL - let dur = Duration::from_secs(60); +LL + let dur = Duration::from_mins(1); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:28:15 + | +LL | let dur = Duration::from_secs(180); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_mins + | +LL - let dur = Duration::from_secs(180); +LL + let dur = Duration::from_mins(3); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:30:15 + | +LL | let dur = Duration::from_secs(10 * 60); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_mins + | +LL - let dur = Duration::from_secs(10 * 60); +LL + let dur = Duration::from_mins(10); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:32:15 + | +LL | let dur = Duration::from_mins(24 * 60); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_hours + | +LL - let dur = Duration::from_mins(24 * 60); +LL + let dur = Duration::from_hours(24); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:34:15 + | +LL | let dur = Duration::from_millis(5_000); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_secs + | +LL - let dur = Duration::from_millis(5_000); +LL + let dur = Duration::from_secs(5); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:36:15 + | +LL | let dur = Duration::from_nanos(13 * 60 * 60 * 1_000 * 1_000 * 1_000); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_hours + | +LL - let dur = Duration::from_nanos(13 * 60 * 60 * 1_000 * 1_000 * 1_000); +LL + let dur = Duration::from_hours(13); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:47:19 + | +LL | let dur = Duration::from_millis(5_000); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_secs + | +LL - let dur = Duration::from_millis(5_000); +LL + let dur = Duration::from_secs(5); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:49:19 + | +LL | let dur = Duration::from_secs(180); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_mins + | +LL - let dur = Duration::from_secs(180); +LL + let dur = Duration::from_mins(3); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:51:19 + | +LL | let dur = Duration::from_mins(24 * 60); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_hours + | +LL - let dur = Duration::from_mins(24 * 60); +LL + let dur = Duration::from_hours(24); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:58:5 + | +LL | std::time::Duration::from_secs(60); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_mins + | +LL - std::time::Duration::from_secs(60); +LL + std::time::Duration::from_mins(1); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:62:16 + | +LL | assert_eq!(Duration::from_secs(3_600), Duration::from_mins(6)); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_hours + | +LL - assert_eq!(Duration::from_secs(3_600), Duration::from_mins(6)); +LL + assert_eq!(Duration::from_hours(1), Duration::from_mins(6)); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units.rs:13:9 + | +LL | Duration::from_secs(300) + | ^^^^^^^^^^^^^^^^^^^^^^^^ +... +LL | let dur = mac!(duration); + | -------------- in this macro invocation + | + = note: this error originates in the macro `mac` (in Nightly builds, run with -Z macro-backtrace for more info) +help: try using from_mins + | +LL - Duration::from_secs(300) +LL + Duration::from_mins(5) + | + +error: aborting due to 12 previous errors + diff --git a/tests/ui/duration_suboptimal_units_days_weeks.fixed b/tests/ui/duration_suboptimal_units_days_weeks.fixed new file mode 100644 index 000000000000..b0abcbb7bf03 --- /dev/null +++ b/tests/ui/duration_suboptimal_units_days_weeks.fixed @@ -0,0 +1,17 @@ +#![warn(clippy::duration_suboptimal_units)] +// The duration_constructors feature enables `Duration::from_days` and `Duration::from_weeks`, so we +// should suggest them +#![feature(duration_constructors)] + +use std::time::Duration; + +fn main() { + let dur = Duration::from_mins(1); + //~^ duration_suboptimal_units + + let dur = Duration::from_days(1); + //~^ duration_suboptimal_units + + let dur = Duration::from_weeks(13); + //~^ duration_suboptimal_units +} diff --git a/tests/ui/duration_suboptimal_units_days_weeks.rs b/tests/ui/duration_suboptimal_units_days_weeks.rs new file mode 100644 index 000000000000..663476905c0f --- /dev/null +++ b/tests/ui/duration_suboptimal_units_days_weeks.rs @@ -0,0 +1,17 @@ +#![warn(clippy::duration_suboptimal_units)] +// The duration_constructors feature enables `Duration::from_days` and `Duration::from_weeks`, so we +// should suggest them +#![feature(duration_constructors)] + +use std::time::Duration; + +fn main() { + let dur = Duration::from_secs(60); + //~^ duration_suboptimal_units + + let dur = Duration::from_hours(24); + //~^ duration_suboptimal_units + + let dur = Duration::from_nanos(13 * 7 * 24 * 60 * 60 * 1_000 * 1_000 * 1_000); + //~^ duration_suboptimal_units +} diff --git a/tests/ui/duration_suboptimal_units_days_weeks.stderr b/tests/ui/duration_suboptimal_units_days_weeks.stderr new file mode 100644 index 000000000000..98325358bfa6 --- /dev/null +++ b/tests/ui/duration_suboptimal_units_days_weeks.stderr @@ -0,0 +1,40 @@ +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units_days_weeks.rs:9:15 + | +LL | let dur = Duration::from_secs(60); + | ^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `-D clippy::duration-suboptimal-units` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::duration_suboptimal_units)]` +help: try using from_mins + | +LL - let dur = Duration::from_secs(60); +LL + let dur = Duration::from_mins(1); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units_days_weeks.rs:12:15 + | +LL | let dur = Duration::from_hours(24); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_days + | +LL - let dur = Duration::from_hours(24); +LL + let dur = Duration::from_days(1); + | + +error: constructing a `Duration` using a smaller unit when a larger unit would be more readable + --> tests/ui/duration_suboptimal_units_days_weeks.rs:15:15 + | +LL | let dur = Duration::from_nanos(13 * 7 * 24 * 60 * 60 * 1_000 * 1_000 * 1_000); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: try using from_weeks + | +LL - let dur = Duration::from_nanos(13 * 7 * 24 * 60 * 60 * 1_000 * 1_000 * 1_000); +LL + let dur = Duration::from_weeks(13); + | + +error: aborting due to 3 previous errors + From bcf059c81eb056c39cb03958feb6baea289690b4 Mon Sep 17 00:00:00 2001 From: cry-inc Date: Fri, 9 Jan 2026 18:23:37 +0100 Subject: [PATCH 0158/1843] Fix issue with ignore attribute for tests where the attribute has a value with the reason --- src/tools/rust-analyzer/crates/hir-def/src/attrs.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/rust-analyzer/crates/hir-def/src/attrs.rs b/src/tools/rust-analyzer/crates/hir-def/src/attrs.rs index 83df11f2d2a4..0b8f65687218 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/attrs.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/attrs.rs @@ -135,6 +135,7 @@ fn match_attr_flags(attr_flags: &mut AttrFlags, attr: Meta) -> ControlFlow match name.text() { "deprecated" => attr_flags.insert(AttrFlags::IS_DEPRECATED), + "ignore" => attr_flags.insert(AttrFlags::IS_IGNORE), "lang" => attr_flags.insert(AttrFlags::LANG_ITEM), "path" => attr_flags.insert(AttrFlags::HAS_PATH), "unstable" => attr_flags.insert(AttrFlags::IS_UNSTABLE), From f73e504eea0a2cdd90fa1e27b8faa1ffd93ed370 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Fri, 9 Jan 2026 20:20:35 +0100 Subject: [PATCH 0159/1843] fix(useless_conversion): respect reduced applicability --- clippy_lints/src/useless_conversion.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clippy_lints/src/useless_conversion.rs b/clippy_lints/src/useless_conversion.rs index 423301edfe83..4ce132e9e3ab 100644 --- a/clippy_lints/src/useless_conversion.rs +++ b/clippy_lints/src/useless_conversion.rs @@ -365,7 +365,7 @@ impl<'tcx> LateLintPass<'tcx> for UselessConversion { format!("useless conversion to the same type: `{b}`"), "consider removing `.into_iter()`", sugg, - Applicability::MachineApplicable, // snippet + applicability, ); } } From 3ec7666a2ac99ccfa9de268a89d855ceb380024c Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Fri, 9 Jan 2026 20:43:26 +0100 Subject: [PATCH 0160/1843] clean-up --- clippy_lints/src/int_plus_one.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clippy_lints/src/int_plus_one.rs b/clippy_lints/src/int_plus_one.rs index 67ce57de254d..540f9bb1e260 100644 --- a/clippy_lints/src/int_plus_one.rs +++ b/clippy_lints/src/int_plus_one.rs @@ -100,7 +100,7 @@ impl IntPlusOne { _ => None, } }, - // case where `... >= y - 1` or `... >= -1 + y` + // case where `... <= y - 1` or `... <= -1 + y` (BinOpKind::Le, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) => { match (rhskind.node, &rhslhs.kind, &rhsrhs.kind) { // `-1 + y` @@ -156,11 +156,11 @@ impl IntPlusOne { } impl EarlyLintPass for IntPlusOne { - fn check_expr(&mut self, cx: &EarlyContext<'_>, item: &Expr) { - if let ExprKind::Binary(ref kind, ref lhs, ref rhs) = item.kind + fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &Expr) { + if let ExprKind::Binary(kind, lhs, rhs) = &expr.kind && let Some(rec) = Self::check_binop(cx, kind.node, lhs, rhs) { - Self::emit_warning(cx, item, rec); + Self::emit_warning(cx, expr, rec); } } } From 0180ec4d5d9e47b1b36bf33d783ac72600c4abd4 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Fri, 9 Jan 2026 22:38:08 +0100 Subject: [PATCH 0161/1843] fix: negative literals are not literals --- clippy_lints/src/int_plus_one.rs | 67 ++++++++++++++++++-------------- tests/ui/int_plus_one.fixed | 16 ++++---- tests/ui/int_plus_one.rs | 16 ++++---- tests/ui/int_plus_one.stderr | 26 ++++++++++++- 4 files changed, 78 insertions(+), 47 deletions(-) diff --git a/clippy_lints/src/int_plus_one.rs b/clippy_lints/src/int_plus_one.rs index 540f9bb1e260..c25821751b3f 100644 --- a/clippy_lints/src/int_plus_one.rs +++ b/clippy_lints/src/int_plus_one.rs @@ -1,7 +1,7 @@ use clippy_utils::diagnostics::span_lint_and_sugg; use clippy_utils::source::SpanRangeExt; -use rustc_ast::ast::{BinOpKind, Expr, ExprKind, LitKind}; -use rustc_ast::token; +use rustc_ast::ast::{BinOpKind, Expr, ExprKind, LitKind, UnOp}; +use rustc_data_structures::packed::Pu128; use rustc_errors::Applicability; use rustc_lint::{EarlyContext, EarlyLintPass}; use rustc_session::declare_lint_pass; @@ -50,25 +50,36 @@ enum Side { } impl IntPlusOne { - #[expect(clippy::cast_sign_loss)] - fn check_lit(token_lit: token::Lit, target_value: i128) -> bool { - if let Ok(LitKind::Int(value, ..)) = LitKind::from_token_lit(token_lit) { - return value == (target_value as u128); + fn is_one(expr: &Expr) -> bool { + if let ExprKind::Lit(token_lit) = expr.kind + && let Ok(LitKind::Int(Pu128(1), ..)) = LitKind::from_token_lit(token_lit) + { + return true; } false } + fn is_neg_one(expr: &Expr) -> bool { + if let ExprKind::Unary(UnOp::Neg, expr) = &expr.kind + && Self::is_one(expr) + { + true + } else { + false + } + } + fn check_binop(cx: &EarlyContext<'_>, binop: BinOpKind, lhs: &Expr, rhs: &Expr) -> Option { match (binop, &lhs.kind, &rhs.kind) { // case where `x - 1 >= ...` or `-1 + x >= ...` (BinOpKind::Ge, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) => { - match (lhskind.node, &lhslhs.kind, &lhsrhs.kind) { + match lhskind.node { // `-1 + x` - (BinOpKind::Add, ExprKind::Lit(lit), _) if Self::check_lit(*lit, -1) => { + BinOpKind::Add if Self::is_neg_one(lhslhs) => { Self::generate_recommendation(cx, binop, lhsrhs, rhs, Side::Lhs) }, // `x - 1` - (BinOpKind::Sub, _, ExprKind::Lit(lit)) if Self::check_lit(*lit, 1) => { + BinOpKind::Sub if Self::is_one(lhsrhs) => { Self::generate_recommendation(cx, binop, lhslhs, rhs, Side::Lhs) }, _ => None, @@ -76,39 +87,35 @@ impl IntPlusOne { }, // case where `... >= y + 1` or `... >= 1 + y` (BinOpKind::Ge, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) if rhskind.node == BinOpKind::Add => { - match (&rhslhs.kind, &rhsrhs.kind) { - // `y + 1` and `1 + y` - (ExprKind::Lit(lit), _) if Self::check_lit(*lit, 1) => { - Self::generate_recommendation(cx, binop, rhsrhs, lhs, Side::Rhs) - }, - (_, ExprKind::Lit(lit)) if Self::check_lit(*lit, 1) => { - Self::generate_recommendation(cx, binop, rhslhs, lhs, Side::Rhs) - }, - _ => None, + // `y + 1` and `1 + y` + if Self::is_one(rhslhs) { + Self::generate_recommendation(cx, binop, rhsrhs, lhs, Side::Rhs) + } else if Self::is_one(rhsrhs) { + Self::generate_recommendation(cx, binop, rhslhs, lhs, Side::Rhs) + } else { + None } }, // case where `x + 1 <= ...` or `1 + x <= ...` (BinOpKind::Le, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) if lhskind.node == BinOpKind::Add => { - match (&lhslhs.kind, &lhsrhs.kind) { - // `1 + x` and `x + 1` - (ExprKind::Lit(lit), _) if Self::check_lit(*lit, 1) => { - Self::generate_recommendation(cx, binop, lhsrhs, rhs, Side::Lhs) - }, - (_, ExprKind::Lit(lit)) if Self::check_lit(*lit, 1) => { - Self::generate_recommendation(cx, binop, lhslhs, rhs, Side::Lhs) - }, - _ => None, + // `1 + x` and `x + 1` + if Self::is_one(lhslhs) { + Self::generate_recommendation(cx, binop, lhsrhs, rhs, Side::Lhs) + } else if Self::is_one(lhsrhs) { + Self::generate_recommendation(cx, binop, lhslhs, rhs, Side::Lhs) + } else { + None } }, // case where `... <= y - 1` or `... <= -1 + y` (BinOpKind::Le, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) => { - match (rhskind.node, &rhslhs.kind, &rhsrhs.kind) { + match rhskind.node { // `-1 + y` - (BinOpKind::Add, ExprKind::Lit(lit), _) if Self::check_lit(*lit, -1) => { + BinOpKind::Add if Self::is_neg_one(rhslhs) => { Self::generate_recommendation(cx, binop, rhsrhs, lhs, Side::Rhs) }, // `y - 1` - (BinOpKind::Sub, _, ExprKind::Lit(lit)) if Self::check_lit(*lit, 1) => { + BinOpKind::Sub if Self::is_one(rhsrhs) => { Self::generate_recommendation(cx, binop, rhslhs, lhs, Side::Rhs) }, _ => None, diff --git a/tests/ui/int_plus_one.fixed b/tests/ui/int_plus_one.fixed index a1aba6bf7f64..cdd19515e9a7 100644 --- a/tests/ui/int_plus_one.fixed +++ b/tests/ui/int_plus_one.fixed @@ -4,15 +4,15 @@ fn main() { let x = 1i32; let y = 0i32; - let _ = x > y; - //~^ int_plus_one - let _ = y < x; - //~^ int_plus_one + let _ = x > y; //~ int_plus_one + let _ = x > y; //~ int_plus_one + let _ = y < x; //~ int_plus_one + let _ = y < x; //~ int_plus_one - let _ = x > y; - //~^ int_plus_one - let _ = y < x; - //~^ int_plus_one + let _ = x > y; //~ int_plus_one + let _ = x > y; //~ int_plus_one + let _ = y < x; //~ int_plus_one + let _ = y < x; //~ int_plus_one let _ = x > y; // should be ok let _ = y < x; // should be ok diff --git a/tests/ui/int_plus_one.rs b/tests/ui/int_plus_one.rs index f804fc9047de..8d7d2e8982d8 100644 --- a/tests/ui/int_plus_one.rs +++ b/tests/ui/int_plus_one.rs @@ -4,15 +4,15 @@ fn main() { let x = 1i32; let y = 0i32; - let _ = x >= y + 1; - //~^ int_plus_one - let _ = y + 1 <= x; - //~^ int_plus_one + let _ = x >= y + 1; //~ int_plus_one + let _ = x >= 1 + y; //~ int_plus_one + let _ = y + 1 <= x; //~ int_plus_one + let _ = 1 + y <= x; //~ int_plus_one - let _ = x - 1 >= y; - //~^ int_plus_one - let _ = y <= x - 1; - //~^ int_plus_one + let _ = x - 1 >= y; //~ int_plus_one + let _ = -1 + x >= y; //~ int_plus_one + let _ = y <= x - 1; //~ int_plus_one + let _ = y <= -1 + x; //~ int_plus_one let _ = x > y; // should be ok let _ = y < x; // should be ok diff --git a/tests/ui/int_plus_one.stderr b/tests/ui/int_plus_one.stderr index 052706028141..8bdff5680bdc 100644 --- a/tests/ui/int_plus_one.stderr +++ b/tests/ui/int_plus_one.stderr @@ -7,23 +7,47 @@ LL | let _ = x >= y + 1; = note: `-D clippy::int-plus-one` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::int_plus_one)]` +error: unnecessary `>= y + 1` or `x - 1 >=` + --> tests/ui/int_plus_one.rs:8:13 + | +LL | let _ = x >= 1 + y; + | ^^^^^^^^^^ help: change it to: `x > y` + error: unnecessary `>= y + 1` or `x - 1 >=` --> tests/ui/int_plus_one.rs:9:13 | LL | let _ = y + 1 <= x; | ^^^^^^^^^^ help: change it to: `y < x` +error: unnecessary `>= y + 1` or `x - 1 >=` + --> tests/ui/int_plus_one.rs:10:13 + | +LL | let _ = 1 + y <= x; + | ^^^^^^^^^^ help: change it to: `y < x` + error: unnecessary `>= y + 1` or `x - 1 >=` --> tests/ui/int_plus_one.rs:12:13 | LL | let _ = x - 1 >= y; | ^^^^^^^^^^ help: change it to: `x > y` +error: unnecessary `>= y + 1` or `x - 1 >=` + --> tests/ui/int_plus_one.rs:13:13 + | +LL | let _ = -1 + x >= y; + | ^^^^^^^^^^^ help: change it to: `x > y` + error: unnecessary `>= y + 1` or `x - 1 >=` --> tests/ui/int_plus_one.rs:14:13 | LL | let _ = y <= x - 1; | ^^^^^^^^^^ help: change it to: `y < x` -error: aborting due to 4 previous errors +error: unnecessary `>= y + 1` or `x - 1 >=` + --> tests/ui/int_plus_one.rs:15:13 + | +LL | let _ = y <= -1 + x; + | ^^^^^^^^^^^ help: change it to: `y < x` + +error: aborting due to 8 previous errors From df65d1a6a8509f9167078aab372f59ac7762af6c Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Fri, 9 Jan 2026 20:39:42 +0100 Subject: [PATCH 0162/1843] introduce `LeOrGe` removes the need for a wildcard match --- clippy_lints/src/int_plus_one.rs | 63 ++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/clippy_lints/src/int_plus_one.rs b/clippy_lints/src/int_plus_one.rs index c25821751b3f..8143dbad79e8 100644 --- a/clippy_lints/src/int_plus_one.rs +++ b/clippy_lints/src/int_plus_one.rs @@ -35,11 +35,11 @@ declare_clippy_lint! { declare_lint_pass!(IntPlusOne => [INT_PLUS_ONE]); // cases: -// BinOpKind::Ge +// LeOrGe::Ge // x >= y + 1 // x - 1 >= y // -// BinOpKind::Le +// LeOrGe::Le // x + 1 <= y // x <= y - 1 @@ -49,6 +49,23 @@ enum Side { Rhs, } +#[derive(Copy, Clone)] +enum LeOrGe { + Le, + Ge, +} + +impl TryFrom for LeOrGe { + type Error = (); + fn try_from(value: BinOpKind) -> Result { + match value { + BinOpKind::Le => Ok(Self::Le), + BinOpKind::Ge => Ok(Self::Ge), + _ => Err(()), + } + } +} + impl IntPlusOne { fn is_one(expr: &Expr) -> bool { if let ExprKind::Lit(token_lit) = expr.kind @@ -69,54 +86,54 @@ impl IntPlusOne { } } - fn check_binop(cx: &EarlyContext<'_>, binop: BinOpKind, lhs: &Expr, rhs: &Expr) -> Option { - match (binop, &lhs.kind, &rhs.kind) { + fn check_binop(cx: &EarlyContext<'_>, le_or_ge: LeOrGe, lhs: &Expr, rhs: &Expr) -> Option { + match (le_or_ge, &lhs.kind, &rhs.kind) { // case where `x - 1 >= ...` or `-1 + x >= ...` - (BinOpKind::Ge, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) => { + (LeOrGe::Ge, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) => { match lhskind.node { // `-1 + x` BinOpKind::Add if Self::is_neg_one(lhslhs) => { - Self::generate_recommendation(cx, binop, lhsrhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs, Side::Lhs) }, // `x - 1` BinOpKind::Sub if Self::is_one(lhsrhs) => { - Self::generate_recommendation(cx, binop, lhslhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs, Side::Lhs) }, _ => None, } }, // case where `... >= y + 1` or `... >= 1 + y` - (BinOpKind::Ge, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) if rhskind.node == BinOpKind::Add => { + (LeOrGe::Ge, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) if rhskind.node == BinOpKind::Add => { // `y + 1` and `1 + y` if Self::is_one(rhslhs) { - Self::generate_recommendation(cx, binop, rhsrhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, rhsrhs, lhs, Side::Rhs) } else if Self::is_one(rhsrhs) { - Self::generate_recommendation(cx, binop, rhslhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, rhslhs, lhs, Side::Rhs) } else { None } }, // case where `x + 1 <= ...` or `1 + x <= ...` - (BinOpKind::Le, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) if lhskind.node == BinOpKind::Add => { + (LeOrGe::Le, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) if lhskind.node == BinOpKind::Add => { // `1 + x` and `x + 1` if Self::is_one(lhslhs) { - Self::generate_recommendation(cx, binop, lhsrhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs, Side::Lhs) } else if Self::is_one(lhsrhs) { - Self::generate_recommendation(cx, binop, lhslhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs, Side::Lhs) } else { None } }, // case where `... <= y - 1` or `... <= -1 + y` - (BinOpKind::Le, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) => { + (LeOrGe::Le, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) => { match rhskind.node { // `-1 + y` BinOpKind::Add if Self::is_neg_one(rhslhs) => { - Self::generate_recommendation(cx, binop, rhsrhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, rhsrhs, lhs, Side::Rhs) }, // `y - 1` BinOpKind::Sub if Self::is_one(rhsrhs) => { - Self::generate_recommendation(cx, binop, rhslhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, rhslhs, lhs, Side::Rhs) }, _ => None, } @@ -127,15 +144,14 @@ impl IntPlusOne { fn generate_recommendation( cx: &EarlyContext<'_>, - binop: BinOpKind, + le_or_ge: LeOrGe, node: &Expr, other_side: &Expr, side: Side, ) -> Option { - let binop_string = match binop { - BinOpKind::Ge => ">", - BinOpKind::Le => "<", - _ => return None, + let binop_string = match le_or_ge { + LeOrGe::Ge => ">", + LeOrGe::Le => "<", }; if let Some(snippet) = node.span.get_source_text(cx) && let Some(other_side_snippet) = other_side.span.get_source_text(cx) @@ -164,8 +180,9 @@ impl IntPlusOne { impl EarlyLintPass for IntPlusOne { fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &Expr) { - if let ExprKind::Binary(kind, lhs, rhs) = &expr.kind - && let Some(rec) = Self::check_binop(cx, kind.node, lhs, rhs) + if let ExprKind::Binary(binop, lhs, rhs) = &expr.kind + && let Ok(le_or_ge) = LeOrGe::try_from(binop.node) + && let Some(rec) = Self::check_binop(cx, le_or_ge, lhs, rhs) { Self::emit_warning(cx, expr, rec); } From d554092c27f0ad5cf8dc013b4fbcbd7853c40c0c Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Fri, 9 Jan 2026 21:01:55 +0100 Subject: [PATCH 0163/1843] get rid of `Side` --- clippy_lints/src/int_plus_one.rs | 33 +++++++++----------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/clippy_lints/src/int_plus_one.rs b/clippy_lints/src/int_plus_one.rs index 8143dbad79e8..97b7afb61d86 100644 --- a/clippy_lints/src/int_plus_one.rs +++ b/clippy_lints/src/int_plus_one.rs @@ -43,12 +43,6 @@ declare_lint_pass!(IntPlusOne => [INT_PLUS_ONE]); // x + 1 <= y // x <= y - 1 -#[derive(Copy, Clone)] -enum Side { - Lhs, - Rhs, -} - #[derive(Copy, Clone)] enum LeOrGe { Le, @@ -93,12 +87,10 @@ impl IntPlusOne { match lhskind.node { // `-1 + x` BinOpKind::Add if Self::is_neg_one(lhslhs) => { - Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs) }, // `x - 1` - BinOpKind::Sub if Self::is_one(lhsrhs) => { - Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs, Side::Lhs) - }, + BinOpKind::Sub if Self::is_one(lhsrhs) => Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs), _ => None, } }, @@ -106,9 +98,9 @@ impl IntPlusOne { (LeOrGe::Ge, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) if rhskind.node == BinOpKind::Add => { // `y + 1` and `1 + y` if Self::is_one(rhslhs) { - Self::generate_recommendation(cx, le_or_ge, rhsrhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, lhs, rhsrhs) } else if Self::is_one(rhsrhs) { - Self::generate_recommendation(cx, le_or_ge, rhslhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, lhs, rhslhs) } else { None } @@ -117,9 +109,9 @@ impl IntPlusOne { (LeOrGe::Le, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) if lhskind.node == BinOpKind::Add => { // `1 + x` and `x + 1` if Self::is_one(lhslhs) { - Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs) } else if Self::is_one(lhsrhs) { - Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs, Side::Lhs) + Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs) } else { None } @@ -129,12 +121,10 @@ impl IntPlusOne { match rhskind.node { // `-1 + y` BinOpKind::Add if Self::is_neg_one(rhslhs) => { - Self::generate_recommendation(cx, le_or_ge, rhsrhs, lhs, Side::Rhs) + Self::generate_recommendation(cx, le_or_ge, lhs, rhsrhs) }, // `y - 1` - BinOpKind::Sub if Self::is_one(rhsrhs) => { - Self::generate_recommendation(cx, le_or_ge, rhslhs, lhs, Side::Rhs) - }, + BinOpKind::Sub if Self::is_one(rhsrhs) => Self::generate_recommendation(cx, le_or_ge, lhs, rhslhs), _ => None, } }, @@ -147,7 +137,6 @@ impl IntPlusOne { le_or_ge: LeOrGe, node: &Expr, other_side: &Expr, - side: Side, ) -> Option { let binop_string = match le_or_ge { LeOrGe::Ge => ">", @@ -156,11 +145,7 @@ impl IntPlusOne { if let Some(snippet) = node.span.get_source_text(cx) && let Some(other_side_snippet) = other_side.span.get_source_text(cx) { - let rec = match side { - Side::Lhs => Some(format!("{snippet} {binop_string} {other_side_snippet}")), - Side::Rhs => Some(format!("{other_side_snippet} {binop_string} {snippet}")), - }; - return rec; + return Some(format!("{snippet} {binop_string} {other_side_snippet}")); } None } From ccaa2fa9841447092bfe96111629747a27fa419a Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Fri, 9 Jan 2026 20:30:20 +0100 Subject: [PATCH 0164/1843] fix: respect reduced applicability By building the whole suggestion in `emit_suggestion`, we avoid the need to thread `Applicability` through `check_binop` into `generate_suggestion` --- clippy_lints/src/int_plus_one.rs | 69 +++++++++++++------------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/clippy_lints/src/int_plus_one.rs b/clippy_lints/src/int_plus_one.rs index 97b7afb61d86..b82ca0205a9f 100644 --- a/clippy_lints/src/int_plus_one.rs +++ b/clippy_lints/src/int_plus_one.rs @@ -1,5 +1,5 @@ -use clippy_utils::diagnostics::span_lint_and_sugg; -use clippy_utils::source::SpanRangeExt; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::sugg; use rustc_ast::ast::{BinOpKind, Expr, ExprKind, LitKind, UnOp}; use rustc_data_structures::packed::Pu128; use rustc_errors::Applicability; @@ -80,17 +80,15 @@ impl IntPlusOne { } } - fn check_binop(cx: &EarlyContext<'_>, le_or_ge: LeOrGe, lhs: &Expr, rhs: &Expr) -> Option { + fn check_binop<'tcx>(le_or_ge: LeOrGe, lhs: &'tcx Expr, rhs: &'tcx Expr) -> Option<(&'tcx Expr, &'tcx Expr)> { match (le_or_ge, &lhs.kind, &rhs.kind) { // case where `x - 1 >= ...` or `-1 + x >= ...` (LeOrGe::Ge, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) => { match lhskind.node { // `-1 + x` - BinOpKind::Add if Self::is_neg_one(lhslhs) => { - Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs) - }, + BinOpKind::Add if Self::is_neg_one(lhslhs) => Some((lhsrhs, rhs)), // `x - 1` - BinOpKind::Sub if Self::is_one(lhsrhs) => Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs), + BinOpKind::Sub if Self::is_one(lhsrhs) => Some((lhslhs, rhs)), _ => None, } }, @@ -98,9 +96,9 @@ impl IntPlusOne { (LeOrGe::Ge, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) if rhskind.node == BinOpKind::Add => { // `y + 1` and `1 + y` if Self::is_one(rhslhs) { - Self::generate_recommendation(cx, le_or_ge, lhs, rhsrhs) + Some((lhs, rhsrhs)) } else if Self::is_one(rhsrhs) { - Self::generate_recommendation(cx, le_or_ge, lhs, rhslhs) + Some((lhs, rhslhs)) } else { None } @@ -109,9 +107,9 @@ impl IntPlusOne { (LeOrGe::Le, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) if lhskind.node == BinOpKind::Add => { // `1 + x` and `x + 1` if Self::is_one(lhslhs) { - Self::generate_recommendation(cx, le_or_ge, lhsrhs, rhs) + Some((lhsrhs, rhs)) } else if Self::is_one(lhsrhs) { - Self::generate_recommendation(cx, le_or_ge, lhslhs, rhs) + Some((lhslhs, rhs)) } else { None } @@ -120,11 +118,9 @@ impl IntPlusOne { (LeOrGe::Le, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) => { match rhskind.node { // `-1 + y` - BinOpKind::Add if Self::is_neg_one(rhslhs) => { - Self::generate_recommendation(cx, le_or_ge, lhs, rhsrhs) - }, + BinOpKind::Add if Self::is_neg_one(rhslhs) => Some((lhs, rhsrhs)), // `y - 1` - BinOpKind::Sub if Self::is_one(rhsrhs) => Self::generate_recommendation(cx, le_or_ge, lhs, rhslhs), + BinOpKind::Sub if Self::is_one(rhsrhs) => Some((lhs, rhslhs)), _ => None, } }, @@ -132,33 +128,24 @@ impl IntPlusOne { } } - fn generate_recommendation( - cx: &EarlyContext<'_>, - le_or_ge: LeOrGe, - node: &Expr, - other_side: &Expr, - ) -> Option { - let binop_string = match le_or_ge { - LeOrGe::Ge => ">", - LeOrGe::Le => "<", - }; - if let Some(snippet) = node.span.get_source_text(cx) - && let Some(other_side_snippet) = other_side.span.get_source_text(cx) - { - return Some(format!("{snippet} {binop_string} {other_side_snippet}")); - } - None - } - - fn emit_warning(cx: &EarlyContext<'_>, block: &Expr, recommendation: String) { - span_lint_and_sugg( + fn emit_warning(cx: &EarlyContext<'_>, expr: &Expr, new_lhs: &Expr, le_or_ge: LeOrGe, new_rhs: &Expr) { + span_lint_and_then( cx, INT_PLUS_ONE, - block.span, + expr.span, "unnecessary `>= y + 1` or `x - 1 >=`", - "change it to", - recommendation, - Applicability::MachineApplicable, // snippet + |diag| { + let mut app = Applicability::MachineApplicable; + let ctxt = expr.span.ctxt(); + let new_lhs = sugg::Sugg::ast(cx, new_lhs, "_", ctxt, &mut app); + let new_rhs = sugg::Sugg::ast(cx, new_rhs, "_", ctxt, &mut app); + let new_binop = match le_or_ge { + LeOrGe::Ge => BinOpKind::Gt, + LeOrGe::Le => BinOpKind::Lt, + }; + let rec = sugg::make_binop(new_binop, &new_lhs, &new_rhs); + diag.span_suggestion(expr.span, "change it to", rec, app); + }, ); } } @@ -167,9 +154,9 @@ impl EarlyLintPass for IntPlusOne { fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &Expr) { if let ExprKind::Binary(binop, lhs, rhs) = &expr.kind && let Ok(le_or_ge) = LeOrGe::try_from(binop.node) - && let Some(rec) = Self::check_binop(cx, le_or_ge, lhs, rhs) + && let Some((new_lhs, new_rhs)) = Self::check_binop(le_or_ge, lhs, rhs) { - Self::emit_warning(cx, expr, rec); + Self::emit_warning(cx, expr, new_lhs, le_or_ge, new_rhs); } } } From ea591b55f43483bacb2e8b2b1cb9f6c7aa999d7e Mon Sep 17 00:00:00 2001 From: Andre Bogus Date: Fri, 9 Jan 2026 23:15:06 +0100 Subject: [PATCH 0165/1843] fix `clippy_utils::std_or_core(_)` --- clippy_utils/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clippy_utils/src/lib.rs b/clippy_utils/src/lib.rs index 2a620e917228..33f5f5c8873d 100644 --- a/clippy_utils/src/lib.rs +++ b/clippy_utils/src/lib.rs @@ -2031,12 +2031,12 @@ pub fn is_expr_temporary_value(cx: &LateContext<'_>, expr: &Expr<'_>) -> bool { } pub fn std_or_core(cx: &LateContext<'_>) -> Option<&'static str> { - if !is_no_std_crate(cx) { - Some("std") - } else if !is_no_core_crate(cx) { + if is_no_core_crate(cx) { + None + } else if is_no_std_crate(cx) { Some("core") } else { - None + Some("std") } } From 3f51a315e0668ff1c72459a08dfc42780032918c Mon Sep 17 00:00:00 2001 From: Paul Mabileau Date: Fri, 9 Jan 2026 21:20:39 +0100 Subject: [PATCH 0166/1843] Fix(lib/win/net): Remove hostname support under Win7 `GetHostNameW` is not available under Windows 7, leading to dynamic linking failures upon program executions. For now, as it is still unstable, this therefore appropriately cfg-gates the feature in order to mark the Win7 as unsupported with regards to this particular feature. Porting the functionality for Windows 7 would require changing the underlying system call and so more work for the immediate need. Signed-off-by: Paul Mabileau --- library/std/src/net/hostname.rs | 8 ++++---- library/std/src/sys/net/hostname/mod.rs | 3 ++- library/std/src/sys/pal/windows/c.rs | 4 ++++ library/std/src/sys/pal/windows/c/bindings.txt | 1 - library/std/src/sys/pal/windows/c/windows_sys.rs | 1 - 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/library/std/src/net/hostname.rs b/library/std/src/net/hostname.rs index b1010cec6005..4042496d534f 100644 --- a/library/std/src/net/hostname.rs +++ b/library/std/src/net/hostname.rs @@ -8,10 +8,10 @@ use crate::ffi::OsString; /// /// # Underlying system calls /// -/// | Platform | System call | -/// |----------|---------------------------------------------------------------------------------------------------------| -/// | UNIX | [`gethostname`](https://www.man7.org/linux/man-pages/man2/gethostname.2.html) | -/// | Windows | [`GetHostNameW`](https://learn.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-gethostnamew) | +/// | Platform | System call | +/// |--------------|---------------------------------------------------------------------------------------------------------| +/// | UNIX | [`gethostname`](https://www.man7.org/linux/man-pages/man2/gethostname.2.html) | +/// | Windows (8+) | [`GetHostNameW`](https://learn.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-gethostnamew) | /// /// Note that platform-specific behavior [may change in the future][changes]. /// diff --git a/library/std/src/sys/net/hostname/mod.rs b/library/std/src/sys/net/hostname/mod.rs index 8ffe4894d718..65fcd6bfb00d 100644 --- a/library/std/src/sys/net/hostname/mod.rs +++ b/library/std/src/sys/net/hostname/mod.rs @@ -3,7 +3,8 @@ cfg_select! { mod unix; pub use unix::hostname; } - target_os = "windows" => { + // `GetHostNameW` is only available starting with Windows 8. + all(target_os = "windows", not(target_vendor = "win7")) => { mod windows; pub use windows::hostname; } diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs index 25c1a82cc426..0f54f80d72ee 100644 --- a/library/std/src/sys/pal/windows/c.rs +++ b/library/std/src/sys/pal/windows/c.rs @@ -237,3 +237,7 @@ cfg_select! { } _ => {} } + +// Only available starting with Windows 8. +#[cfg(not(target_vendor = "win7"))] +windows_targets::link!("ws2_32.dll" "system" fn GetHostNameW(name : PWSTR, namelen : i32) -> i32); diff --git a/library/std/src/sys/pal/windows/c/bindings.txt b/library/std/src/sys/pal/windows/c/bindings.txt index 9009aa09f48e..12babcb84ccf 100644 --- a/library/std/src/sys/pal/windows/c/bindings.txt +++ b/library/std/src/sys/pal/windows/c/bindings.txt @@ -2170,7 +2170,6 @@ GetFileType GETFINALPATHNAMEBYHANDLE_FLAGS GetFinalPathNameByHandleW GetFullPathNameW -GetHostNameW GetLastError GetModuleFileNameW GetModuleHandleA diff --git a/library/std/src/sys/pal/windows/c/windows_sys.rs b/library/std/src/sys/pal/windows/c/windows_sys.rs index 98f277b33780..edc9d2d11f7c 100644 --- a/library/std/src/sys/pal/windows/c/windows_sys.rs +++ b/library/std/src/sys/pal/windows/c/windows_sys.rs @@ -49,7 +49,6 @@ windows_targets::link!("kernel32.dll" "system" fn GetFileSizeEx(hfile : HANDLE, windows_targets::link!("kernel32.dll" "system" fn GetFileType(hfile : HANDLE) -> FILE_TYPE); windows_targets::link!("kernel32.dll" "system" fn GetFinalPathNameByHandleW(hfile : HANDLE, lpszfilepath : PWSTR, cchfilepath : u32, dwflags : GETFINALPATHNAMEBYHANDLE_FLAGS) -> u32); windows_targets::link!("kernel32.dll" "system" fn GetFullPathNameW(lpfilename : PCWSTR, nbufferlength : u32, lpbuffer : PWSTR, lpfilepart : *mut PWSTR) -> u32); -windows_targets::link!("ws2_32.dll" "system" fn GetHostNameW(name : PWSTR, namelen : i32) -> i32); windows_targets::link!("kernel32.dll" "system" fn GetLastError() -> WIN32_ERROR); windows_targets::link!("kernel32.dll" "system" fn GetModuleFileNameW(hmodule : HMODULE, lpfilename : PWSTR, nsize : u32) -> u32); windows_targets::link!("kernel32.dll" "system" fn GetModuleHandleA(lpmodulename : PCSTR) -> HMODULE); From f4a6e4f0d94670e162370c2a495b0f1a8bb43262 Mon Sep 17 00:00:00 2001 From: reddevilmidzy Date: Thu, 8 Jan 2026 12:19:43 +0900 Subject: [PATCH 0167/1843] Fix clippy --- clippy_lints/src/utils/author.rs | 1 + clippy_utils/src/consts.rs | 1 + clippy_utils/src/hir_utils.rs | 11 +++++++++++ 3 files changed, 13 insertions(+) diff --git a/clippy_lints/src/utils/author.rs b/clippy_lints/src/utils/author.rs index acea701b2e83..8243077cab89 100644 --- a/clippy_lints/src/utils/author.rs +++ b/clippy_lints/src/utils/author.rs @@ -321,6 +321,7 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { }, ConstArgKind::Struct(..) => chain!(self, "let ConstArgKind::Struct(..) = {const_arg}.kind"), ConstArgKind::TupleCall(..) => chain!(self, "let ConstArgKind::TupleCall(..) = {const_arg}.kind"), + ConstArgKind::Array(..) => chain!(self, "let ConstArgKind::Array(..) = {const_arg}.kind"), ConstArgKind::Infer(..) => chain!(self, "let ConstArgKind::Infer(..) = {const_arg}.kind"), ConstArgKind::Error(..) => chain!(self, "let ConstArgKind::Error(..) = {const_arg}.kind"), ConstArgKind::Tup(..) => chain!(self, "let ConstArgKind::Tup(..) = {const_arg}.kind"), diff --git a/clippy_utils/src/consts.rs b/clippy_utils/src/consts.rs index 538f1fd2628c..e2ada37d53b3 100644 --- a/clippy_utils/src/consts.rs +++ b/clippy_utils/src/consts.rs @@ -1144,6 +1144,7 @@ pub fn const_item_rhs_to_expr<'tcx>(tcx: TyCtxt<'tcx>, ct_rhs: ConstItemRhs<'tcx | ConstArgKind::Tup(..) | ConstArgKind::Literal(..) | ConstArgKind::TupleCall(..) + | ConstArgKind::Array(..) | ConstArgKind::Path(_) | ConstArgKind::Error(..) | ConstArgKind::Infer(..) => None, diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs index 0bb641568879..c7bb3a064a09 100644 --- a/clippy_utils/src/hir_utils.rs +++ b/clippy_utils/src/hir_utils.rs @@ -687,6 +687,11 @@ impl HirEqInterExpr<'_, '_, '_> { .all(|(arg_a, arg_b)| self.eq_const_arg(arg_a, arg_b)) }, (ConstArgKind::Literal(kind_l), ConstArgKind::Literal(kind_r)) => kind_l == kind_r, + (ConstArgKind::Array(l_arr), ConstArgKind::Array(r_arr)) => { + l_arr.elems.len() == r_arr.elems.len() + && l_arr.elems.iter().zip(r_arr.elems.iter()) + .all(|(l_elem, r_elem)| self.eq_const_arg(l_elem, r_elem)) + } // Use explicit match for now since ConstArg is undergoing flux. ( ConstArgKind::Path(..) @@ -696,6 +701,7 @@ impl HirEqInterExpr<'_, '_, '_> { | ConstArgKind::Infer(..) | ConstArgKind::Struct(..) | ConstArgKind::Literal(..) + | ConstArgKind::Array(..) | ConstArgKind::Error(..), _, ) => false, @@ -1575,6 +1581,11 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> { self.hash_const_arg(arg); } }, + ConstArgKind::Array(array_expr) => { + for elem in array_expr.elems { + self.hash_const_arg(elem); + } + }, ConstArgKind::Infer(..) | ConstArgKind::Error(..) => {}, ConstArgKind::Literal(lit) => lit.hash(&mut self.s), } From c87bb01b9e8bc2419f95bba7f06f1ccee0cabca5 Mon Sep 17 00:00:00 2001 From: Aliaksei Semianiuk Date: Sat, 10 Jan 2026 15:34:41 +0500 Subject: [PATCH 0168/1843] Changelog for Clippy 1.93 --- CHANGELOG.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cda8003b05c..afefd436aada 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,66 @@ document. ## Unreleased / Beta / In Rust Nightly -[d9fb15c...master](https://github.com/rust-lang/rust-clippy/compare/d9fb15c...master) +[92b4b68...master](https://github.com/rust-lang/rust-clippy/compare/92b4b68...master) + +## Rust 1.93 + +Current stable, released 2026-01-22 + +[View all 96 merged pull requests](https://github.com/rust-lang/rust-clippy/pulls?q=merged%3A2025-10-17T15%3A48%3A11Z..2025-11-28T19%3A22%3A54Z+base%3Amaster) + +### New Lints + +* Added [`doc_paragraphs_missing_punctuation`] to `restriction` + [#15758](https://github.com/rust-lang/rust-clippy/pull/15758) + +### Moves and Deprecations + +* Renamed [`needless_if`] to [`needless_ifs`] + [#15961](https://github.com/rust-lang/rust-clippy/pull/15961) +* Renamed [`empty_enum`] to [`empty_enums`] + [#15912](https://github.com/rust-lang/rust-clippy/pull/15912) + +### Enhancements + +* [`result_large_err`] added `large_error_ignored` configuration + [#15697](https://github.com/rust-lang/rust-clippy/pull/15697) +* [`explicit_deref_methods`] don't lint in `impl Deref(Mut)` + [#16113](https://github.com/rust-lang/rust-clippy/pull/16113) +* [`missing_docs_in_private_items`] don't lint items in bodies and automatically derived impls; + better detect when things are accessible from the crate root; lint unnameable items which are + accessible outside the crate + [#14741](https://github.com/rust-lang/rust-clippy/pull/14741) +* [`unnecessary_unwrap`] and [`panicking_unwrap`] lint field accesses + [#15949](https://github.com/rust-lang/rust-clippy/pull/15949) +* [`ok_expect`] add autofix + [#15867](https://github.com/rust-lang/rust-clippy/pull/15867) +* [`let_and_return`] disallow _any_ text between let and return + [#16006](https://github.com/rust-lang/rust-clippy/pull/16006) +* [`needless_collect`] extend to lint more cases + [#14361](https://github.com/rust-lang/rust-clippy/pull/14361) +* [`needless_doctest_main`] and [`test_attr_in_doctest`] now handle whitespace in language tags + [#15967](https://github.com/rust-lang/rust-clippy/pull/15967) +* [`search_is_some`] now fixes code spanning multiple lines + [#15902](https://github.com/rust-lang/rust-clippy/pull/15902) +* [`unnecessary_find_map`] and [`unnecessary_filter_map`] make diagnostic spans more precise + [#15929](https://github.com/rust-lang/rust-clippy/pull/15929) +* [`precedence`] warn about ambiguity when a closure is used as a method call receiver + [#14421](https://github.com/rust-lang/rust-clippy/pull/14421) +* [`match_as_ref`] suggest `as_ref` when the reference needs to be cast; improve diagnostics + [#15934](https://github.com/rust-lang/rust-clippy/pull/15934) + [#15928](https://github.com/rust-lang/rust-clippy/pull/15928) + +### False Positive Fixes + +* [`single_range_in_vec_init`] fix FP for explicit `Range` + [#16043](https://github.com/rust-lang/rust-clippy/pull/16043) +* [`mod_module_files`] fix false positive for integration tests in workspace crates + [#16048](https://github.com/rust-lang/rust-clippy/pull/16048) +* [`replace_box`] fix FP when the box is moved + [#15984](https://github.com/rust-lang/rust-clippy/pull/15984) +* [`len_zero`] fix FP on unstable methods + [#15894](https://github.com/rust-lang/rust-clippy/pull/15894) ## Rust 1.92 From c825a504ab7d284610adf43d496fbfe899ed90aa Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jan 2026 09:51:42 +0100 Subject: [PATCH 0169/1843] Cleanup --- .../rust-analyzer/crates/hir/src/symbols.rs | 145 +++--- .../crates/ide-db/src/symbol_index.rs | 185 ++------ .../ide-db/src/test_data/test_doc_alias.txt | 112 +++-- .../test_symbol_index_collection.txt | 424 +++++++++++------- .../test_symbols_exclude_imports.txt | 12 +- .../test_data/test_symbols_with_imports.txt | 24 +- .../crates/ide/src/navigation_target.rs | 4 +- .../crates/mbe/src/expander/matcher.rs | 5 +- .../rust-analyzer/tests/slow-tests/main.rs | 44 +- .../rust-analyzer/crates/syntax/src/ptr.rs | 2 +- 10 files changed, 507 insertions(+), 450 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir/src/symbols.rs b/src/tools/rust-analyzer/crates/hir/src/symbols.rs index 544c759ed3a7..f9002f31fd15 100644 --- a/src/tools/rust-analyzer/crates/hir/src/symbols.rs +++ b/src/tools/rust-analyzer/crates/hir/src/symbols.rs @@ -21,12 +21,9 @@ use hir_ty::{ }; use intern::Symbol; use rustc_hash::FxHashMap; -use syntax::{ - AstNode, AstPtr, SyntaxNode, SyntaxNodePtr, ToSmolStr, - ast::{HasModuleItem, HasName}, -}; +use syntax::{AstNode, AstPtr, SyntaxNode, SyntaxNodePtr, ToSmolStr, ast::HasName}; -use crate::{Crate, HasCrate, Module, ModuleDef, Semantics}; +use crate::{HasCrate, Module, ModuleDef, Semantics}; /// The actual data that is stored in the index. It should be as compact as /// possible. @@ -44,14 +41,14 @@ pub struct FileSymbol<'db> { _marker: PhantomData<&'db ()>, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct DeclarationLocation { /// The file id for both the `ptr` and `name_ptr`. pub hir_file_id: HirFileId, /// This points to the whole syntax node of the declaration. pub ptr: SyntaxNodePtr, /// This points to the [`syntax::ast::Name`] identifier of the declaration. - pub name_ptr: AstPtr>, + pub name_ptr: Option>>, } impl DeclarationLocation { @@ -61,70 +58,6 @@ impl DeclarationLocation { } } -impl<'db> FileSymbol<'db> { - /// Create a `FileSymbol` representing a crate's root module. - /// This is used for crate search queries like `::` or `::foo`. - pub fn for_crate_root(db: &'db dyn HirDatabase, krate: Crate) -> Option> { - let display_name = krate.display_name(db)?; - let crate_name = display_name.crate_name(); - let root_module = krate.root_module(db); - let def_map = crate_def_map(db, krate.into()); - let module_data = &def_map[root_module.into()]; - - // Get the definition source (the source file for crate roots) - let definition = module_data.origin.definition_source(db); - let hir_file_id = definition.file_id; - - // For a crate root, the "declaration" is the source file itself - // We use the entire file's syntax node as the location - let syntax_node = definition.value.node(); - let ptr = SyntaxNodePtr::new(&syntax_node); - - // For the name, we need to create a synthetic name pointer. - // We'll use the first token of the file as a placeholder since crate roots - // don't have an explicit name in the source. - // We create a name_ptr pointing to the start of the file. - let name_ptr = match &definition.value { - crate::ModuleSource::SourceFile(sf) => { - // Try to find the first item with a name as a reasonable location for focus - // This is a bit of a hack but works for navigation purposes - let first_item: Option = sf.items().next(); - if let Some(item) = first_item { - if let Some(name) = item.syntax().children().find_map(syntax::ast::Name::cast) { - AstPtr::new(&name).wrap_left() - } else { - // No name found, try to use a NameRef instead - if let Some(name_ref) = - item.syntax().descendants().find_map(syntax::ast::NameRef::cast) - { - AstPtr::new(&name_ref).wrap_right() - } else { - return None; - } - } - } else { - return None; - } - } - _ => return None, - }; - - let loc = DeclarationLocation { hir_file_id, ptr, name_ptr }; - - Some(FileSymbol { - name: Symbol::intern(crate_name.as_str()), - def: ModuleDef::Module(root_module), - loc, - container_name: None, - is_alias: false, - is_assoc: false, - is_import: false, - do_not_complete: Complete::Yes, - _marker: PhantomData, - }) - } -} - /// Represents an outstanding module that the symbol collector must collect symbols from. #[derive(Debug)] struct SymbolCollectorWork { @@ -167,6 +100,11 @@ impl<'a> SymbolCollector<'a> { let _p = tracing::info_span!("SymbolCollector::collect", ?module).entered(); tracing::info!(?module, "SymbolCollector::collect"); + // If this is a crate root module, add a symbol for the crate itself + if module.is_crate_root(self.db) { + self.push_crate_root(module); + } + // The initial work is the root module we're collecting, additional work will // be populated as we traverse the module's definitions. self.work.push(SymbolCollectorWork { module_id: module.into(), parent: None }); @@ -176,6 +114,51 @@ impl<'a> SymbolCollector<'a> { } } + /// Push a symbol for a crate's root module. + /// This allows crate roots to appear in the symbol index for queries like `::` or `::foo`. + fn push_crate_root(&mut self, module: Module) { + let krate = module.krate(self.db); + let Some(display_name) = krate.display_name(self.db) else { return }; + let crate_name = display_name.crate_name(); + let canonical_name = display_name.canonical_name(); + + let def_map = crate_def_map(self.db, krate.into()); + let module_data = &def_map[def_map.crate_root(self.db)]; + + let definition = module_data.origin.definition_source(self.db); + let hir_file_id = definition.file_id; + let syntax_node = definition.value.node(); + let ptr = SyntaxNodePtr::new(&syntax_node); + + let loc = DeclarationLocation { hir_file_id, ptr, name_ptr: None }; + + self.symbols.insert(FileSymbol { + name: crate_name.symbol().clone(), + def: ModuleDef::Module(module), + loc, + container_name: None, + is_alias: false, + is_assoc: false, + is_import: false, + do_not_complete: Complete::Yes, + _marker: PhantomData, + }); + + if canonical_name != crate_name.symbol() { + self.symbols.insert(FileSymbol { + name: canonical_name.clone(), + def: ModuleDef::Module(module), + loc, + container_name: None, + is_alias: false, + is_assoc: false, + is_import: false, + do_not_complete: Complete::Yes, + _marker: PhantomData, + }); + } + } + pub fn finish(self) -> Box<[FileSymbol<'a>]> { self.symbols.into_iter().collect() } @@ -277,7 +260,7 @@ impl<'a> SymbolCollector<'a> { let dec_loc = DeclarationLocation { hir_file_id: source.file_id, ptr: SyntaxNodePtr::new(use_tree_src.syntax()), - name_ptr: AstPtr::new(&name_syntax), + name_ptr: Some(AstPtr::new(&name_syntax)), }; this.symbols.insert(FileSymbol { name: name.symbol().clone(), @@ -312,7 +295,7 @@ impl<'a> SymbolCollector<'a> { let dec_loc = DeclarationLocation { hir_file_id: source.file_id, ptr: SyntaxNodePtr::new(source.value.syntax()), - name_ptr: AstPtr::new(&name_syntax), + name_ptr: Some(AstPtr::new(&name_syntax)), }; this.symbols.insert(FileSymbol { name: name.symbol().clone(), @@ -477,10 +460,10 @@ impl<'a> SymbolCollector<'a> { let source = loc.source(self.db); let Some(name_node) = source.value.name() else { return Complete::Yes }; let def = ModuleDef::from(id.into()); - let dec_loc = DeclarationLocation { + let loc = DeclarationLocation { hir_file_id: source.file_id, ptr: SyntaxNodePtr::new(source.value.syntax()), - name_ptr: AstPtr::new(&name_node).wrap_left(), + name_ptr: Some(AstPtr::new(&name_node).wrap_left()), }; let mut do_not_complete = Complete::Yes; @@ -495,7 +478,7 @@ impl<'a> SymbolCollector<'a> { self.symbols.insert(FileSymbol { name: alias.clone(), def, - loc: dec_loc.clone(), + loc, container_name: self.current_container_name.clone(), is_alias: true, is_assoc, @@ -510,7 +493,7 @@ impl<'a> SymbolCollector<'a> { name: name.symbol().clone(), def, container_name: self.current_container_name.clone(), - loc: dec_loc, + loc, is_alias: false, is_assoc, is_import: false, @@ -527,10 +510,10 @@ impl<'a> SymbolCollector<'a> { let Some(declaration) = module_data.origin.declaration() else { return }; let module = declaration.to_node(self.db); let Some(name_node) = module.name() else { return }; - let dec_loc = DeclarationLocation { + let loc = DeclarationLocation { hir_file_id: declaration.file_id, ptr: SyntaxNodePtr::new(module.syntax()), - name_ptr: AstPtr::new(&name_node).wrap_left(), + name_ptr: Some(AstPtr::new(&name_node).wrap_left()), }; let def = ModuleDef::Module(module_id.into()); @@ -543,7 +526,7 @@ impl<'a> SymbolCollector<'a> { self.symbols.insert(FileSymbol { name: alias.clone(), def, - loc: dec_loc.clone(), + loc, container_name: self.current_container_name.clone(), is_alias: true, is_assoc: false, @@ -558,7 +541,7 @@ impl<'a> SymbolCollector<'a> { name: name.symbol().clone(), def: ModuleDef::Module(module_id.into()), container_name: self.current_container_name.clone(), - loc: dec_loc, + loc, is_alias: false, is_assoc: false, is_import: false, diff --git a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs index ca0d5ec1e5e6..05c3f360fa87 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs +++ b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs @@ -129,58 +129,19 @@ impl Query { /// - `anchor_to_crate`: Whether the first segment must be a crate name fn parse_path_query(query: &str) -> (Vec, String, bool) { // Check for leading :: (absolute path / crate search) - let anchor_to_crate = query.starts_with("::"); - let query = if anchor_to_crate { &query[2..] } else { query }; - - // Handle sole "::" - return all crates - if query.is_empty() && anchor_to_crate { - return (vec![], String::new(), true); - } - - // Check for trailing :: (module browsing - returns all items in module) - let return_all_in_module = query.ends_with("::"); - let query = if return_all_in_module { query.trim_end_matches("::") } else { query }; - - if !query.contains("::") { - // No path separator - single segment - if anchor_to_crate && !return_all_in_module { - // "::foo" - fuzzy search crate names only - return (vec![], query.to_string(), true); - } - if return_all_in_module { - // "foo::" - browse all items in module "foo" - // path_filter = ["foo"], query = "", anchor_to_crate = false/true - return (vec![query.to_string()], String::new(), anchor_to_crate); - } - // Plain "foo" - normal fuzzy search - return (vec![], query.to_string(), false); - } - - // Filter out empty segments (e.g., "foo::::bar" -> "foo::bar") - let segments: Vec<&str> = query.split("::").filter(|s| !s.is_empty()).collect(); - - if segments.is_empty() { - return (vec![], String::new(), anchor_to_crate); - } - - let path: Vec = - segments[..segments.len() - 1].iter().map(|s| s.to_string()).collect(); - let item = if return_all_in_module { - // All segments go to path, item is empty - let mut path = path; - path.push(segments.last().unwrap().to_string()); - return (path, String::new(), anchor_to_crate); - } else { - segments.last().unwrap_or(&"").to_string() + let (query, anchor_to_crate) = match query.strip_prefix("::") { + Some(q) => (q, true), + None => (query, false), }; - (path, item, anchor_to_crate) - } + let Some((prefix, query)) = query.rsplit_once("::") else { + return (vec![], query.to_owned(), anchor_to_crate); + }; - /// Returns true if this query should return all items in a module - /// (i.e., the original query ended with `::`) - fn is_module_browsing(&self) -> bool { - self.query.is_empty() && !self.path_filter.is_empty() + let prefix: Vec<_> = + prefix.split("::").filter(|s| !s.is_empty()).map(ToOwned::to_owned).collect(); + + (prefix, query.to_owned(), anchor_to_crate) } /// Returns true if this query is searching for crates @@ -245,11 +206,14 @@ pub fn crate_symbols(db: &dyn HirDatabase, krate: Crate) -> Box<[&SymbolIndex<'_ // That is, `#` switches from "types" to all symbols, `*` switches from the current // workspace to dependencies. // -// Note that filtering does not currently work in VSCode due to the editor never -// sending the special symbols to the language server. Instead, you can configure -// the filtering via the `rust-analyzer.workspace.symbol.search.scope` and -// `rust-analyzer.workspace.symbol.search.kind` settings. Symbols prefixed -// with `__` are hidden from the search results unless configured otherwise. +// This also supports general Rust path syntax with the usual rules. +// +// Note that paths do not currently work in VSCode due to the editor never +// sending the special symbols to the language server. Some other editors might not support the # or +// * search either, instead, you can configure the filtering via the +// `rust-analyzer.workspace.symbol.search.scope` and `rust-analyzer.workspace.symbol.search.kind` +// settings. Symbols prefixed with `__` are hidden from the search results unless configured +// otherwise. // // | Editor | Shortcut | // |---------|-----------| @@ -257,12 +221,11 @@ pub fn crate_symbols(db: &dyn HirDatabase, krate: Crate) -> Box<[&SymbolIndex<'_ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { let _p = tracing::info_span!("world_symbols", query = ?query.query).entered(); - // Handle special case: "::" alone or "::foo" for crate search if query.is_crate_search() { return search_crates(db, &query); } - // If we have a path filter, resolve it to target modules first + // If we have a path filter, resolve it to target modules let indices: Vec<_> = if !query.path_filter.is_empty() { let target_modules = resolve_path_to_modules( db, @@ -272,13 +235,11 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { ); if target_modules.is_empty() { - return vec![]; // Path doesn't resolve to any module + return vec![]; } - // Get symbol indices only for the resolved modules target_modules.iter().map(|&module| SymbolIndex::module_symbols(db, module)).collect() } else if query.libs { - // Original behavior for non-path queries searching libs LibraryRoots::get(db) .roots(db) .par_iter() @@ -289,7 +250,6 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { .map(|&root| SymbolIndex::library_symbols(db, root)) .collect() } else { - // Original behavior for non-path queries searching local crates let mut crates = Vec::new(); for &root in LocalRoots::get(db).roots(db).iter() { @@ -303,23 +263,11 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> { let mut res = vec![]; - // For module browsing (empty query, non-empty path_filter), return all symbols - if query.is_module_browsing() { - for index in &indices { - for symbol in index.symbols.iter() { - // Apply existing filters (only_types, assoc_mode, exclude_imports, etc.) - if query.matches_symbol_filters(symbol) { - res.push(symbol.clone()); - } - } - } - } else { - // Normal search: use FST to match item name - query.search::<()>(&indices, |f| { - res.push(f.clone()); - ControlFlow::Continue(()) - }); - } + // Normal search: use FST to match item name + query.search::<()>(&indices, |f| { + res.push(f.clone()); + ControlFlow::Continue(()) + }); res } @@ -341,9 +289,15 @@ fn search_crates<'db>(db: &'db RootDatabase, query: &Query) -> Vec) -> bool { - // Check only_types filter - if self.only_types - && !matches!( - symbol.def, - hir::ModuleDef::Adt(..) - | hir::ModuleDef::TypeAlias(..) - | hir::ModuleDef::BuiltinType(..) - | hir::ModuleDef::Trait(..) - ) - { - return false; - } - - // Check assoc_mode filter - if !self.matches_assoc_mode(symbol.is_assoc) { - return false; - } - - // Check exclude_imports filter - if self.exclude_imports && symbol.is_import { - return false; - } - - // Check underscore prefix - let ignore_underscore_prefixed = !self.query.starts_with("__"); - if ignore_underscore_prefixed && symbol.name.as_str().starts_with("__") { - return false; - } - - true - } } #[cfg(test)] @@ -939,34 +858,6 @@ pub struct Foo; assert!(!anchor); } - #[test] - fn test_query_modes() { - // Test is_module_browsing - let query = Query::new("foo::".to_owned()); - assert!(query.is_module_browsing()); - assert!(!query.is_crate_search()); - - // Test is_crate_search with sole :: - let query = Query::new("::".to_owned()); - assert!(!query.is_module_browsing()); - assert!(query.is_crate_search()); - - // Test is_crate_search with ::foo - let query = Query::new("::foo".to_owned()); - assert!(!query.is_module_browsing()); - assert!(query.is_crate_search()); - - // Normal query should be neither - let query = Query::new("foo".to_owned()); - assert!(!query.is_module_browsing()); - assert!(!query.is_crate_search()); - - // Path query should be neither - let query = Query::new("foo::bar".to_owned()); - assert!(!query.is_module_browsing()); - assert!(!query.is_crate_search()); - } - #[test] fn test_path_search() { let (mut db, _) = RootDatabase::with_many_files( diff --git a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt index 5783d97564d0..71680699b739 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt +++ b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt @@ -27,11 +27,13 @@ kind: STRUCT, range: 83..119, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 109..118, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 109..118, + }, + ), ), }, container_name: None, @@ -62,11 +64,13 @@ kind: STRUCT, range: 0..81, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 74..80, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 74..80, + }, + ), ), }, container_name: None, @@ -97,11 +101,13 @@ kind: STRUCT, range: 0..81, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 74..80, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 74..80, + }, + ), ), }, container_name: None, @@ -132,11 +138,13 @@ kind: STRUCT, range: 0..81, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 74..80, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 74..80, + }, + ), ), }, container_name: None, @@ -146,6 +154,34 @@ do_not_complete: Yes, _marker: PhantomData<&()>, }, + FileSymbol { + name: "ra_test_fixture", + def: Module( + Module { + id: ModuleIdLt { + [salsa id]: Id(3800), + }, + }, + ), + loc: DeclarationLocation { + hir_file_id: FileId( + EditionedFileId( + Id(3000), + ), + ), + ptr: SyntaxNodePtr { + kind: SOURCE_FILE, + range: 0..128, + }, + name_ptr: None, + }, + container_name: None, + is_alias: false, + is_assoc: false, + is_import: false, + do_not_complete: Yes, + _marker: PhantomData<&()>, + }, FileSymbol { name: "s1", def: Adt( @@ -167,11 +203,13 @@ kind: STRUCT, range: 0..81, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 74..80, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 74..80, + }, + ), ), }, container_name: None, @@ -202,11 +240,13 @@ kind: STRUCT, range: 83..119, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 109..118, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 109..118, + }, + ), ), }, container_name: None, @@ -237,11 +277,13 @@ kind: STRUCT, range: 0..81, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 74..80, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 74..80, + }, + ), ), }, container_name: None, diff --git a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt index 7692a7d61abf..2d62a56fe22d 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt +++ b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt @@ -25,11 +25,13 @@ kind: VARIANT, range: 201..202, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 201..202, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 201..202, + }, + ), ), }, container_name: Some( @@ -60,11 +62,13 @@ kind: TYPE_ALIAS, range: 470..490, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 475..480, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 475..480, + }, + ), ), }, container_name: None, @@ -93,11 +97,13 @@ kind: VARIANT, range: 204..205, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 204..205, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 204..205, + }, + ), ), }, container_name: Some( @@ -128,11 +134,13 @@ kind: CONST, range: 413..434, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 419..424, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 419..424, + }, + ), ), }, container_name: None, @@ -161,11 +169,13 @@ kind: CONST, range: 593..665, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 599..615, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 599..615, + }, + ), ), }, container_name: None, @@ -196,11 +206,13 @@ kind: ENUM, range: 185..207, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 190..194, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 190..194, + }, + ), ), }, container_name: None, @@ -231,11 +243,13 @@ kind: USE_TREE, range: 727..749, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 736..749, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 736..749, + }, + ), ), }, container_name: None, @@ -266,11 +280,13 @@ kind: MACRO_DEF, range: 153..168, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 159..164, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 159..164, + }, + ), ), }, container_name: None, @@ -299,11 +315,13 @@ kind: STATIC, range: 435..469, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 442..448, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 442..448, + }, + ), ), }, container_name: None, @@ -334,11 +352,13 @@ kind: STRUCT, range: 170..184, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 177..183, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 177..183, + }, + ), ), }, container_name: None, @@ -369,11 +389,13 @@ kind: STRUCT, range: 0..22, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 6..21, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 6..21, + }, + ), ), }, container_name: None, @@ -404,11 +426,13 @@ kind: STRUCT, range: 391..409, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 398..408, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 398..408, + }, + ), ), }, container_name: Some( @@ -441,11 +465,13 @@ kind: STRUCT, range: 628..654, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 635..653, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 635..653, + }, + ), ), }, container_name: Some( @@ -478,11 +504,13 @@ kind: STRUCT, range: 552..580, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 559..579, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 559..579, + }, + ), ), }, container_name: None, @@ -513,11 +541,13 @@ kind: STRUCT, range: 261..279, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 268..275, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 268..275, + }, + ), ), }, container_name: None, @@ -546,11 +576,13 @@ kind: TRAIT, range: 334..373, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 340..345, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 340..345, + }, + ), ), }, container_name: None, @@ -581,11 +613,13 @@ kind: USE_TREE, range: 755..769, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 764..769, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 764..769, + }, + ), ), }, container_name: None, @@ -616,11 +650,13 @@ kind: UNION, range: 208..222, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 214..219, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 214..219, + }, + ), ), }, container_name: None, @@ -649,11 +685,13 @@ kind: MODULE, range: 492..530, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 496..501, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 496..501, + }, + ), ), }, container_name: None, @@ -682,11 +720,13 @@ kind: MODULE, range: 667..677, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 671..676, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 671..676, + }, + ), ), }, container_name: None, @@ -717,11 +757,13 @@ kind: MACRO_RULES, range: 51..131, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 64..77, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 64..77, + }, + ), ), }, container_name: None, @@ -750,11 +792,13 @@ kind: FN, range: 307..330, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 310..325, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 310..325, + }, + ), ), }, container_name: Some( @@ -785,11 +829,13 @@ kind: FN, range: 242..257, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 245..252, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 245..252, + }, + ), ), }, container_name: Some( @@ -822,11 +868,13 @@ kind: MACRO_RULES, range: 1..48, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 14..31, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 14..31, + }, + ), ), }, container_name: None, @@ -855,11 +903,13 @@ kind: FN, range: 375..411, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 378..382, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 378..382, + }, + ), ), }, container_name: None, @@ -869,6 +919,34 @@ do_not_complete: Yes, _marker: PhantomData<&()>, }, + FileSymbol { + name: "ra_test_fixture", + def: Module( + Module { + id: ModuleIdLt { + [salsa id]: Id(3800), + }, + }, + ), + loc: DeclarationLocation { + hir_file_id: FileId( + EditionedFileId( + Id(3000), + ), + ), + ptr: SyntaxNodePtr { + kind: SOURCE_FILE, + range: 0..793, + }, + name_ptr: None, + }, + container_name: None, + is_alias: false, + is_assoc: false, + is_import: false, + do_not_complete: Yes, + _marker: PhantomData<&()>, + }, FileSymbol { name: "really_define_struct", def: Macro( @@ -890,11 +968,13 @@ kind: USE_TREE, range: 684..721, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 701..721, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 701..721, + }, + ), ), }, container_name: None, @@ -923,11 +1003,13 @@ kind: FN, range: 352..371, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 355..363, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 355..363, + }, + ), ), }, container_name: Some( @@ -969,11 +1051,13 @@ kind: STRUCT, range: 508..528, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 515..527, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 515..527, + }, + ), ), }, container_name: None, @@ -1011,11 +1095,13 @@ kind: USE_TREE, range: 141..173, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 157..173, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 157..173, + }, + ), ), }, container_name: None, @@ -1046,11 +1132,13 @@ kind: USE_TREE, range: 141..173, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 157..173, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 157..173, + }, + ), ), }, container_name: None, @@ -1081,11 +1169,13 @@ kind: STRUCT, range: 0..20, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 7..19, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 7..19, + }, + ), ), }, container_name: None, @@ -1116,11 +1206,13 @@ kind: USE_TREE, range: 35..69, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 51..69, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 51..69, + }, + ), ), }, container_name: None, @@ -1151,11 +1243,13 @@ kind: USE_TREE, range: 85..125, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 115..125, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 115..125, + }, + ), ), }, container_name: None, diff --git a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_exclude_imports.txt b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_exclude_imports.txt index 6f5f8f889c7d..87f0c7d9a817 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_exclude_imports.txt +++ b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_exclude_imports.txt @@ -20,11 +20,13 @@ kind: STRUCT, range: 0..15, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 11..14, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 11..14, + }, + ), ), }, container_name: None, diff --git a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_with_imports.txt b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_with_imports.txt index 5d3fe4d2658d..e96aa889ba06 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_with_imports.txt +++ b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbols_with_imports.txt @@ -20,11 +20,13 @@ kind: STRUCT, range: 0..15, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME, - range: 11..14, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME, + range: 11..14, + }, + ), ), }, container_name: None, @@ -55,11 +57,13 @@ kind: USE_TREE, range: 17..25, }, - name_ptr: AstPtr( - SyntaxNodePtr { - kind: NAME_REF, - range: 22..25, - }, + name_ptr: Some( + AstPtr( + SyntaxNodePtr { + kind: NAME_REF, + range: 22..25, + }, + ), ), }, container_name: None, diff --git a/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs b/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs index a271cac6fcd0..047df309eca6 100644 --- a/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs +++ b/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs @@ -19,7 +19,7 @@ use ide_db::{ }; use stdx::never; use syntax::{ - AstNode, SyntaxNode, TextRange, + AstNode, AstPtr, SyntaxNode, TextRange, ast::{self, HasName}, }; @@ -253,7 +253,7 @@ impl<'db> TryToNav for FileSymbol<'db> { db, self.loc.hir_file_id, self.loc.ptr.text_range(), - Some(self.loc.name_ptr.text_range()), + self.loc.name_ptr.map(AstPtr::text_range), ) .map(|(FileRange { file_id, range: full_range }, focus_range)| { NavigationTarget { diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs index 8f6627a60fe6..fe01fb1f1063 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs @@ -414,8 +414,9 @@ fn match_loop_inner<'t>( } // Check if we need a separator. - if item.sep.is_some() && !item.sep_matched { - let sep = item.sep.as_ref().unwrap(); + if let Some(sep) = &item.sep + && !item.sep_matched + { let mut fork = src.clone(); if expect_separator(&mut fork, sep) { // HACK: here we use `meta_result` to pass `TtIter` back to caller because diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs index eb1b8c5dd0e6..9f3c6742d651 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs @@ -1447,7 +1447,27 @@ foo = { path = "../foo" } .server() .wait_until_workspace_is_loaded(); - server.request::(Default::default(), json!([])); + server.request::( + Default::default(), + json!([ + { + "name": "bar", + "kind": 2, + "location": { + "uri": "file:///[..]bar/src/lib.rs", + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + } + } + }]), + ); let server = Project::with_fixture( r#" @@ -1486,7 +1506,27 @@ version = "0.0.0" .server() .wait_until_workspace_is_loaded(); - server.request::(Default::default(), json!([])); + server.request::( + Default::default(), + json!([ + { + "name": "baz", + "kind": 2, + "location": { + "uri": "file:///[..]baz/src/lib.rs", + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + } + } + }]), + ); } #[test] diff --git a/src/tools/rust-analyzer/crates/syntax/src/ptr.rs b/src/tools/rust-analyzer/crates/syntax/src/ptr.rs index 34c07598d200..c4979b8e3ae8 100644 --- a/src/tools/rust-analyzer/crates/syntax/src/ptr.rs +++ b/src/tools/rust-analyzer/crates/syntax/src/ptr.rs @@ -68,7 +68,7 @@ impl AstPtr { self.raw } - pub fn text_range(&self) -> TextRange { + pub fn text_range(self) -> TextRange { self.raw.text_range() } From 1caf7b222859c6de1b6fce064e32d3fdc50048c5 Mon Sep 17 00:00:00 2001 From: Lofty Inclination Date: Sat, 10 Jan 2026 12:54:50 +0000 Subject: [PATCH 0170/1843] Add miri specific shims for managing threads in no_std projects --- src/tools/miri/src/shims/foreign_items.rs | 36 ++++++++++++++++++- src/tools/miri/src/shims/sig.rs | 1 + .../concurrency/miri_thread_join_spawned.rs | 28 +++++++++++++++ src/tools/miri/tests/pass/miri-alloc.rs | 10 +++--- src/tools/miri/tests/utils/miri_extern.rs | 13 +++++++ 5 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 src/tools/miri/tests/pass/concurrency/miri_thread_join_spawned.rs diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 571f5efc9739..4c25caf56446 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -2,7 +2,7 @@ use std::collections::hash_map::Entry; use std::io::Write; use std::path::Path; -use rustc_abi::{Align, CanonAbi, Size}; +use rustc_abi::{Align, CanonAbi, ExternAbi, Size}; use rustc_ast::expand::allocator::NO_ALLOC_SHIM_IS_UNSTABLE; use rustc_data_structures::either::Either; use rustc_hir::attrs::Linkage; @@ -435,6 +435,40 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { // Return value: 0 on success, otherwise the size it would have needed. this.write_int(if success { 0 } else { needed_size }, dest)?; } + "miri_thread_spawn" => { + // FIXME: `check_shim_sig` does not work with function pointers. + let [start_routine, func_arg] = + this.check_shim_sig_lenient(abi, CanonAbi::Rust, link_name, args)?; + let start_routine = this.read_pointer(start_routine)?; + let func_arg = this.read_immediate(func_arg)?; + + this.start_regular_thread( + Some(dest.clone()), + start_routine, + ExternAbi::Rust, + func_arg, + this.machine.layouts.unit, + )?; + } + "miri_thread_join" => { + let [thread_id] = this.check_shim_sig( + shim_sig!(extern "Rust" fn(usize) -> bool), + link_name, + abi, + args, + )?; + + let thread = this.read_target_usize(thread_id)?; + if let Ok(thread) = this.thread_id_try_from(thread) { + this.join_thread_exclusive( + thread, + /* success_retval */ Scalar::from_bool(true), + dest, + )?; + } else { + this.write_scalar(Scalar::from_bool(false), dest)?; + } + } // Hint that a loop is spinning indefinitely. "miri_spin_loop" => { let [] = this.check_shim_sig_lenient(abi, CanonAbi::Rust, link_name, args)?; diff --git a/src/tools/miri/src/shims/sig.rs b/src/tools/miri/src/shims/sig.rs index e88cc66b6ea5..4e4b1bc2b9fe 100644 --- a/src/tools/miri/src/shims/sig.rs +++ b/src/tools/miri/src/shims/sig.rs @@ -57,6 +57,7 @@ macro_rules! shim_sig_arg { "u128" => $this.tcx.types.u128, "usize" => $this.tcx.types.usize, "()" => $this.tcx.types.unit, + "bool" => $this.tcx.types.bool, "*const _" => $this.machine.layouts.const_raw_ptr.ty, "*mut _" => $this.machine.layouts.mut_raw_ptr.ty, ty if let Some(win_ty) = ty.strip_prefix("winapi::") => diff --git a/src/tools/miri/tests/pass/concurrency/miri_thread_join_spawned.rs b/src/tools/miri/tests/pass/concurrency/miri_thread_join_spawned.rs new file mode 100644 index 000000000000..5b78068b6853 --- /dev/null +++ b/src/tools/miri/tests/pass/concurrency/miri_thread_join_spawned.rs @@ -0,0 +1,28 @@ +#![no_std] +#![no_main] +//@compile-flags: -Cpanic=abort +//@ignore-target: windows # no-std not supported on Windows + +#[path = "../../utils/mod.no_std.rs"] +mod utils; + +extern "Rust" fn thread_start(_null: *mut ()) { + unsafe { + utils::miri_spin_loop(); + utils::miri_spin_loop(); + } +} + +#[no_mangle] +fn miri_start(_argc: isize, _argv: *const *const u8) -> isize { + unsafe { + let thread_id = utils::miri_thread_spawn(thread_start, core::ptr::null_mut()); + assert_eq!(utils::miri_thread_join(thread_id), true); + } + 0 +} + +#[panic_handler] +fn panic_handler(_: &core::panic::PanicInfo) -> ! { + loop {} +} diff --git a/src/tools/miri/tests/pass/miri-alloc.rs b/src/tools/miri/tests/pass/miri-alloc.rs index 20269d8ced03..7469e29a8c96 100644 --- a/src/tools/miri/tests/pass/miri-alloc.rs +++ b/src/tools/miri/tests/pass/miri-alloc.rs @@ -6,17 +6,15 @@ // windows linker section, we can run this test on windows again. //@ignore-target: windows # no-std not supported on Windows -extern "Rust" { - fn miri_alloc(size: usize, align: usize) -> *mut u8; - fn miri_dealloc(ptr: *mut u8, size: usize, align: usize); -} +#[path = "../utils/mod.no_std.rs"] +mod utils; #[no_mangle] fn miri_start(_argc: isize, _argv: *const *const u8) -> isize { unsafe { - let ptr = miri_alloc(123, 1); + let ptr = utils::miri_alloc(123, 1); core::ptr::write_bytes(ptr, 0u8, 123); - miri_dealloc(ptr, 123, 1); + utils::miri_dealloc(ptr, 123, 1); } 0 } diff --git a/src/tools/miri/tests/utils/miri_extern.rs b/src/tools/miri/tests/utils/miri_extern.rs index bd01866dc34c..e9cde20412f4 100644 --- a/src/tools/miri/tests/utils/miri_extern.rs +++ b/src/tools/miri/tests/utils/miri_extern.rs @@ -156,6 +156,19 @@ extern "Rust" { /// Blocks the current execution if the argument is false pub fn miri_genmc_assume(condition: bool); + /// Miri-provided extern function to spawn a new thread in the interpreter. + /// + /// Returns the thread id. + /// + /// This is useful when no fundamental way of spawning threads is available, e.g. when using + /// `no_std`. + pub fn miri_thread_spawn(t: extern "Rust" fn(*mut ()), data: *mut ()) -> usize; + + /// Miri-provided extern function to join a thread that was spawned by Miri. + pub fn miri_thread_join(thread_id: usize) -> bool; + /// Indicate to Miri that this thread is busy-waiting in a spin loop. + /// + /// As far as Miri is concerned, this is equivalent to `yield_now`. pub fn miri_spin_loop(); } From 9eb4d7c56ebf768576183e5c5fd3e8159520dd22 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Sat, 10 Jan 2026 13:58:47 +0100 Subject: [PATCH 0171/1843] remove `no-rustfix` the suggestions are no longer overlapping?.. --- tests/ui/suspicious_to_owned.1.fixed | 73 ++++++++++++++++++++++++++++ tests/ui/suspicious_to_owned.2.fixed | 73 ++++++++++++++++++++++++++++ tests/ui/suspicious_to_owned.rs | 1 - tests/ui/suspicious_to_owned.stderr | 12 ++--- 4 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 tests/ui/suspicious_to_owned.1.fixed create mode 100644 tests/ui/suspicious_to_owned.2.fixed diff --git a/tests/ui/suspicious_to_owned.1.fixed b/tests/ui/suspicious_to_owned.1.fixed new file mode 100644 index 000000000000..6fd536a38ed1 --- /dev/null +++ b/tests/ui/suspicious_to_owned.1.fixed @@ -0,0 +1,73 @@ +#![warn(clippy::suspicious_to_owned)] +#![warn(clippy::implicit_clone)] +#![allow(clippy::redundant_clone)] +use std::borrow::Cow; +use std::ffi::{CStr, c_char}; + +fn main() { + let moo = "Moooo"; + let c_moo = b"Moooo\0"; + let c_moo_ptr = c_moo.as_ptr() as *const c_char; + let moos = ['M', 'o', 'o']; + let moos_vec = moos.to_vec(); + + // we expect this to be linted + let cow = Cow::Borrowed(moo); + let _ = cow.into_owned(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = Cow::Borrowed(moo); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = Cow::Borrowed(moo); + let _ = cow.clone(); + + // we expect this to be linted + let cow = Cow::Borrowed(&moos); + let _ = cow.into_owned(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = Cow::Borrowed(&moos); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = Cow::Borrowed(&moos); + let _ = cow.clone(); + + // we expect this to be linted + let cow = Cow::Borrowed(&moos_vec); + let _ = cow.into_owned(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = Cow::Borrowed(&moos_vec); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = Cow::Borrowed(&moos_vec); + let _ = cow.clone(); + + // we expect this to be linted + let cow = unsafe { CStr::from_ptr(c_moo_ptr) }.to_string_lossy(); + let _ = cow.into_owned(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = unsafe { CStr::from_ptr(c_moo_ptr) }.to_string_lossy(); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = unsafe { CStr::from_ptr(c_moo_ptr) }.to_string_lossy(); + let _ = cow.clone(); + + // we expect no lints for these + let _ = moo.to_owned(); + let _ = c_moo.to_owned(); + let _ = moos.to_owned(); + + // we expect implicit_clone lints for these + let _ = String::from(moo).clone(); + //~^ implicit_clone + + let _ = moos_vec.clone(); + //~^ implicit_clone +} diff --git a/tests/ui/suspicious_to_owned.2.fixed b/tests/ui/suspicious_to_owned.2.fixed new file mode 100644 index 000000000000..841adf8ea274 --- /dev/null +++ b/tests/ui/suspicious_to_owned.2.fixed @@ -0,0 +1,73 @@ +#![warn(clippy::suspicious_to_owned)] +#![warn(clippy::implicit_clone)] +#![allow(clippy::redundant_clone)] +use std::borrow::Cow; +use std::ffi::{CStr, c_char}; + +fn main() { + let moo = "Moooo"; + let c_moo = b"Moooo\0"; + let c_moo_ptr = c_moo.as_ptr() as *const c_char; + let moos = ['M', 'o', 'o']; + let moos_vec = moos.to_vec(); + + // we expect this to be linted + let cow = Cow::Borrowed(moo); + let _ = cow.clone(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = Cow::Borrowed(moo); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = Cow::Borrowed(moo); + let _ = cow.clone(); + + // we expect this to be linted + let cow = Cow::Borrowed(&moos); + let _ = cow.clone(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = Cow::Borrowed(&moos); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = Cow::Borrowed(&moos); + let _ = cow.clone(); + + // we expect this to be linted + let cow = Cow::Borrowed(&moos_vec); + let _ = cow.clone(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = Cow::Borrowed(&moos_vec); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = Cow::Borrowed(&moos_vec); + let _ = cow.clone(); + + // we expect this to be linted + let cow = unsafe { CStr::from_ptr(c_moo_ptr) }.to_string_lossy(); + let _ = cow.clone(); + //~^ suspicious_to_owned + + // we expect no lints for this + let cow = unsafe { CStr::from_ptr(c_moo_ptr) }.to_string_lossy(); + let _ = cow.into_owned(); + // we expect no lints for this + let cow = unsafe { CStr::from_ptr(c_moo_ptr) }.to_string_lossy(); + let _ = cow.clone(); + + // we expect no lints for these + let _ = moo.to_owned(); + let _ = c_moo.to_owned(); + let _ = moos.to_owned(); + + // we expect implicit_clone lints for these + let _ = String::from(moo).clone(); + //~^ implicit_clone + + let _ = moos_vec.clone(); + //~^ implicit_clone +} diff --git a/tests/ui/suspicious_to_owned.rs b/tests/ui/suspicious_to_owned.rs index 2eec05ccaf4c..f59b3fd6ed0c 100644 --- a/tests/ui/suspicious_to_owned.rs +++ b/tests/ui/suspicious_to_owned.rs @@ -1,4 +1,3 @@ -//@no-rustfix: overlapping suggestions #![warn(clippy::suspicious_to_owned)] #![warn(clippy::implicit_clone)] #![allow(clippy::redundant_clone)] diff --git a/tests/ui/suspicious_to_owned.stderr b/tests/ui/suspicious_to_owned.stderr index f90bea5fb8ff..1e631d4d4a14 100644 --- a/tests/ui/suspicious_to_owned.stderr +++ b/tests/ui/suspicious_to_owned.stderr @@ -1,5 +1,5 @@ error: this `to_owned` call clones the Cow<'_, str> itself and does not cause the Cow<'_, str> contents to become owned - --> tests/ui/suspicious_to_owned.rs:17:13 + --> tests/ui/suspicious_to_owned.rs:16:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ @@ -17,7 +17,7 @@ LL + let _ = cow.clone(); | error: this `to_owned` call clones the Cow<'_, [char; 3]> itself and does not cause the Cow<'_, [char; 3]> contents to become owned - --> tests/ui/suspicious_to_owned.rs:29:13 + --> tests/ui/suspicious_to_owned.rs:28:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ @@ -33,7 +33,7 @@ LL + let _ = cow.clone(); | error: this `to_owned` call clones the Cow<'_, Vec> itself and does not cause the Cow<'_, Vec> contents to become owned - --> tests/ui/suspicious_to_owned.rs:41:13 + --> tests/ui/suspicious_to_owned.rs:40:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ @@ -49,7 +49,7 @@ LL + let _ = cow.clone(); | error: this `to_owned` call clones the Cow<'_, str> itself and does not cause the Cow<'_, str> contents to become owned - --> tests/ui/suspicious_to_owned.rs:53:13 + --> tests/ui/suspicious_to_owned.rs:52:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ @@ -65,7 +65,7 @@ LL + let _ = cow.clone(); | error: implicitly cloning a `String` by calling `to_owned` on its dereferenced type - --> tests/ui/suspicious_to_owned.rs:69:13 + --> tests/ui/suspicious_to_owned.rs:68:13 | LL | let _ = String::from(moo).to_owned(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using: `String::from(moo).clone()` @@ -74,7 +74,7 @@ LL | let _ = String::from(moo).to_owned(); = help: to override `-D warnings` add `#[allow(clippy::implicit_clone)]` error: implicitly cloning a `Vec` by calling `to_owned` on its dereferenced type - --> tests/ui/suspicious_to_owned.rs:72:13 + --> tests/ui/suspicious_to_owned.rs:71:13 | LL | let _ = moos_vec.to_owned(); | ^^^^^^^^^^^^^^^^^^^ help: consider using: `moos_vec.clone()` From f228056c644969488958f01cb88a89f0a93f642d Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Sat, 10 Jan 2026 13:51:44 +0100 Subject: [PATCH 0172/1843] improve lint messages - only mention the type once - put types in backticks - only highlight the method name in the suggestion - removes the need for a snippet - makes for finer diffs (seen through `cargo dev lint`) --- clippy_lints/src/methods/mod.rs | 2 +- .../src/methods/suspicious_to_owned.rs | 23 +++++++----------- tests/ui/suspicious_to_owned.stderr | 24 +++++++++---------- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/clippy_lints/src/methods/mod.rs b/clippy_lints/src/methods/mod.rs index 248a147cfd77..d483496549fd 100644 --- a/clippy_lints/src/methods/mod.rs +++ b/clippy_lints/src/methods/mod.rs @@ -5576,7 +5576,7 @@ impl Methods { unnecessary_fallible_conversions::check_method(cx, expr); }, (sym::to_owned, []) => { - if !suspicious_to_owned::check(cx, expr, recv) { + if !suspicious_to_owned::check(cx, expr, span) { implicit_clone::check(cx, name, expr, recv); } }, diff --git a/clippy_lints/src/methods/suspicious_to_owned.rs b/clippy_lints/src/methods/suspicious_to_owned.rs index bcd1f11931fc..e9a5104eb3b4 100644 --- a/clippy_lints/src/methods/suspicious_to_owned.rs +++ b/clippy_lints/src/methods/suspicious_to_owned.rs @@ -1,15 +1,14 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::res::MaybeDef; -use clippy_utils::source::snippet_with_context; use rustc_errors::Applicability; use rustc_hir as hir; use rustc_lint::LateContext; use rustc_middle::ty::print::with_forced_trimmed_paths; -use rustc_span::sym; +use rustc_span::{Span, sym}; use super::SUSPICIOUS_TO_OWNED; -pub fn check(cx: &LateContext<'_>, expr: &hir::Expr<'_>, recv: &hir::Expr<'_>) -> bool { +pub fn check(cx: &LateContext<'_>, expr: &hir::Expr<'_>, method_span: Span) -> bool { if cx .typeck_results() .type_dependent_def_id(expr.hir_id) @@ -18,28 +17,22 @@ pub fn check(cx: &LateContext<'_>, expr: &hir::Expr<'_>, recv: &hir::Expr<'_>) - && let input_type = cx.typeck_results().expr_ty(expr) && input_type.is_diag_item(cx, sym::Cow) { - let mut app = Applicability::MaybeIncorrect; - let recv_snip = snippet_with_context(cx, recv.span, expr.span.ctxt(), "..", &mut app).0; + let app = Applicability::MaybeIncorrect; span_lint_and_then( cx, SUSPICIOUS_TO_OWNED, expr.span, with_forced_trimmed_paths!(format!( - "this `to_owned` call clones the {input_type} itself and does not cause the {input_type} contents to become owned" + "this `to_owned` call clones the `{input_type}` itself and does not cause its contents to become owned" )), |diag| { diag.span_suggestion( - expr.span, - "depending on intent, either make the Cow an Owned variant", - format!("{recv_snip}.into_owned()"), - app, - ); - diag.span_suggestion( - expr.span, - "or clone the Cow itself", - format!("{recv_snip}.clone()"), + method_span, + "depending on intent, either make the `Cow` an `Owned` variant", + "into_owned".to_string(), app, ); + diag.span_suggestion(method_span, "or clone the `Cow` itself", "clone".to_string(), app); }, ); return true; diff --git a/tests/ui/suspicious_to_owned.stderr b/tests/ui/suspicious_to_owned.stderr index 1e631d4d4a14..5fda1ed1cc9c 100644 --- a/tests/ui/suspicious_to_owned.stderr +++ b/tests/ui/suspicious_to_owned.stderr @@ -1,4 +1,4 @@ -error: this `to_owned` call clones the Cow<'_, str> itself and does not cause the Cow<'_, str> contents to become owned +error: this `to_owned` call clones the `Cow<'_, str>` itself and does not cause its contents to become owned --> tests/ui/suspicious_to_owned.rs:16:13 | LL | let _ = cow.to_owned(); @@ -6,59 +6,59 @@ LL | let _ = cow.to_owned(); | = note: `-D clippy::suspicious-to-owned` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::suspicious_to_owned)]` -help: depending on intent, either make the Cow an Owned variant +help: depending on intent, either make the `Cow` an `Owned` variant | LL | let _ = cow.into_owned(); | ++ -help: or clone the Cow itself +help: or clone the `Cow` itself | LL - let _ = cow.to_owned(); LL + let _ = cow.clone(); | -error: this `to_owned` call clones the Cow<'_, [char; 3]> itself and does not cause the Cow<'_, [char; 3]> contents to become owned +error: this `to_owned` call clones the `Cow<'_, [char; 3]>` itself and does not cause its contents to become owned --> tests/ui/suspicious_to_owned.rs:28:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ | -help: depending on intent, either make the Cow an Owned variant +help: depending on intent, either make the `Cow` an `Owned` variant | LL | let _ = cow.into_owned(); | ++ -help: or clone the Cow itself +help: or clone the `Cow` itself | LL - let _ = cow.to_owned(); LL + let _ = cow.clone(); | -error: this `to_owned` call clones the Cow<'_, Vec> itself and does not cause the Cow<'_, Vec> contents to become owned +error: this `to_owned` call clones the `Cow<'_, Vec>` itself and does not cause its contents to become owned --> tests/ui/suspicious_to_owned.rs:40:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ | -help: depending on intent, either make the Cow an Owned variant +help: depending on intent, either make the `Cow` an `Owned` variant | LL | let _ = cow.into_owned(); | ++ -help: or clone the Cow itself +help: or clone the `Cow` itself | LL - let _ = cow.to_owned(); LL + let _ = cow.clone(); | -error: this `to_owned` call clones the Cow<'_, str> itself and does not cause the Cow<'_, str> contents to become owned +error: this `to_owned` call clones the `Cow<'_, str>` itself and does not cause its contents to become owned --> tests/ui/suspicious_to_owned.rs:52:13 | LL | let _ = cow.to_owned(); | ^^^^^^^^^^^^^^ | -help: depending on intent, either make the Cow an Owned variant +help: depending on intent, either make the `Cow` an `Owned` variant | LL | let _ = cow.into_owned(); | ++ -help: or clone the Cow itself +help: or clone the `Cow` itself | LL - let _ = cow.to_owned(); LL + let _ = cow.clone(); From bd31b9d3bcd430a4236162d6b8e87661b8904964 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 10 Jan 2026 14:22:48 +0100 Subject: [PATCH 0173/1843] use io::Result for read/write helpers, and add read_until_eof_into_slice --- .../tests/pass-dep/libc/libc-socketpair.rs | 11 ++--- src/tools/miri/tests/utils/libc.rs | 46 +++++++++---------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs b/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs index 2031149aaf4f..20424fc86dc2 100644 --- a/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs +++ b/src/tools/miri/tests/pass-dep/libc/libc-socketpair.rs @@ -62,14 +62,11 @@ fn test_socketpair() { errno_check(unsafe { libc::close(fds[0]) }); // Reading the other end should return that data, then EOF. let mut buf: [u8; 5] = [0; 5]; - read_all_into_slice(fds[1], &mut buf[0..3]).unwrap(); - assert_eq!(&buf[0..3], data); - let res = read_into_slice(fds[1], &mut buf[3..5]).unwrap().0.len(); - assert_eq!(res, 0); // 0-sized read: EOF. + let (res, _) = read_until_eof_into_slice(fds[1], &mut buf).unwrap(); + assert_eq!(res, data); // Writing the other end should emit EPIPE. - let res = write_all_from_slice(fds[1], &mut buf); - assert_eq!(res, Err(-1)); - assert_eq!(std::io::Error::last_os_error().raw_os_error(), Some(libc::EPIPE)); + let err = write_all_from_slice(fds[1], &mut buf).unwrap_err(); + assert_eq!(err.raw_os_error(), Some(libc::EPIPE)); } fn test_socketpair_threaded() { diff --git a/src/tools/miri/tests/utils/libc.rs b/src/tools/miri/tests/utils/libc.rs index 0765bacb6bd8..4e9a94718c38 100644 --- a/src/tools/miri/tests/utils/libc.rs +++ b/src/tools/miri/tests/utils/libc.rs @@ -40,21 +40,17 @@ pub unsafe fn read_all( return read_so_far as libc::ssize_t; } -/// Try to fill the given slice by reading from `fd`. Error if that many bytes could not be read. +/// Try to fill the given slice by reading from `fd`. Panic if that many bytes could not be read. #[track_caller] -pub fn read_all_into_slice(fd: libc::c_int, buf: &mut [u8]) -> Result<(), libc::ssize_t> { - let res = unsafe { read_all(fd, buf.as_mut_ptr().cast(), buf.len()) }; - if res >= 0 { - assert_eq!(res as usize, buf.len()); - Ok(()) - } else { - Err(res) - } +pub fn read_all_into_slice(fd: libc::c_int, buf: &mut [u8]) -> io::Result<()> { + let res = errno_result(unsafe { read_all(fd, buf.as_mut_ptr().cast(), buf.len()) })?; + assert_eq!(res as usize, buf.len()); + Ok(()) } /// Read exactly `N` bytes from `fd`. Error if that many bytes could not be read. #[track_caller] -pub fn read_all_into_array(fd: libc::c_int) -> Result<[u8; N], libc::ssize_t> { +pub fn read_all_into_array(fd: libc::c_int) -> io::Result<[u8; N]> { let mut buf = [0; N]; read_all_into_slice(fd, &mut buf)?; Ok(buf) @@ -63,12 +59,20 @@ pub fn read_all_into_array(fd: libc::c_int) -> Result<[u8; N], l /// Do a single read from `fd` and return the part of the buffer that was written into, /// and the rest. #[track_caller] -pub fn read_into_slice( +pub fn read_into_slice(fd: libc::c_int, buf: &mut [u8]) -> io::Result<(&mut [u8], &mut [u8])> { + let res = errno_result(unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) })?; + Ok(buf.split_at_mut(res as usize)) +} + +/// Read from `fd` until we get EOF and return the part of the buffer that was written into, +/// and the rest. +#[track_caller] +pub fn read_until_eof_into_slice( fd: libc::c_int, buf: &mut [u8], -) -> Result<(&mut [u8], &mut [u8]), libc::ssize_t> { - let res = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) }; - if res >= 0 { Ok(buf.split_at_mut(res as usize)) } else { Err(res) } +) -> io::Result<(&mut [u8], &mut [u8])> { + let res = errno_result(unsafe { read_all(fd, buf.as_mut_ptr().cast(), buf.len()) })?; + Ok(buf.split_at_mut(res as usize)) } pub unsafe fn write_all( @@ -89,16 +93,12 @@ pub unsafe fn write_all( return written_so_far as libc::ssize_t; } -/// Write the entire `buf` to `fd`. Error if not all bytes could be written. +/// Write the entire `buf` to `fd`. Panic if not all bytes could be written. #[track_caller] -pub fn write_all_from_slice(fd: libc::c_int, buf: &[u8]) -> Result<(), libc::ssize_t> { - let res = unsafe { write_all(fd, buf.as_ptr().cast(), buf.len()) }; - if res >= 0 { - assert_eq!(res as usize, buf.len()); - Ok(()) - } else { - Err(res) - } +pub fn write_all_from_slice(fd: libc::c_int, buf: &[u8]) -> io::Result<()> { + let res = errno_result(unsafe { write_all(fd, buf.as_ptr().cast(), buf.len()) })?; + assert_eq!(res as usize, buf.len()); + Ok(()) } #[cfg(any(target_os = "linux", target_os = "android", target_os = "illumos"))] From 03ad5b07ee3aae10cfb9d70c9ff75ef3b9570e01 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 10 Jan 2026 15:40:04 +0100 Subject: [PATCH 0174/1843] readme: clarify 'single-threaded interpreter' --- src/tools/miri/README.md | 3 ++- src/tools/miri/src/bin/miri.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tools/miri/README.md b/src/tools/miri/README.md index d5ef9c767414..925b85f58766 100644 --- a/src/tools/miri/README.md +++ b/src/tools/miri/README.md @@ -228,7 +228,8 @@ and macOS targets are usually on par. Windows is supported less well. ### Running tests in parallel -Though it implements Rust threading, Miri itself is a single-threaded interpreter. +Though it implements Rust threading, Miri itself is a single-threaded interpreter +(it works like a multi-threaded OS on a single-core CPU). This means that when running `cargo miri test`, you will probably see a dramatic increase in the amount of time it takes to run your whole test suite due to the inherent interpreter slowdown and a loss of parallelism. diff --git a/src/tools/miri/src/bin/miri.rs b/src/tools/miri/src/bin/miri.rs index 19fbf90246c9..55c1fa203eb9 100644 --- a/src/tools/miri/src/bin/miri.rs +++ b/src/tools/miri/src/bin/miri.rs @@ -710,7 +710,7 @@ fn main() { if !miri_config.native_lib.is_empty() && miri_config.provenance_mode == ProvenanceMode::Strict { fatal_error!("strict provenance is not compatible with calling native functions"); } - // Native calls and many-seeds are an "intersting" combination. + // Native calls and many-seeds are an "interesting" combination. if !miri_config.native_lib.is_empty() && many_seeds.is_some() { eprintln!( "warning: `-Zmiri-many-seeds` runs multiple instances of the program in the same address space, \ From 4e18f1dad2be32a53a43840a39cab9457262d1c1 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jan 2026 14:53:22 +0100 Subject: [PATCH 0175/1843] Abstract proc-macro-srv input and output away --- .../proc-macro-api/src/legacy_protocol.rs | 4 +- .../proc-macro-api/src/legacy_protocol/msg.rs | 4 +- .../crates/proc-macro-api/src/lib.rs | 41 +++++- .../crates/proc-macro-api/src/process.rs | 138 +++++++++++++----- .../crates/proc-macro-srv-cli/src/main.rs | 12 +- .../proc-macro-srv-cli/src/main_loop.rs | 72 +++++---- 6 files changed, 193 insertions(+), 78 deletions(-) diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs index 22a7d9868e21..4524d1b66bfe 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs @@ -162,11 +162,11 @@ fn send_request( req: Request, buf: &mut P::Buf, ) -> Result, ServerError> { - req.write::<_, P>(&mut writer).map_err(|err| ServerError { + req.write::

(&mut writer).map_err(|err| ServerError { message: "failed to write request".into(), io: Some(Arc::new(err)), })?; - let res = Response::read::<_, P>(&mut reader, buf).map_err(|err| ServerError { + let res = Response::read::

(&mut reader, buf).map_err(|err| ServerError { message: "failed to read response".into(), io: Some(Arc::new(err)), })?; diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs index 4146b619ec0c..1b6590693354 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -155,13 +155,13 @@ impl ExpnGlobals { } pub trait Message: serde::Serialize + DeserializeOwned { - fn read(inp: &mut R, buf: &mut C::Buf) -> io::Result> { + fn read(inp: &mut dyn BufRead, buf: &mut C::Buf) -> io::Result> { Ok(match C::read(inp, buf)? { None => None, Some(buf) => Some(C::decode(buf)?), }) } - fn write(self, out: &mut W) -> io::Result<()> { + fn write(self, out: &mut dyn Write) -> io::Result<()> { let value = C::encode(&self)?; C::write(out, &value) } diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs index f5fcc99f14a3..98ee6817c2d2 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs @@ -18,7 +18,7 @@ extern crate rustc_driver as _; pub mod bidirectional_protocol; pub mod legacy_protocol; -mod process; +pub mod process; pub mod transport; use paths::{AbsPath, AbsPathBuf}; @@ -44,6 +44,25 @@ pub mod version { pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID; } +#[derive(Copy, Clone)] +pub enum ProtocolFormat { + JsonLegacy, + PostcardLegacy, + BidirectionalPostcardPrototype, +} + +impl fmt::Display for ProtocolFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProtocolFormat::JsonLegacy => write!(f, "json-legacy"), + ProtocolFormat::PostcardLegacy => write!(f, "postcard-legacy"), + ProtocolFormat::BidirectionalPostcardPrototype => { + write!(f, "bidirectional-postcard-prototype") + } + } + } +} + /// Represents different kinds of procedural macros that can be expanded by the external server. #[derive(Copy, Clone, Eq, PartialEq, Debug, serde_derive::Serialize, serde_derive::Deserialize)] pub enum ProcMacroKind { @@ -132,7 +151,25 @@ impl ProcMacroClient { > + Clone, version: Option<&Version>, ) -> io::Result { - let process = ProcMacroServerProcess::run(process_path, env, version)?; + let process = ProcMacroServerProcess::spawn(process_path, env, version)?; + Ok(ProcMacroClient { process: Arc::new(process), path: process_path.to_owned() }) + } + + /// Invokes `spawn` and returns a client connected to the resulting read and write handles. + /// + /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. + pub fn with_io_channels( + process_path: &AbsPath, + spawn: impl Fn( + Option, + ) -> io::Result<( + Box, + Box, + Box, + )>, + version: Option<&Version>, + ) -> io::Result { + let process = ProcMacroServerProcess::run(spawn, version, || "".to_owned())?; Ok(ProcMacroClient { process: Arc::new(process), path: process_path.to_owned() }) } diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs index f6a656e3ce3a..4f8762158790 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs @@ -13,14 +13,13 @@ use span::Span; use stdx::JodChild; use crate::{ - Codec, ProcMacro, ProcMacroKind, ServerError, + Codec, ProcMacro, ProcMacroKind, ProtocolFormat, ServerError, bidirectional_protocol::{self, SubCallback, msg::BidirectionalMessage, reject_subrequests}, legacy_protocol::{self, SpanMode}, version, }; /// Represents a process handling proc-macro communication. -#[derive(Debug)] pub(crate) struct ProcMacroServerProcess { /// The state of the proc-macro server process, the protocol is currently strictly sequential /// hence the lock on the state. @@ -31,6 +30,16 @@ pub(crate) struct ProcMacroServerProcess { exited: OnceLock>, } +impl std::fmt::Debug for ProcMacroServerProcess { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProcMacroServerProcess") + .field("version", &self.version) + .field("protocol", &self.protocol) + .field("exited", &self.exited) + .finish() + } +} + #[derive(Debug, Clone)] pub(crate) enum Protocol { LegacyJson { mode: SpanMode }, @@ -38,22 +47,83 @@ pub(crate) enum Protocol { BidirectionalPostcardPrototype { mode: SpanMode }, } +pub trait ProcessExit: Send + Sync { + fn exit_err(&mut self) -> Option; +} + +impl ProcessExit for Process { + fn exit_err(&mut self) -> Option { + match self.child.try_wait() { + Ok(None) | Err(_) => None, + Ok(Some(status)) => { + let mut msg = String::new(); + if !status.success() + && let Some(stderr) = self.child.stderr.as_mut() + { + _ = stderr.read_to_string(&mut msg); + } + Some(ServerError { + message: format!( + "proc-macro server exited with {status}{}{msg}", + if msg.is_empty() { "" } else { ": " } + ), + io: None, + }) + } + } + } +} + /// Maintains the state of the proc-macro server process. -#[derive(Debug)] struct ProcessSrvState { - process: Process, - stdin: ChildStdin, - stdout: BufReader, + process: Box, + stdin: Box, + stdout: Box, } impl ProcMacroServerProcess { /// Starts the proc-macro server and performs a version check - pub(crate) fn run<'a>( + pub(crate) fn spawn<'a>( process_path: &AbsPath, env: impl IntoIterator< Item = (impl AsRef, &'a Option>), > + Clone, version: Option<&Version>, + ) -> io::Result { + Self::run( + |format| { + let mut process = Process::run( + process_path, + env.clone(), + format.map(|format| format.to_string()).as_deref(), + )?; + let (stdin, stdout) = process.stdio().expect("couldn't access child stdio"); + + Ok((Box::new(process), Box::new(stdin), Box::new(stdout))) + }, + version, + || { + #[expect(clippy::disallowed_methods)] + Command::new(process_path) + .arg("--version") + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned()) + .unwrap_or_else(|_| "unknown version".to_owned()) + }, + ) + } + + /// Invokes `spawn` and performs a version check. + pub(crate) fn run( + spawn: impl Fn( + Option, + ) -> io::Result<( + Box, + Box, + Box, + )>, + version: Option<&Version>, + binary_server_version: impl Fn() -> String, ) -> io::Result { const VERSION: Version = Version::new(1, 93, 0); // we do `>` for nightly as this started working in the middle of the 1.93 nightly release, so we dont want to break on half of the nightlies @@ -65,27 +135,33 @@ impl ProcMacroServerProcess { && has_working_format_flag { &[ - ( - Some("bidirectional-postcard-prototype"), - Protocol::BidirectionalPostcardPrototype { mode: SpanMode::Id }, - ), - (Some("postcard-legacy"), Protocol::LegacyPostcard { mode: SpanMode::Id }), - (Some("json-legacy"), Protocol::LegacyJson { mode: SpanMode::Id }), + Some(ProtocolFormat::BidirectionalPostcardPrototype), + Some(ProtocolFormat::PostcardLegacy), + Some(ProtocolFormat::JsonLegacy), ] } else { - &[(None, Protocol::LegacyJson { mode: SpanMode::Id })] + &[None] }; let mut err = None; - for &(format, ref protocol) in formats { + for &format in formats { let create_srv = || { - let mut process = Process::run(process_path, env.clone(), format)?; - let (stdin, stdout) = process.stdio().expect("couldn't access child stdio"); + let (process, stdin, stdout) = spawn(format)?; io::Result::Ok(ProcMacroServerProcess { state: Mutex::new(ProcessSrvState { process, stdin, stdout }), version: 0, - protocol: protocol.clone(), + protocol: match format { + Some(ProtocolFormat::BidirectionalPostcardPrototype) => { + Protocol::BidirectionalPostcardPrototype { mode: SpanMode::Id } + } + Some(ProtocolFormat::PostcardLegacy) => { + Protocol::LegacyPostcard { mode: SpanMode::Id } + } + Some(ProtocolFormat::JsonLegacy) | None => { + Protocol::LegacyJson { mode: SpanMode::Id } + } + }, exited: OnceLock::new(), }) }; @@ -93,12 +169,7 @@ impl ProcMacroServerProcess { tracing::info!("sending proc-macro server version check"); match srv.version_check(Some(&mut reject_subrequests)) { Ok(v) if v > version::CURRENT_API_VERSION => { - #[allow(clippy::disallowed_methods)] - let process_version = Command::new(process_path) - .arg("--version") - .output() - .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned()) - .unwrap_or_else(|_| "unknown version".to_owned()); + let process_version = binary_server_version(); err = Some(io::Error::other(format!( "Your installed proc-macro server is too new for your rust-analyzer. API version: {}, server version: {process_version}. \ This will prevent proc-macro expansion from working. Please consider updating your rust-analyzer to ensure compatibility with your current toolchain.", @@ -275,22 +346,9 @@ impl ProcMacroServerProcess { f(&mut state.stdin, &mut state.stdout, &mut buf).map_err(|e| { if e.io.as_ref().map(|it| it.kind()) == Some(io::ErrorKind::BrokenPipe) { - match state.process.child.try_wait() { - Ok(None) | Err(_) => e, - Ok(Some(status)) => { - let mut msg = String::new(); - if !status.success() - && let Some(stderr) = state.process.child.stderr.as_mut() - { - _ = stderr.read_to_string(&mut msg); - } - let server_error = ServerError { - message: format!( - "proc-macro server exited with {status}{}{msg}", - if msg.is_empty() { "" } else { ": " } - ), - io: None, - }; + match state.process.exit_err() { + None => e, + Some(server_error) => { self.exited.get_or_init(|| AssertUnwindSafe(server_error)).0.clone() } } diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs index bdfdb50002e1..189a1eea5c19 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs @@ -45,7 +45,11 @@ fn main() -> std::io::Result<()> { } let &format = matches.get_one::("format").expect("format value should always be present"); - run(format) + + let mut stdin = std::io::BufReader::new(std::io::stdin()); + let mut stdout = std::io::stdout(); + + run(&mut stdin, &mut stdout, format) } #[derive(Copy, Clone)] @@ -88,7 +92,11 @@ impl ValueEnum for ProtocolFormat { } #[cfg(not(feature = "sysroot-abi"))] -fn run(_: ProtocolFormat) -> std::io::Result<()> { +fn run( + _: &mut std::io::BufReader, + _: &mut std::io::Stdout, + _: ProtocolFormat, +) -> std::io::Result<()> { Err(std::io::Error::new( std::io::ErrorKind::Unsupported, "proc-macro-srv-cli needs to be compiled with the `sysroot-abi` feature to function" diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs index 22536a4e52b1..0c651d22b41b 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs @@ -6,7 +6,7 @@ use proc_macro_api::{ transport::codec::{json::JsonProtocol, postcard::PostcardProtocol}, version::CURRENT_API_VERSION, }; -use std::io; +use std::io::{self, BufRead, Write}; use legacy::Message; @@ -32,15 +32,24 @@ impl legacy::SpanTransformer for SpanTrans { } } -pub(crate) fn run(format: ProtocolFormat) -> io::Result<()> { +pub(crate) fn run( + stdin: &mut (dyn BufRead + Send + Sync), + stdout: &mut (dyn Write + Send + Sync), + format: ProtocolFormat, +) -> io::Result<()> { match format { - ProtocolFormat::JsonLegacy => run_::(), - ProtocolFormat::PostcardLegacy => run_::(), - ProtocolFormat::BidirectionalPostcardPrototype => run_new::(), + ProtocolFormat::JsonLegacy => run_old::(stdin, stdout), + ProtocolFormat::PostcardLegacy => run_old::(stdin, stdout), + ProtocolFormat::BidirectionalPostcardPrototype => { + run_new::(stdin, stdout) + } } } -fn run_new() -> io::Result<()> { +fn run_new( + stdin: &mut (dyn BufRead + Send + Sync), + stdout: &mut (dyn Write + Send + Sync), +) -> io::Result<()> { fn macro_kind_to_api(kind: proc_macro_srv::ProcMacroKind) -> proc_macro_api::ProcMacroKind { match kind { proc_macro_srv::ProcMacroKind::CustomDerive => { @@ -52,8 +61,6 @@ fn run_new() -> io::Result<()> { } let mut buf = C::Buf::default(); - let mut stdin = io::stdin(); - let mut stdout = io::stdout(); let env_snapshot = EnvSnapshot::default(); let srv = proc_macro_srv::ProcMacroSrv::new(&env_snapshot); @@ -61,8 +68,7 @@ fn run_new() -> io::Result<()> { let mut span_mode = legacy::SpanMode::Id; 'outer: loop { - let req_opt = - bidirectional::BidirectionalMessage::read::<_, C>(&mut stdin.lock(), &mut buf)?; + let req_opt = bidirectional::BidirectionalMessage::read::(stdin, &mut buf)?; let Some(req) = req_opt else { break 'outer; }; @@ -77,22 +83,22 @@ fn run_new() -> io::Result<()> { .collect() }); - send_response::(&stdout, bidirectional::Response::ListMacros(res))?; + send_response::(stdout, bidirectional::Response::ListMacros(res))?; } bidirectional::Request::ApiVersionCheck {} => { send_response::( - &stdout, + stdout, bidirectional::Response::ApiVersionCheck(CURRENT_API_VERSION), )?; } bidirectional::Request::SetConfig(config) => { span_mode = config.span_mode; - send_response::(&stdout, bidirectional::Response::SetConfig(config))?; + send_response::(stdout, bidirectional::Response::SetConfig(config))?; } bidirectional::Request::ExpandMacro(task) => { - handle_expand::(&srv, &mut stdin, &mut stdout, &mut buf, span_mode, *task)?; + handle_expand::(&srv, stdin, stdout, &mut buf, span_mode, *task)?; } }, _ => continue, @@ -104,8 +110,8 @@ fn run_new() -> io::Result<()> { fn handle_expand( srv: &proc_macro_srv::ProcMacroSrv<'_>, - stdin: &io::Stdin, - stdout: &io::Stdout, + stdin: &mut (dyn BufRead + Send + Sync), + stdout: &mut (dyn Write + Send + Sync), buf: &mut C::Buf, span_mode: legacy::SpanMode, task: bidirectional::ExpandMacro, @@ -118,7 +124,7 @@ fn handle_expand( fn handle_expand_id( srv: &proc_macro_srv::ProcMacroSrv<'_>, - stdout: &io::Stdout, + stdout: &mut dyn Write, task: bidirectional::ExpandMacro, ) -> io::Result<()> { let bidirectional::ExpandMacro { lib, env, current_dir, data } = task; @@ -157,12 +163,12 @@ fn handle_expand_id( }) .map_err(|e| legacy::PanicMessage(e.into_string().unwrap_or_default())); - send_response::(&stdout, bidirectional::Response::ExpandMacro(res)) + send_response::(stdout, bidirectional::Response::ExpandMacro(res)) } struct ProcMacroClientHandle<'a, C: Codec> { - stdin: &'a io::Stdin, - stdout: &'a io::Stdout, + stdin: &'a mut (dyn BufRead + Send + Sync), + stdout: &'a mut (dyn Write + Send + Sync), buf: &'a mut C::Buf, } @@ -173,11 +179,11 @@ impl<'a, C: Codec> ProcMacroClientHandle<'a, C> { ) -> Option { let msg = bidirectional::BidirectionalMessage::SubRequest(req); - if msg.write::<_, C>(&mut self.stdout.lock()).is_err() { + if msg.write::(&mut *self.stdout).is_err() { return None; } - match bidirectional::BidirectionalMessage::read::<_, C>(&mut self.stdin.lock(), self.buf) { + match bidirectional::BidirectionalMessage::read::(&mut *self.stdin, self.buf) { Ok(Some(msg)) => Some(msg), _ => None, } @@ -238,8 +244,8 @@ impl proc_macro_srv::ProcMacroClientInterface for ProcMacroClientHandl fn handle_expand_ra( srv: &proc_macro_srv::ProcMacroSrv<'_>, - stdin: &io::Stdin, - stdout: &io::Stdout, + stdin: &mut (dyn BufRead + Send + Sync), + stdout: &mut (dyn Write + Send + Sync), buf: &mut C::Buf, task: bidirectional::ExpandMacro, ) -> io::Result<()> { @@ -301,10 +307,13 @@ fn handle_expand_ra( .map(|(tree, span_data_table)| bidirectional::ExpandMacroExtended { tree, span_data_table }) .map_err(|e| legacy::PanicMessage(e.into_string().unwrap_or_default())); - send_response::(&stdout, bidirectional::Response::ExpandMacroExtended(res)) + send_response::(stdout, bidirectional::Response::ExpandMacroExtended(res)) } -fn run_() -> io::Result<()> { +fn run_old( + stdin: &mut (dyn BufRead + Send + Sync), + stdout: &mut (dyn Write + Send + Sync), +) -> io::Result<()> { fn macro_kind_to_api(kind: proc_macro_srv::ProcMacroKind) -> proc_macro_api::ProcMacroKind { match kind { proc_macro_srv::ProcMacroKind::CustomDerive => { @@ -316,8 +325,8 @@ fn run_() -> io::Result<()> { } let mut buf = C::Buf::default(); - let mut read_request = || legacy::Request::read::<_, C>(&mut io::stdin().lock(), &mut buf); - let write_response = |msg: legacy::Response| msg.write::<_, C>(&mut io::stdout().lock()); + let mut read_request = || legacy::Request::read::(stdin, &mut buf); + let mut write_response = |msg: legacy::Response| msg.write::(stdout); let env = EnvSnapshot::default(); let srv = proc_macro_srv::ProcMacroSrv::new(&env); @@ -446,7 +455,10 @@ fn run_() -> io::Result<()> { Ok(()) } -fn send_response(stdout: &io::Stdout, resp: bidirectional::Response) -> io::Result<()> { +fn send_response( + stdout: &mut dyn Write, + resp: bidirectional::Response, +) -> io::Result<()> { let resp = bidirectional::BidirectionalMessage::Response(resp); - resp.write::<_, C>(&mut stdout.lock()) + resp.write::(stdout) } From 27fef0ccbe56e9e4494d19780cdd7c41660d764d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jan 2026 16:01:09 +0100 Subject: [PATCH 0176/1843] internal: Landing integration test infra for proc-macro-srv-cli --- src/tools/rust-analyzer/Cargo.lock | 6 + .../crates/proc-macro-api/src/lib.rs | 6 +- .../crates/proc-macro-srv-cli/Cargo.toml | 14 ++ .../crates/proc-macro-srv-cli/src/lib.rs | 6 + .../crates/proc-macro-srv-cli/src/main.rs | 43 ++-- .../proc-macro-srv-cli/src/main_loop.rs | 5 +- .../proc-macro-srv-cli/tests/common/utils.rs | 213 +++++++++++++++++ .../proc-macro-srv-cli/tests/legacy_json.rs | 224 ++++++++++++++++++ 8 files changed, 494 insertions(+), 23 deletions(-) create mode 100644 src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/lib.rs create mode 100644 src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs create mode 100644 src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock index 5bdde7c7c3e6..d6c6250e13dc 100644 --- a/src/tools/rust-analyzer/Cargo.lock +++ b/src/tools/rust-analyzer/Cargo.lock @@ -1879,9 +1879,15 @@ name = "proc-macro-srv-cli" version = "0.0.0" dependencies = [ "clap", + "expect-test", + "intern", + "paths", "postcard", "proc-macro-api", "proc-macro-srv", + "proc-macro-test", + "span", + "tt", ] [[package]] diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs index 98ee6817c2d2..822809943a36 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs @@ -44,10 +44,14 @@ pub mod version { pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID; } -#[derive(Copy, Clone)] +/// Protocol format for communication between client and server. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ProtocolFormat { + /// JSON-based legacy protocol (newline-delimited JSON). JsonLegacy, + /// Postcard-based legacy protocol (COBS-encoded postcard). PostcardLegacy, + /// Bidirectional postcard protocol with sub-request support. BidirectionalPostcardPrototype, } diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml index 6b2db0b269d5..a25e3b64ad42 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml @@ -10,12 +10,26 @@ license.workspace = true rust-version.workspace = true publish = false +[lib] +doctest = false + [dependencies] proc-macro-srv.workspace = true proc-macro-api.workspace = true postcard.workspace = true clap = {version = "4.5.42", default-features = false, features = ["std"]} +[dev-dependencies] +expect-test.workspace = true +paths.workspace = true +# span = {workspace = true, default-features = false} does not work +span = { path = "../span", default-features = false} +tt.workspace = true +intern.workspace = true + +# used as proc macro test target +proc-macro-test.path = "../proc-macro-srv/proc-macro-test" + [features] default = [] # default = ["sysroot-abi"] diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/lib.rs new file mode 100644 index 000000000000..9e6f03bf4604 --- /dev/null +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/lib.rs @@ -0,0 +1,6 @@ +//! Library interface for `proc-macro-srv-cli`. +//! +//! This module exposes the server main loop and protocol format for integration testing. + +#[cfg(feature = "sysroot-abi")] +pub mod main_loop; diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs index 189a1eea5c19..a246d4d3f28f 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs @@ -9,11 +9,11 @@ extern crate rustc_driver as _; mod version; -#[cfg(feature = "sysroot-abi")] -mod main_loop; use clap::{Command, ValueEnum}; +use proc_macro_api::ProtocolFormat; + #[cfg(feature = "sysroot-abi")] -use main_loop::run; +use proc_macro_srv_cli::main_loop::run; fn main() -> std::io::Result<()> { let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); @@ -32,7 +32,7 @@ fn main() -> std::io::Result<()> { .long("format") .action(clap::ArgAction::Set) .default_value("json-legacy") - .value_parser(clap::builder::EnumValueParser::::new()), + .value_parser(clap::builder::EnumValueParser::::new()), clap::Arg::new("version") .long("version") .action(clap::ArgAction::SetTrue) @@ -43,33 +43,37 @@ fn main() -> std::io::Result<()> { println!("rust-analyzer-proc-macro-srv {}", version::version()); return Ok(()); } - let &format = - matches.get_one::("format").expect("format value should always be present"); + let &format = matches + .get_one::("format") + .expect("format value should always be present"); let mut stdin = std::io::BufReader::new(std::io::stdin()); let mut stdout = std::io::stdout(); - run(&mut stdin, &mut stdout, format) + run(&mut stdin, &mut stdout, format.into()) } +/// Wrapper for CLI argument parsing that implements `ValueEnum`. #[derive(Copy, Clone)] -enum ProtocolFormat { - JsonLegacy, - PostcardLegacy, - BidirectionalPostcardPrototype, +struct ProtocolFormatArg(ProtocolFormat); + +impl From for ProtocolFormat { + fn from(arg: ProtocolFormatArg) -> Self { + arg.0 + } } -impl ValueEnum for ProtocolFormat { +impl ValueEnum for ProtocolFormatArg { fn value_variants<'a>() -> &'a [Self] { &[ - ProtocolFormat::JsonLegacy, - ProtocolFormat::PostcardLegacy, - ProtocolFormat::BidirectionalPostcardPrototype, + ProtocolFormatArg(ProtocolFormat::JsonLegacy), + ProtocolFormatArg(ProtocolFormat::PostcardLegacy), + ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), ] } fn to_possible_value(&self) -> Option { - match self { + match self.0 { ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), ProtocolFormat::PostcardLegacy => { Some(clap::builder::PossibleValue::new("postcard-legacy")) @@ -79,12 +83,13 @@ impl ValueEnum for ProtocolFormat { } } } + fn from_str(input: &str, _ignore_case: bool) -> Result { match input { - "json-legacy" => Ok(ProtocolFormat::JsonLegacy), - "postcard-legacy" => Ok(ProtocolFormat::PostcardLegacy), + "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), + "postcard-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::PostcardLegacy)), "bidirectional-postcard-prototype" => { - Ok(ProtocolFormat::BidirectionalPostcardPrototype) + Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) } _ => Err(format!("unknown protocol format: {input}")), } diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs index 0c651d22b41b..b927eea46b58 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs @@ -1,6 +1,6 @@ //! The main loop of the proc-macro server. use proc_macro_api::{ - Codec, + Codec, ProtocolFormat, bidirectional_protocol::msg as bidirectional, legacy_protocol::msg as legacy, transport::codec::{json::JsonProtocol, postcard::PostcardProtocol}, @@ -12,7 +12,6 @@ use legacy::Message; use proc_macro_srv::{EnvSnapshot, SpanId}; -use crate::ProtocolFormat; struct SpanTrans; impl legacy::SpanTransformer for SpanTrans { @@ -32,7 +31,7 @@ impl legacy::SpanTransformer for SpanTrans { } } -pub(crate) fn run( +pub fn run( stdin: &mut (dyn BufRead + Send + Sync), stdout: &mut (dyn Write + Send + Sync), format: ProtocolFormat, diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs new file mode 100644 index 000000000000..722e92eec7e5 --- /dev/null +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs @@ -0,0 +1,213 @@ +use std::{ + collections::VecDeque, + io::{self, BufRead, Read, Write}, + sync::{Arc, Condvar, Mutex}, + thread, +}; + +use paths::Utf8PathBuf; +use proc_macro_api::{ + legacy_protocol::msg::{FlatTree, Message, Request, Response, SpanDataIndexMap}, + transport::codec::json::JsonProtocol, +}; +use span::{Edition, EditionedFileId, FileId, Span, SpanAnchor, SyntaxContext, TextRange}; +use tt::{Delimiter, DelimiterKind, TopSubtreeBuilder}; + +/// Shared state for an in-memory byte channel. +#[derive(Default)] +struct ChannelState { + buffer: VecDeque, + closed: bool, +} + +type InMemoryChannel = Arc<(Mutex, Condvar)>; + +/// Writer end of an in-memory channel. +pub(crate) struct ChannelWriter { + state: InMemoryChannel, +} + +impl Write for ChannelWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let (lock, cvar) = &*self.state; + let mut state = lock.lock().unwrap(); + if state.closed { + return Err(io::Error::new(io::ErrorKind::BrokenPipe, "channel closed")); + } + state.buffer.extend(buf); + cvar.notify_all(); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Drop for ChannelWriter { + fn drop(&mut self) { + let (lock, cvar) = &*self.state; + let mut state = lock.lock().unwrap(); + state.closed = true; + cvar.notify_all(); + } +} + +/// Reader end of an in-memory channel. +pub(crate) struct ChannelReader { + state: InMemoryChannel, + internal_buf: Vec, +} + +impl Read for ChannelReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let (lock, cvar) = &*self.state; + let mut state = lock.lock().unwrap(); + + while state.buffer.is_empty() && !state.closed { + state = cvar.wait(state).unwrap(); + } + + if state.buffer.is_empty() && state.closed { + return Ok(0); + } + + let to_read = buf.len().min(state.buffer.len()); + for (dst, src) in buf.iter_mut().zip(state.buffer.drain(..to_read)) { + *dst = src; + } + Ok(to_read) + } +} + +impl BufRead for ChannelReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + let (lock, cvar) = &*self.state; + let mut state = lock.lock().unwrap(); + + while state.buffer.is_empty() && !state.closed { + state = cvar.wait(state).unwrap(); + } + + self.internal_buf.clear(); + self.internal_buf.extend(&state.buffer); + Ok(&self.internal_buf) + } + + fn consume(&mut self, amt: usize) { + let (lock, _) = &*self.state; + let mut state = lock.lock().unwrap(); + let to_drain = amt.min(state.buffer.len()); + drop(state.buffer.drain(..to_drain)); + } +} + +/// Creates a connected pair of channels for bidirectional communication. +fn create_channel_pair() -> (ChannelWriter, ChannelReader, ChannelWriter, ChannelReader) { + // Channel for client -> server communication + let client_to_server = Arc::new(( + Mutex::new(ChannelState { buffer: VecDeque::new(), closed: false }), + Condvar::new(), + )); + let client_writer = ChannelWriter { state: client_to_server.clone() }; + let server_reader = ChannelReader { state: client_to_server, internal_buf: Vec::new() }; + + // Channel for server -> client communication + let server_to_client = Arc::new(( + Mutex::new(ChannelState { buffer: VecDeque::new(), closed: false }), + Condvar::new(), + )); + + let server_writer = ChannelWriter { state: server_to_client.clone() }; + let client_reader = ChannelReader { state: server_to_client, internal_buf: Vec::new() }; + + (client_writer, client_reader, server_writer, server_reader) +} + +pub(crate) fn proc_macro_test_dylib_path() -> Utf8PathBuf { + let path = proc_macro_test::PROC_MACRO_TEST_LOCATION; + if path.is_empty() { + panic!("proc-macro-test dylib not available (requires nightly toolchain)"); + } + path.into() +} + +/// Runs a test with the server in a background thread. +pub(crate) fn with_server(test_fn: F) -> R +where + F: FnOnce(&mut dyn Write, &mut dyn BufRead) -> R, +{ + let (mut client_writer, mut client_reader, mut server_writer, mut server_reader) = + create_channel_pair(); + + let server_handle = thread::spawn(move || { + proc_macro_srv_cli::main_loop::run( + &mut server_reader, + &mut server_writer, + proc_macro_api::ProtocolFormat::JsonLegacy, + ) + }); + + let result = test_fn(&mut client_writer, &mut client_reader); + + // Close the client writer to signal the server to stop + drop(client_writer); + + // Wait for server to finish + match server_handle.join() { + Ok(Ok(())) => {} + Ok(Err(e)) => { + // IO error from server is expected when client disconnects + if matches!( + e.kind(), + io::ErrorKind::BrokenPipe + | io::ErrorKind::UnexpectedEof + | io::ErrorKind::InvalidData + ) { + panic!("Server error: {e}"); + } + } + Err(e) => std::panic::resume_unwind(e), + } + + result +} + +/// Sends a request and reads the response using JSON protocol. +pub(crate) fn request( + writer: &mut dyn Write, + reader: &mut dyn BufRead, + request: Request, +) -> Response { + request.write::(writer).expect("failed to write request"); + + let mut buf = String::new(); + Response::read::(reader, &mut buf) + .expect("failed to read response") + .expect("no response received") +} + +/// Creates a simple empty token tree suitable for testing. +pub(crate) fn create_empty_token_tree( + version: u32, + span_data_table: &mut SpanDataIndexMap, +) -> FlatTree { + let anchor = SpanAnchor { + file_id: EditionedFileId::new(FileId::from_raw(0), Edition::CURRENT), + ast_id: span::ROOT_ERASED_FILE_AST_ID, + }; + let span = Span { + range: TextRange::empty(0.into()), + anchor, + ctx: SyntaxContext::root(Edition::CURRENT), + }; + + let builder = TopSubtreeBuilder::new(Delimiter { + open: span, + close: span, + kind: DelimiterKind::Invisible, + }); + let tt = builder.build(); + + FlatTree::from_subtree(tt.view(), version, span_data_table) +} diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs new file mode 100644 index 000000000000..1fa886219a8a --- /dev/null +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs @@ -0,0 +1,224 @@ +//! Integration tests for the proc-macro-srv-cli main loop. +//! +//! These tests exercise the full client-server RPC procedure using in-memory +//! channels without needing to spawn the actual server and client processes. + +#![cfg(feature = "sysroot-abi")] + +mod common { + pub(crate) mod utils; +} + +use common::utils::{create_empty_token_tree, proc_macro_test_dylib_path, request, with_server}; +use expect_test::expect; +use proc_macro_api::{ + legacy_protocol::msg::{ + ExpandMacro, ExpandMacroData, ExpnGlobals, PanicMessage, Request, Response, ServerConfig, + SpanDataIndexMap, SpanMode, + }, + version::CURRENT_API_VERSION, +}; + +#[test] +fn test_version_check() { + with_server(|writer, reader| { + let response = request(writer, reader, Request::ApiVersionCheck {}); + + match response { + Response::ApiVersionCheck(version) => { + assert_eq!(version, CURRENT_API_VERSION); + } + other => panic!("unexpected response: {other:?}"), + } + }); +} + +#[test] +fn test_list_macros() { + with_server(|writer, reader| { + let dylib_path = proc_macro_test_dylib_path(); + let response = request(writer, reader, Request::ListMacros { dylib_path }); + + let Response::ListMacros(Ok(macros)) = response else { + panic!("expected successful ListMacros response"); + }; + + let mut macro_list: Vec<_> = + macros.iter().map(|(name, kind)| format!("{name} [{kind:?}]")).collect(); + macro_list.sort(); + let macro_list_str = macro_list.join("\n"); + + expect![[r#" + DeriveEmpty [CustomDerive] + DeriveError [CustomDerive] + DerivePanic [CustomDerive] + DeriveReemit [CustomDerive] + attr_error [Attr] + attr_noop [Attr] + attr_panic [Attr] + fn_like_clone_tokens [Bang] + fn_like_error [Bang] + fn_like_mk_idents [Bang] + fn_like_mk_literals [Bang] + fn_like_noop [Bang] + fn_like_panic [Bang] + fn_like_span_join [Bang] + fn_like_span_line_column [Bang] + fn_like_span_ops [Bang]"#]] + .assert_eq(¯o_list_str); + }); +} + +#[test] +fn test_list_macros_invalid_path() { + with_server(|writer, reader| { + let response = request( + writer, + reader, + Request::ListMacros { dylib_path: "/nonexistent/path/to/dylib.so".into() }, + ); + + match response { + Response::ListMacros(Err(e)) => assert!( + e.starts_with("Cannot create expander for /nonexistent/path/to/dylib.so"), + "{e}" + ), + other => panic!("expected error response, got: {other:?}"), + } + }); +} + +#[test] +fn test_set_config() { + with_server(|writer, reader| { + let config = ServerConfig { span_mode: SpanMode::Id }; + let response = request(writer, reader, Request::SetConfig(config)); + + match response { + Response::SetConfig(returned_config) => { + assert_eq!(returned_config.span_mode, SpanMode::Id); + } + other => panic!("unexpected response: {other:?}"), + } + }); +} + +#[test] +fn test_set_config_rust_analyzer_mode() { + with_server(|writer, reader| { + let config = ServerConfig { span_mode: SpanMode::RustAnalyzer }; + let response = request(writer, reader, Request::SetConfig(config)); + + match response { + Response::SetConfig(returned_config) => { + assert_eq!(returned_config.span_mode, SpanMode::RustAnalyzer); + } + other => panic!("unexpected response: {other:?}"), + } + }); +} + +#[test] +fn test_expand_macro_panic() { + with_server(|writer, reader| { + let dylib_path = proc_macro_test_dylib_path(); + + let version_response = request(writer, reader, Request::ApiVersionCheck {}); + let Response::ApiVersionCheck(version) = version_response else { + panic!("expected version check response"); + }; + + let mut span_data_table = SpanDataIndexMap::default(); + let macro_body = create_empty_token_tree(version, &mut span_data_table); + + let expand_request = Request::ExpandMacro(Box::new(ExpandMacro { + lib: dylib_path, + env: vec![], + current_dir: None, + data: ExpandMacroData { + macro_body, + macro_name: "fn_like_panic".to_owned(), + attributes: None, + has_global_spans: ExpnGlobals { + serialize: version >= 3, + def_site: 0, + call_site: 0, + mixed_site: 0, + }, + span_data_table: vec![], + }, + })); + + let response = request(writer, reader, expand_request); + + match response { + Response::ExpandMacro(Err(PanicMessage(msg))) => { + assert!(msg.contains("fn_like_panic"), "panic message should mention the macro"); + } + Response::ExpandMacro(Ok(_)) => { + panic!("expected panic, but macro succeeded"); + } + other => panic!("unexpected response: {other:?}"), + } + }); +} + +#[test] +fn test_basic_call_flow() { + with_server(|writer, reader| { + let dylib_path = proc_macro_test_dylib_path(); + + let response1 = request(writer, reader, Request::ApiVersionCheck {}); + assert!(matches!(response1, Response::ApiVersionCheck(_))); + + let response2 = + request(writer, reader, Request::SetConfig(ServerConfig { span_mode: SpanMode::Id })); + assert!(matches!(response2, Response::SetConfig(_))); + + let response3 = + request(writer, reader, Request::ListMacros { dylib_path: dylib_path.clone() }); + assert!(matches!(response3, Response::ListMacros(Ok(_)))); + }); +} + +#[test] +fn test_expand_nonexistent_macro() { + with_server(|writer, reader| { + let dylib_path = proc_macro_test_dylib_path(); + + let version_response = request(writer, reader, Request::ApiVersionCheck {}); + let Response::ApiVersionCheck(version) = version_response else { + panic!("expected version check response"); + }; + + let mut span_data_table = SpanDataIndexMap::default(); + let macro_body = create_empty_token_tree(version, &mut span_data_table); + + let expand_request = Request::ExpandMacro(Box::new(ExpandMacro { + lib: dylib_path, + env: vec![], + current_dir: None, + data: ExpandMacroData { + macro_body, + macro_name: "NonexistentMacro".to_owned(), + attributes: None, + has_global_spans: ExpnGlobals { + serialize: version >= 3, + def_site: 0, + call_site: 0, + mixed_site: 0, + }, + span_data_table: vec![], + }, + })); + + let response = request(writer, reader, expand_request); + + match response { + Response::ExpandMacro(Err(PanicMessage(msg))) => { + expect!["proc-macro `NonexistentMacro` is missing"].assert_eq(&msg) + } + other => panic!("expected error for nonexistent macro, got: {other:?}"), + } + }); +} From e4dd9475234017adeda572d02320876f98e26775 Mon Sep 17 00:00:00 2001 From: Ada Alakbarova Date: Sat, 10 Jan 2026 19:57:28 +0100 Subject: [PATCH 0177/1843] introduce `Self::as_x_{plus,minus}_one` reduces duplication --- clippy_lints/src/int_plus_one.rs | 91 +++++++++++++++----------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/clippy_lints/src/int_plus_one.rs b/clippy_lints/src/int_plus_one.rs index b82ca0205a9f..f8184b30f400 100644 --- a/clippy_lints/src/int_plus_one.rs +++ b/clippy_lints/src/int_plus_one.rs @@ -63,7 +63,7 @@ impl TryFrom for LeOrGe { impl IntPlusOne { fn is_one(expr: &Expr) -> bool { if let ExprKind::Lit(token_lit) = expr.kind - && let Ok(LitKind::Int(Pu128(1), ..)) = LitKind::from_token_lit(token_lit) + && matches!(LitKind::from_token_lit(token_lit), Ok(LitKind::Int(Pu128(1), ..))) { return true; } @@ -71,60 +71,57 @@ impl IntPlusOne { } fn is_neg_one(expr: &Expr) -> bool { - if let ExprKind::Unary(UnOp::Neg, expr) = &expr.kind - && Self::is_one(expr) - { - true + if let ExprKind::Unary(UnOp::Neg, expr) = &expr.kind { + Self::is_one(expr) } else { false } } + /// Checks whether `expr` is `x + 1` or `1 + x`, and if so, returns `x` + fn as_x_plus_one(expr: &Expr) -> Option<&Expr> { + if let ExprKind::Binary(op, lhs, rhs) = &expr.kind + && op.node == BinOpKind::Add + { + if Self::is_one(rhs) { + // x + 1 + return Some(lhs); + } else if Self::is_one(lhs) { + // 1 + x + return Some(rhs); + } + } + None + } + + /// Checks whether `expr` is `x - 1` or `-1 + x`, and if so, returns `x` + fn as_x_minus_one(expr: &Expr) -> Option<&Expr> { + if let ExprKind::Binary(op, lhs, rhs) = &expr.kind { + if op.node == BinOpKind::Sub && Self::is_one(rhs) { + // x - 1 + return Some(lhs); + } else if op.node == BinOpKind::Add && Self::is_neg_one(lhs) { + // -1 + x + return Some(rhs); + } + } + None + } + fn check_binop<'tcx>(le_or_ge: LeOrGe, lhs: &'tcx Expr, rhs: &'tcx Expr) -> Option<(&'tcx Expr, &'tcx Expr)> { - match (le_or_ge, &lhs.kind, &rhs.kind) { - // case where `x - 1 >= ...` or `-1 + x >= ...` - (LeOrGe::Ge, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) => { - match lhskind.node { - // `-1 + x` - BinOpKind::Add if Self::is_neg_one(lhslhs) => Some((lhsrhs, rhs)), - // `x - 1` - BinOpKind::Sub if Self::is_one(lhsrhs) => Some((lhslhs, rhs)), - _ => None, - } + match le_or_ge { + LeOrGe::Ge => { + // case where `x - 1 >= ...` or `-1 + x >= ...` + (Self::as_x_minus_one(lhs).map(|new_lhs| (new_lhs, rhs))) + // case where `... >= y + 1` or `... >= 1 + y` + .or_else(|| Self::as_x_plus_one(rhs).map(|new_rhs| (lhs, new_rhs))) }, - // case where `... >= y + 1` or `... >= 1 + y` - (LeOrGe::Ge, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) if rhskind.node == BinOpKind::Add => { - // `y + 1` and `1 + y` - if Self::is_one(rhslhs) { - Some((lhs, rhsrhs)) - } else if Self::is_one(rhsrhs) { - Some((lhs, rhslhs)) - } else { - None - } + LeOrGe::Le => { + // case where `x + 1 <= ...` or `1 + x <= ...` + (Self::as_x_plus_one(lhs).map(|new_lhs| (new_lhs, rhs))) + // case where `... <= y - 1` or `... <= -1 + y` + .or_else(|| Self::as_x_minus_one(rhs).map(|new_rhs| (lhs, new_rhs))) }, - // case where `x + 1 <= ...` or `1 + x <= ...` - (LeOrGe::Le, ExprKind::Binary(lhskind, lhslhs, lhsrhs), _) if lhskind.node == BinOpKind::Add => { - // `1 + x` and `x + 1` - if Self::is_one(lhslhs) { - Some((lhsrhs, rhs)) - } else if Self::is_one(lhsrhs) { - Some((lhslhs, rhs)) - } else { - None - } - }, - // case where `... <= y - 1` or `... <= -1 + y` - (LeOrGe::Le, _, ExprKind::Binary(rhskind, rhslhs, rhsrhs)) => { - match rhskind.node { - // `-1 + y` - BinOpKind::Add if Self::is_neg_one(rhslhs) => Some((lhs, rhsrhs)), - // `y - 1` - BinOpKind::Sub if Self::is_one(rhsrhs) => Some((lhs, rhslhs)), - _ => None, - } - }, - _ => None, } } From 306254b40e83189ae65fdda6501b1523d8c19c03 Mon Sep 17 00:00:00 2001 From: relaxcn Date: Sun, 11 Jan 2026 03:51:36 +0800 Subject: [PATCH 0178/1843] improve the doc of `doc_paragraphs_missing_punctuation` --- clippy_lints/src/doc/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clippy_lints/src/doc/mod.rs b/clippy_lints/src/doc/mod.rs index 2b41275ee3a4..ecf7acbd7ce6 100644 --- a/clippy_lints/src/doc/mod.rs +++ b/clippy_lints/src/doc/mod.rs @@ -692,6 +692,12 @@ declare_clippy_lint! { /// /// /// /// It was chosen by a fair dice roll. /// ``` + /// + /// ### Terminal punctuation marks + /// This lint treats these characters as end markers: '.', '?', '!', '…' and ':'. + /// + /// The colon is not exactly a terminal punctuation mark, but this is required for paragraphs that + /// introduce a table or a list for example. #[clippy::version = "1.93.0"] pub DOC_PARAGRAPHS_MISSING_PUNCTUATION, restriction, From e40bd1cf6ece4bf9a2fb05474a20cd399694be93 Mon Sep 17 00:00:00 2001 From: A4-Tacks Date: Sun, 11 Jan 2026 15:06:38 +0800 Subject: [PATCH 0179/1843] Add inherit attributes for extract_function assist Example --- ```rust #[cfg(test)] fn foo() { foo($01 + 1$0); } ``` **Before this PR** ```rust #[cfg(test)] fn foo() { foo(fun_name()); } fn $0fun_name() -> i32 { 1 + 1 } ``` **After this PR** ```rust #[cfg(test)] fn foo() { foo(fun_name()); } #[cfg(test)] fn $0fun_name() -> i32 { 1 + 1 } ``` --- .../src/handlers/extract_function.rs | 69 ++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/extract_function.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/extract_function.rs index 231df9b5b3e1..294e5f7da8b3 100644 --- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/extract_function.rs +++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/extract_function.rs @@ -25,7 +25,7 @@ use syntax::{ SyntaxKind::{self, COMMENT}, SyntaxNode, SyntaxToken, T, TextRange, TextSize, TokenAtOffset, WalkEvent, ast::{ - self, AstNode, AstToken, HasGenericParams, HasName, edit::IndentLevel, + self, AstNode, AstToken, HasAttrs, HasGenericParams, HasName, edit::IndentLevel, edit_in_place::Indent, }, match_ast, ted, @@ -375,6 +375,7 @@ struct ContainerInfo<'db> { ret_type: Option>, generic_param_lists: Vec, where_clauses: Vec, + attrs: Vec, edition: Edition, } @@ -911,6 +912,7 @@ impl FunctionBody { let parents = generic_parents(&parent); let generic_param_lists = parents.iter().filter_map(|it| it.generic_param_list()).collect(); let where_clauses = parents.iter().filter_map(|it| it.where_clause()).collect(); + let attrs = parents.iter().flat_map(|it| it.attrs()).filter(is_inherit_attr).collect(); Some(( ContainerInfo { @@ -919,6 +921,7 @@ impl FunctionBody { ret_type: ty, generic_param_lists, where_clauses, + attrs, edition, }, contains_tail_expr, @@ -1103,6 +1106,14 @@ impl GenericParent { GenericParent::Trait(trait_) => trait_.where_clause(), } } + + fn attrs(&self) -> impl Iterator { + match self { + GenericParent::Fn(fn_) => fn_.attrs(), + GenericParent::Impl(impl_) => impl_.attrs(), + GenericParent::Trait(trait_) => trait_.attrs(), + } + } } /// Search `parent`'s ancestors for items with potentially applicable generic parameters @@ -1578,7 +1589,7 @@ fn format_function( let (generic_params, where_clause) = make_generic_params_and_where_clause(ctx, fun); make::fn_( - None, + fun.mods.attrs.clone(), None, fun_name, generic_params, @@ -1958,6 +1969,11 @@ fn format_type(ty: &hir::Type<'_>, ctx: &AssistContext<'_>, module: hir::Module) ty.display_source_code(ctx.db(), module.into(), true).ok().unwrap_or_else(|| "_".to_owned()) } +fn is_inherit_attr(attr: &ast::Attr) -> bool { + let Some(name) = attr.simple_name() else { return false }; + matches!(name.as_str(), "track_caller" | "cfg") +} + fn make_ty(ty: &hir::Type<'_>, ctx: &AssistContext<'_>, module: hir::Module) -> ast::Type { let ty_str = format_type(ty, ctx, module); make::ty(&ty_str) @@ -6372,6 +6388,55 @@ fn foo() { fn $0fun_name(mut a: i32, mut b: i32) { (a, b) = (b, a); } +"#, + ); + } + + #[test] + fn with_cfg_attr() { + check_assist( + extract_function, + r#" +//- /main.rs crate:main cfg:test +#[cfg(test)] +fn foo() { + foo($01 + 1$0); +} +"#, + r#" +#[cfg(test)] +fn foo() { + foo(fun_name()); +} + +#[cfg(test)] +fn $0fun_name() -> i32 { + 1 + 1 +} +"#, + ); + } + + #[test] + fn with_track_caller() { + check_assist( + extract_function, + r#" +#[track_caller] +fn foo() { + foo($01 + 1$0); +} +"#, + r#" +#[track_caller] +fn foo() { + foo(fun_name()); +} + +#[track_caller] +fn $0fun_name() -> i32 { + 1 + 1 +} "#, ); } From ea93fb548c7c6e838b35893f3f9cc2ed3011aaea Mon Sep 17 00:00:00 2001 From: yukang Date: Sun, 11 Jan 2026 16:30:38 +0800 Subject: [PATCH 0180/1843] Underscore-prefixed bindings are explicitly allowed to be unused --- compiler/rustc_mir_transform/src/liveness.rs | 5 ++++ .../unused/underscore-capture-issue-149889.rs | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/ui/lint/unused/underscore-capture-issue-149889.rs diff --git a/compiler/rustc_mir_transform/src/liveness.rs b/compiler/rustc_mir_transform/src/liveness.rs index 1d1ba455a81e..a5a2fcd71a9a 100644 --- a/compiler/rustc_mir_transform/src/liveness.rs +++ b/compiler/rustc_mir_transform/src/liveness.rs @@ -1086,6 +1086,11 @@ impl<'a, 'tcx> AssignmentResult<'a, 'tcx> { let Some((name, decl_span)) = self.checked_places.names[index] else { continue }; + // By convention, underscore-prefixed bindings are explicitly allowed to be unused. + if name.as_str().starts_with('_') { + continue; + } + let is_maybe_drop_guard = maybe_drop_guard( tcx, self.typing_env, diff --git a/tests/ui/lint/unused/underscore-capture-issue-149889.rs b/tests/ui/lint/unused/underscore-capture-issue-149889.rs new file mode 100644 index 000000000000..3fbaf2588516 --- /dev/null +++ b/tests/ui/lint/unused/underscore-capture-issue-149889.rs @@ -0,0 +1,29 @@ +//@ check-pass +#![deny(unused_assignments)] + +fn lock() -> impl Drop { + struct Handle; + + impl Drop for Handle { + fn drop(&mut self) {} + } + + Handle +} + +fn bar(_f: impl FnMut(bool)) {} + +pub fn foo() { + let mut _handle = None; + bar(move |l| { + if l { + _handle = Some(lock()); + } else { + _handle = None; + } + }) +} + +fn main() { + foo(); +} From 8150413bf5c1327c0a58b544662b50e5ef4bedc2 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Wed, 7 Jan 2026 17:19:33 +0530 Subject: [PATCH 0181/1843] add byte range subrequest/response --- .../crates/proc-macro-api/src/bidirectional_protocol/msg.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs index 0e3b700dcc5a..57e7b1ee8f68 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs @@ -1,5 +1,7 @@ //! Bidirectional protocol messages +use std::ops::Range; + use paths::Utf8PathBuf; use serde::{Deserialize, Serialize}; @@ -14,6 +16,7 @@ pub enum SubRequest { SourceText { file_id: u32, ast_id: u32, start: u32, end: u32 }, LocalFilePath { file_id: u32 }, LineColumn { file_id: u32, ast_id: u32, offset: u32 }, + ByteRange { file_id: u32, ast_id: u32, start: u32, end: u32 }, } #[derive(Debug, Serialize, Deserialize)] @@ -32,6 +35,9 @@ pub enum SubResponse { line: u32, column: u32, }, + ByteRangeResult { + range: Range, + }, } #[derive(Debug, Serialize, Deserialize)] From e909b4b28286d914c1d67f12f419cf1ed88daa8d Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Wed, 7 Jan 2026 17:20:06 +0530 Subject: [PATCH 0182/1843] update proc-macro-srv to include byte-range --- src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs | 3 +++ .../proc-macro-srv/src/server_impl/rust_analyzer_span.rs | 5 ++++- .../rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs index c1ef49a7176b..ac9f89352c2b 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs @@ -41,6 +41,7 @@ use std::{ env, ffi::OsString, fs, + ops::Range, path::{Path, PathBuf}, sync::{Arc, Mutex, PoisonError}, thread, @@ -100,6 +101,8 @@ pub trait ProcMacroClientInterface { fn local_file(&mut self, file_id: span::FileId) -> Option; /// Line and column are 1-based. fn line_column(&mut self, span: Span) -> Option<(u32, u32)>; + + fn byte_range(&mut self, span: Span) -> Range; } const EXPANDER_STACK_SIZE: usize = 8 * 1024 * 1024; diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index 3a25391b573b..9946608247c3 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -162,7 +162,10 @@ impl server::Span for RaSpanServer<'_> { span } fn byte_range(&mut self, span: Self::Span) -> Range { - // FIXME requires db to resolve the ast id, THIS IS NOT INCREMENTAL + if let Some(cb) = self.callback.as_mut() { + return cb.byte_range(span); + } + Range { start: span.range.start().into(), end: span.range.end().into() } } fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs index 81ff1965d68b..b7c5c4fdd21f 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs @@ -4,6 +4,7 @@ use expect_test::Expect; use span::{ EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, }; +use std::ops::Range; use crate::{ EnvSnapshot, ProcMacroClientInterface, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, @@ -137,6 +138,10 @@ impl ProcMacroClientInterface for MockCallback<'_> { // proc_macro uses 1-based line/column Some((line_col.line as u32 + 1, line_col.col as u32 + 1)) } + + fn byte_range(&mut self, span: Span) -> Range { + Range { start: span.range.start().into(), end: span.range.end().into() } + } } pub fn assert_expand_with_callback( From e68a654dca8b43e4f430485249fdef00ed9d85a7 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Wed, 7 Jan 2026 17:20:37 +0530 Subject: [PATCH 0183/1843] add byte range to main loop and direct the request via callback and define the callback on client side --- .../crates/load-cargo/src/lib.rs | 10 ++++++++++ .../proc-macro-srv-cli/src/main_loop.rs | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs index 33468a5003c3..904f704221cf 100644 --- a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs +++ b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs @@ -591,6 +591,16 @@ impl ProcMacroExpander for Expander { Ok(SubResponse::FilePathResult { name }) } + SubRequest::ByteRange { file_id, ast_id, start, end } => { + let range = resolve_sub_span( + db, + file_id, + ast_id, + TextRange::new(TextSize::from(start), TextSize::from(end)), + ); + + Ok(SubResponse::ByteRangeResult { range: range.range.into() }) + } }; match self.0.expand( subtree.view(), diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs index 0c651d22b41b..e35f832716dd 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs @@ -6,7 +6,7 @@ use proc_macro_api::{ transport::codec::{json::JsonProtocol, postcard::PostcardProtocol}, version::CURRENT_API_VERSION, }; -use std::io::{self, BufRead, Write}; +use std::{io, ops::Range}; use legacy::Message; @@ -240,6 +240,23 @@ impl proc_macro_srv::ProcMacroClientInterface for ProcMacroClientHandl _ => None, } } + + fn byte_range( + &mut self, + proc_macro_srv::span::Span { range, anchor, ctx: _ }: proc_macro_srv::span::Span, + ) -> Range { + match self.roundtrip(bidirectional::SubRequest::ByteRange { + file_id: anchor.file_id.as_u32(), + ast_id: anchor.ast_id.into_raw(), + start: range.start().into(), + end: range.end().into(), + }) { + Some(bidirectional::BidirectionalMessage::SubResponse( + bidirectional::SubResponse::ByteRangeResult { range }, + )) => range, + _ => Range { start: range.start().into(), end: range.end().into() }, + } + } } fn handle_expand_ra( From 78d243c751e420c38c3a92f707aae78bafc8ad1e Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 8 Jan 2026 22:43:03 +0530 Subject: [PATCH 0184/1843] add comment on incrementality of subrequest --- src/tools/rust-analyzer/crates/load-cargo/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs index 904f704221cf..8342492a33a4 100644 --- a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs +++ b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs @@ -553,6 +553,7 @@ impl ProcMacroExpander for Expander { Ok(SubResponse::LocalFilePathResult { name }) } + // Not incremental: requires full file text. SubRequest::SourceText { file_id, ast_id, start, end } => { let range = resolve_sub_span( db, @@ -567,6 +568,7 @@ impl ProcMacroExpander for Expander { Ok(SubResponse::SourceTextResult { text }) } + // Not incremental: requires building line index. SubRequest::LineColumn { file_id, ast_id, offset } => { let range = resolve_sub_span(db, file_id, ast_id, TextRange::empty(TextSize::from(offset))); @@ -591,6 +593,7 @@ impl ProcMacroExpander for Expander { Ok(SubResponse::FilePathResult { name }) } + // Not incremental: requires global span resolution. SubRequest::ByteRange { file_id, ast_id, start, end } => { let range = resolve_sub_span( db, From 47b987eccb7c2defece8f053c197836784baf421 Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Sun, 11 Jan 2026 10:15:05 +0100 Subject: [PATCH 0185/1843] Do not show spans from external crates in `missing_trait_methods` Pointing to the missing method definition in external crates will use paths which depend on the system, and even on the Rust compiler version used when the iterator is defined in the standard library. When the trait being implemented and whose method is missing is external, point only to the trait implementation. The user will be able to figure out, or even navigate using their IDE, to the proper definition. As a side-effect, this will remove a large lintcheck churn at every Rustup for this lint. --- clippy_lints/src/missing_trait_methods.rs | 10 +++++++++- tests/ui/missing_trait_methods.rs | 7 +++++++ tests/ui/missing_trait_methods.stderr | 10 +++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/clippy_lints/src/missing_trait_methods.rs b/clippy_lints/src/missing_trait_methods.rs index 8e9400e9d583..7a791f949943 100644 --- a/clippy_lints/src/missing_trait_methods.rs +++ b/clippy_lints/src/missing_trait_methods.rs @@ -1,6 +1,7 @@ use clippy_utils::diagnostics::span_lint_and_then; use clippy_utils::is_lint_allowed; use clippy_utils::macros::span_is_local; +use clippy_utils::source::snippet_opt; use rustc_hir::def_id::DefIdSet; use rustc_hir::{Impl, Item, ItemKind}; use rustc_lint::{LateContext, LateLintPass}; @@ -84,7 +85,14 @@ impl<'tcx> LateLintPass<'tcx> for MissingTraitMethods { cx.tcx.def_span(item.owner_id), format!("missing trait method provided by default: `{}`", assoc.name()), |diag| { - diag.span_help(cx.tcx.def_span(assoc.def_id), "implement the method"); + if assoc.def_id.is_local() { + diag.span_help(cx.tcx.def_span(assoc.def_id), "implement the method"); + } else if let Some(snippet) = snippet_opt(cx, of_trait.trait_ref.path.span) { + diag.help(format!( + "implement the missing `{}` method of the `{snippet}` trait", + assoc.name(), + )); + } }, ); } diff --git a/tests/ui/missing_trait_methods.rs b/tests/ui/missing_trait_methods.rs index 7af186ba3322..67070a299951 100644 --- a/tests/ui/missing_trait_methods.rs +++ b/tests/ui/missing_trait_methods.rs @@ -62,3 +62,10 @@ impl MissingMultiple for Partial {} //~| missing_trait_methods fn main() {} + +//~v missing_trait_methods +impl PartialEq for Partial { + fn eq(&self, other: &Partial) -> bool { + todo!() + } +} diff --git a/tests/ui/missing_trait_methods.stderr b/tests/ui/missing_trait_methods.stderr index d9fb8951ab3d..e5155ad587cb 100644 --- a/tests/ui/missing_trait_methods.stderr +++ b/tests/ui/missing_trait_methods.stderr @@ -60,5 +60,13 @@ help: implement the method LL | fn three() {} | ^^^^^^^^^^ -error: aborting due to 5 previous errors +error: missing trait method provided by default: `ne` + --> tests/ui/missing_trait_methods.rs:67:1 + | +LL | impl PartialEq for Partial { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: implement the missing `ne` method of the `PartialEq` trait + +error: aborting due to 6 previous errors From d30f7c9f7c8c375c2b806b770dd11f3d481979e9 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Sun, 11 Jan 2026 15:51:33 +0530 Subject: [PATCH 0186/1843] add write read imports --- .../rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs index e35f832716dd..3beaeb0697ec 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs @@ -6,7 +6,10 @@ use proc_macro_api::{ transport::codec::{json::JsonProtocol, postcard::PostcardProtocol}, version::CURRENT_API_VERSION, }; -use std::{io, ops::Range}; +use std::{ + io::{self, BufRead, Write}, + ops::Range, +}; use legacy::Message; From c7c5adbe6452c1c904fbe6b6bb3fdd0ed7671a55 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sun, 11 Jan 2026 07:00:54 -0600 Subject: [PATCH 0187/1843] compiler-builtins: Remove the no-f16-f128 feature This option was used to gate `f16` and `f128` when support across backends and targets was inconsistent. We now have the rustc builtin cfg `target_has_reliable{f16,f128}` which has taken over this usecase. Remove no-f16-f128 since it is now unused and redundant. --- library/alloc/Cargo.toml | 1 - library/compiler-builtins/builtins-shim/Cargo.toml | 4 ---- library/compiler-builtins/builtins-test/Cargo.toml | 1 - library/compiler-builtins/ci/run.sh | 4 ---- library/compiler-builtins/compiler-builtins/Cargo.toml | 4 ---- library/compiler-builtins/compiler-builtins/configure.rs | 7 ++----- library/compiler-builtins/libm/configure.rs | 7 ++----- library/std/Cargo.toml | 1 - library/sysroot/Cargo.toml | 1 - 9 files changed, 4 insertions(+), 26 deletions(-) diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml index fb1f8c86dbfd..541257b6cda6 100644 --- a/library/alloc/Cargo.toml +++ b/library/alloc/Cargo.toml @@ -21,7 +21,6 @@ compiler_builtins = { path = "../compiler-builtins/compiler-builtins", features [features] compiler-builtins-mem = ['compiler_builtins/mem'] compiler-builtins-c = ["compiler_builtins/c"] -compiler-builtins-no-f16-f128 = ["compiler_builtins/no-f16-f128"] # Choose algorithms that are optimized for binary size instead of runtime performance optimize_for_size = ["core/optimize_for_size"] diff --git a/library/compiler-builtins/builtins-shim/Cargo.toml b/library/compiler-builtins/builtins-shim/Cargo.toml index ac77224f5ce1..746d5b21dc3f 100644 --- a/library/compiler-builtins/builtins-shim/Cargo.toml +++ b/library/compiler-builtins/builtins-shim/Cargo.toml @@ -47,10 +47,6 @@ c = ["dep:cc"] # the generic versions on all platforms. no-asm = [] -# Workaround for codegen backends which haven't yet implemented `f16` and -# `f128` support. Disabled any intrinsics which use those types. -no-f16-f128 = [] - # Flag this library as the unstable compiler-builtins lib compiler-builtins = [] diff --git a/library/compiler-builtins/builtins-test/Cargo.toml b/library/compiler-builtins/builtins-test/Cargo.toml index 9346ea65420b..550f736a76db 100644 --- a/library/compiler-builtins/builtins-test/Cargo.toml +++ b/library/compiler-builtins/builtins-test/Cargo.toml @@ -33,7 +33,6 @@ utest-macros = { git = "https://github.com/japaric/utest" } default = ["mangled-names"] c = ["compiler_builtins/c"] no-asm = ["compiler_builtins/no-asm"] -no-f16-f128 = ["compiler_builtins/no-f16-f128"] mem = ["compiler_builtins/mem"] mangled-names = ["compiler_builtins/mangled-names"] # Skip tests that rely on f128 symbols being available on the system diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh index bc94d42fe837..0c07b32c74b9 100755 --- a/library/compiler-builtins/ci/run.sh +++ b/library/compiler-builtins/ci/run.sh @@ -36,8 +36,6 @@ else "${test_builtins[@]}" --features c --release "${test_builtins[@]}" --features no-asm "${test_builtins[@]}" --features no-asm --release - "${test_builtins[@]}" --features no-f16-f128 - "${test_builtins[@]}" --features no-f16-f128 --release "${test_builtins[@]}" --benches "${test_builtins[@]}" --benches --release @@ -63,8 +61,6 @@ symcheck+=(-- build-and-check) "${symcheck[@]}" "$target" -- -p compiler_builtins --features c --release "${symcheck[@]}" "$target" -- -p compiler_builtins --features no-asm "${symcheck[@]}" "$target" -- -p compiler_builtins --features no-asm --release -"${symcheck[@]}" "$target" -- -p compiler_builtins --features no-f16-f128 -"${symcheck[@]}" "$target" -- -p compiler_builtins --features no-f16-f128 --release run_intrinsics_test() { build_args=(--verbose --manifest-path builtins-test-intrinsics/Cargo.toml) diff --git a/library/compiler-builtins/compiler-builtins/Cargo.toml b/library/compiler-builtins/compiler-builtins/Cargo.toml index 0845861dcfe3..496dde2d4cf2 100644 --- a/library/compiler-builtins/compiler-builtins/Cargo.toml +++ b/library/compiler-builtins/compiler-builtins/Cargo.toml @@ -45,10 +45,6 @@ c = ["dep:cc"] # the generic versions on all platforms. no-asm = [] -# Workaround for codegen backends which haven't yet implemented `f16` and -# `f128` support. Disabled any intrinsics which use those types. -no-f16-f128 = [] - # Flag this library as the unstable compiler-builtins lib compiler-builtins = [] diff --git a/library/compiler-builtins/compiler-builtins/configure.rs b/library/compiler-builtins/compiler-builtins/configure.rs index 79e238abc0f6..f16da6b58f81 100644 --- a/library/compiler-builtins/compiler-builtins/configure.rs +++ b/library/compiler-builtins/compiler-builtins/configure.rs @@ -95,16 +95,13 @@ pub fn configure_aliases(target: &Target) { * * https://github.com/rust-lang/rustc_codegen_cranelift/blob/c713ffab3c6e28ab4b4dd4e392330f786ea657ad/src/lib.rs#L196-L226 */ - // If the feature is set, disable both of these types. - let no_f16_f128 = target.cargo_features.iter().any(|s| s == "no-f16-f128"); - println!("cargo::rustc-check-cfg=cfg(f16_enabled)"); - if target.reliable_f16 && !no_f16_f128 { + if target.reliable_f16 { println!("cargo::rustc-cfg=f16_enabled"); } println!("cargo::rustc-check-cfg=cfg(f128_enabled)"); - if target.reliable_f128 && !no_f16_f128 { + if target.reliable_f128 { println!("cargo::rustc-cfg=f128_enabled"); } } diff --git a/library/compiler-builtins/libm/configure.rs b/library/compiler-builtins/libm/configure.rs index 857a30229716..ee65a3a8d624 100644 --- a/library/compiler-builtins/libm/configure.rs +++ b/library/compiler-builtins/libm/configure.rs @@ -143,16 +143,13 @@ fn emit_f16_f128_cfg(cfg: &Config) { /* See the compiler-builtins configure file for info about the meaning of these options */ - // If the feature is set, disable both of these types. - let no_f16_f128 = cfg.cargo_features.iter().any(|s| s == "no-f16-f128"); - println!("cargo:rustc-check-cfg=cfg(f16_enabled)"); - if cfg.reliable_f16 && !no_f16_f128 { + if cfg.reliable_f16 { println!("cargo:rustc-cfg=f16_enabled"); } println!("cargo:rustc-check-cfg=cfg(f128_enabled)"); - if cfg.reliable_f128 && !no_f16_f128 { + if cfg.reliable_f128 { println!("cargo:rustc-cfg=f128_enabled"); } } diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index 5c9ae52d9e6c..b6683a2ae9ec 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -115,7 +115,6 @@ backtrace-trace-only = [] panic-unwind = ["dep:panic_unwind"] compiler-builtins-c = ["alloc/compiler-builtins-c"] compiler-builtins-mem = ["alloc/compiler-builtins-mem"] -compiler-builtins-no-f16-f128 = ["alloc/compiler-builtins-no-f16-f128"] llvm-libunwind = ["unwind/llvm-libunwind"] system-llvm-libunwind = ["unwind/system-llvm-libunwind"] diff --git a/library/sysroot/Cargo.toml b/library/sysroot/Cargo.toml index a18868082916..b2069ef6a613 100644 --- a/library/sysroot/Cargo.toml +++ b/library/sysroot/Cargo.toml @@ -25,7 +25,6 @@ backtrace = ["std/backtrace"] backtrace-trace-only = ["std/backtrace-trace-only"] compiler-builtins-c = ["std/compiler-builtins-c"] compiler-builtins-mem = ["std/compiler-builtins-mem"] -compiler-builtins-no-f16-f128 = ["std/compiler-builtins-no-f16-f128"] debug_refcell = ["std/debug_refcell"] llvm-libunwind = ["std/llvm-libunwind"] system-llvm-libunwind = ["std/system-llvm-libunwind"] From e21256031782bdb355f3ff2e4ed126360b9cc7ab Mon Sep 17 00:00:00 2001 From: Pavel Grigorenko Date: Sun, 9 Nov 2025 23:17:09 +0300 Subject: [PATCH 0188/1843] Stabilize `alloc_layout_extra` --- ...sroot_tests-128bit-atomic-operations.patch | 3 +- library/alloc/src/alloc.rs | 6 +- library/alloc/src/boxed.rs | 2 +- library/alloc/src/boxed/thin.rs | 4 +- library/alloc/src/lib.rs | 1 - library/alloc/src/rc.rs | 8 +-- library/alloc/src/sync.rs | 8 +-- library/alloctests/lib.rs | 1 - library/alloctests/tests/boxed.rs | 2 +- library/alloctests/tests/lib.rs | 1 - library/core/src/alloc/layout.rs | 65 +++++++++++-------- library/coretests/tests/alloc.rs | 2 +- library/coretests/tests/lib.rs | 1 - library/std/src/alloc.rs | 4 +- library/std/src/lib.rs | 1 - 15 files changed, 58 insertions(+), 51 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch index f3d1d5c43ea1..6ed0b17f679c 100644 --- a/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch +++ b/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch @@ -14,11 +14,10 @@ diff --git a/coretests/tests/lib.rs b/coretests/tests/lib.rs index 1e336bf..35e6f54 100644 --- a/coretests/tests/lib.rs +++ b/coretests/tests/lib.rs -@@ -2,5 +2,4 @@ +@@ -2,4 +2,3 @@ // tidy-alphabetical-start -#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))] #![cfg_attr(test, feature(cfg_select))] - #![feature(alloc_layout_extra)] #![feature(array_ptr_get)] diff --git a/coretests/tests/atomic.rs b/coretests/tests/atomic.rs index b735957..ea728b6 100644 diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index 5dd828bd54e1..cd1c2ea8fcd1 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -184,7 +184,7 @@ impl Global { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces fn alloc_impl_runtime(layout: Layout, zeroed: bool) -> Result, AllocError> { match layout.size() { - 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling_ptr(), 0)), // SAFETY: `layout` is non-zero in size, size => unsafe { let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) }; @@ -277,7 +277,7 @@ impl Global { // SAFETY: conditions must be upheld by the caller 0 => unsafe { self.deallocate(ptr, old_layout); - Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) + Ok(NonNull::slice_from_raw_parts(new_layout.dangling_ptr(), 0)) }, // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller @@ -368,7 +368,7 @@ impl Global { #[rustc_const_unstable(feature = "const_heap", issue = "79597")] const fn alloc_impl_const(layout: Layout, zeroed: bool) -> Result, AllocError> { match layout.size() { - 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling_ptr(), 0)), // SAFETY: `layout` is non-zero in size, size => unsafe { let raw_ptr = core::intrinsics::const_allocate(layout.size(), layout.align()); diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs index 60758551cc04..eb741c9211f2 100644 --- a/library/alloc/src/boxed.rs +++ b/library/alloc/src/boxed.rs @@ -834,7 +834,7 @@ impl Box { } let layout = Layout::for_value::(src); let (ptr, guard) = if layout.size() == 0 { - (layout.dangling(), None) + (layout.dangling_ptr(), None) } else { // Safety: layout is non-zero-sized let ptr = alloc.allocate(layout)?.cast(); diff --git a/library/alloc/src/boxed/thin.rs b/library/alloc/src/boxed/thin.rs index 1cce36606d2c..b50810b8d923 100644 --- a/library/alloc/src/boxed/thin.rs +++ b/library/alloc/src/boxed/thin.rs @@ -245,7 +245,7 @@ impl WithHeader { // Some paranoia checking, mostly so that the ThinBox tests are // more able to catch issues. debug_assert!(value_offset == 0 && T::IS_ZST && H::IS_ZST); - layout.dangling() + layout.dangling_ptr() } else { let ptr = alloc::alloc(layout); if ptr.is_null() { @@ -282,7 +282,7 @@ impl WithHeader { // Some paranoia checking, mostly so that the ThinBox tests are // more able to catch issues. debug_assert!(value_offset == 0 && size_of::() == 0 && size_of::() == 0); - layout.dangling() + layout.dangling_ptr() } else { let ptr = alloc::alloc(layout); if ptr.is_null() { diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 3ca9f3ae0bc9..f7167650635d 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -86,7 +86,6 @@ // Library features: // tidy-alphabetical-start #![cfg_attr(not(no_global_oom_handling), feature(string_replace_in_place))] -#![feature(alloc_layout_extra)] #![feature(allocator_api)] #![feature(array_into_iter_constructors)] #![feature(ascii_char)] diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs index 8a72748668cc..f58ebd488d7c 100644 --- a/library/alloc/src/rc.rs +++ b/library/alloc/src/rc.rs @@ -261,7 +261,7 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; #[cfg(not(no_global_oom_handling))] use core::pin::Pin; use core::pin::PinCoerceUnsized; -use core::ptr::{self, NonNull, drop_in_place}; +use core::ptr::{self, Alignment, NonNull, drop_in_place}; #[cfg(not(no_global_oom_handling))] use core::slice::from_raw_parts_mut; use core::{borrow, fmt, hint}; @@ -3847,11 +3847,11 @@ unsafe fn data_offset(ptr: *const T) -> usize { // and extern types, the input safety requirement is currently enough to // satisfy the requirements of align_of_val_raw; this is an implementation // detail of the language that must not be relied upon outside of std. - unsafe { data_offset_align(align_of_val_raw(ptr)) } + unsafe { data_offset_align(Alignment::new_unchecked(align_of_val_raw(ptr))) } } #[inline] -fn data_offset_align(align: usize) -> usize { +fn data_offset_align(align: Alignment) -> usize { let layout = Layout::new::>(); layout.size() + layout.padding_needed_for(align) } @@ -4478,7 +4478,7 @@ impl UniqueRcUninit { /// Returns the pointer to be written into to initialize the [`Rc`]. fn data_ptr(&mut self) -> *mut T { - let offset = data_offset_align(self.layout_for_value.align()); + let offset = data_offset_align(self.layout_for_value.alignment()); unsafe { self.ptr.as_ptr().byte_add(offset) as *mut T } } diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs index 4180fe91cb55..fc44a468c8a4 100644 --- a/library/alloc/src/sync.rs +++ b/library/alloc/src/sync.rs @@ -26,7 +26,7 @@ use core::ops::{CoerceUnsized, Deref, DerefMut, DerefPure, DispatchFromDyn, Lega use core::ops::{Residual, Try}; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::{Pin, PinCoerceUnsized}; -use core::ptr::{self, NonNull}; +use core::ptr::{self, Alignment, NonNull}; #[cfg(not(no_global_oom_handling))] use core::slice::from_raw_parts_mut; use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; @@ -4208,11 +4208,11 @@ unsafe fn data_offset(ptr: *const T) -> usize { // and extern types, the input safety requirement is currently enough to // satisfy the requirements of align_of_val_raw; this is an implementation // detail of the language that must not be relied upon outside of std. - unsafe { data_offset_align(align_of_val_raw(ptr)) } + unsafe { data_offset_align(Alignment::new_unchecked(align_of_val_raw(ptr))) } } #[inline] -fn data_offset_align(align: usize) -> usize { +fn data_offset_align(align: Alignment) -> usize { let layout = Layout::new::>(); layout.size() + layout.padding_needed_for(align) } @@ -4258,7 +4258,7 @@ impl UniqueArcUninit { /// Returns the pointer to be written into to initialize the [`Arc`]. fn data_ptr(&mut self) -> *mut T { - let offset = data_offset_align(self.layout_for_value.align()); + let offset = data_offset_align(self.layout_for_value.alignment()); unsafe { self.ptr.as_ptr().byte_add(offset) as *mut T } } diff --git a/library/alloctests/lib.rs b/library/alloctests/lib.rs index b85fc8eb9970..32a876cdc5f7 100644 --- a/library/alloctests/lib.rs +++ b/library/alloctests/lib.rs @@ -14,7 +14,6 @@ // // Library features: // tidy-alphabetical-start -#![feature(alloc_layout_extra)] #![feature(allocator_api)] #![feature(array_into_iter_constructors)] #![feature(assert_matches)] diff --git a/library/alloctests/tests/boxed.rs b/library/alloctests/tests/boxed.rs index 94389cf2de93..83fd1ef7449a 100644 --- a/library/alloctests/tests/boxed.rs +++ b/library/alloctests/tests/boxed.rs @@ -104,7 +104,7 @@ pub struct ConstAllocator; unsafe impl Allocator for ConstAllocator { fn allocate(&self, layout: Layout) -> Result, AllocError> { match layout.size() { - 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling_ptr(), 0)), _ => unsafe { let ptr = core::intrinsics::const_allocate(layout.size(), layout.align()); Ok(NonNull::new_unchecked(ptr as *mut [u8; 0] as *mut [u8])) diff --git a/library/alloctests/tests/lib.rs b/library/alloctests/tests/lib.rs index c7fdc2054f71..2926248edbf5 100644 --- a/library/alloctests/tests/lib.rs +++ b/library/alloctests/tests/lib.rs @@ -1,5 +1,4 @@ #![feature(allocator_api)] -#![feature(alloc_layout_extra)] #![feature(const_heap)] #![feature(deque_extend_front)] #![feature(iter_array_chunks)] diff --git a/library/core/src/alloc/layout.rs b/library/core/src/alloc/layout.rs index a4e25b8734a9..3a2111350a4e 100644 --- a/library/core/src/alloc/layout.rs +++ b/library/core/src/alloc/layout.rs @@ -217,10 +217,11 @@ impl Layout { /// be that of a valid pointer, which means this must not be used /// as a "not yet initialized" sentinel value. /// Types that lazily allocate must track initialization by some other means. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] #[must_use] #[inline] - pub const fn dangling(&self) -> NonNull { + pub const fn dangling_ptr(&self) -> NonNull { NonNull::without_provenance(self.align.as_nonzero()) } @@ -250,29 +251,23 @@ impl Layout { } /// Returns the amount of padding we must insert after `self` - /// to ensure that the following address will satisfy `align` - /// (measured in bytes). + /// to ensure that the following address will satisfy `alignment`. /// - /// e.g., if `self.size()` is 9, then `self.padding_needed_for(4)` + /// e.g., if `self.size()` is 9, then `self.padding_needed_for(alignment4)` + /// (where `alignment4.as_usize() == 4`) /// returns 3, because that is the minimum number of bytes of /// padding required to get a 4-aligned address (assuming that the /// corresponding memory block starts at a 4-aligned address). /// - /// The return value of this function has no meaning if `align` is - /// not a power-of-two. - /// - /// Note that the utility of the returned value requires `align` + /// Note that the utility of the returned value requires `alignment` /// to be less than or equal to the alignment of the starting /// address for the whole allocated block of memory. One way to - /// satisfy this constraint is to ensure `align <= self.align()`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] - #[must_use = "this returns the padding needed, \ - without modifying the `Layout`"] + /// satisfy this constraint is to ensure `alignment.as_usize() <= self.align()`. + #[unstable(feature = "ptr_alignment_type", issue = "102070")] + #[must_use = "this returns the padding needed, without modifying the `Layout`"] #[inline] - pub const fn padding_needed_for(&self, align: usize) -> usize { - // FIXME: Can we just change the type on this to `Alignment`? - let Some(align) = Alignment::new(align) else { return usize::MAX }; - let len_rounded_up = self.size_rounded_up_to_custom_align(align); + pub const fn padding_needed_for(&self, alignment: Alignment) -> usize { + let len_rounded_up = self.size_rounded_up_to_custom_align(alignment); // SAFETY: Cannot overflow because the rounded-up value is never less unsafe { unchecked_sub(len_rounded_up, self.size) } } @@ -335,6 +330,8 @@ impl Layout { /// layout of the array and `offs` is the distance between the start /// of each element in the array. /// + /// Does not include padding after the trailing element. + /// /// (That distance between elements is sometimes known as "stride".) /// /// On arithmetic overflow, returns `LayoutError`. @@ -342,7 +339,6 @@ impl Layout { /// # Examples /// /// ``` - /// #![feature(alloc_layout_extra)] /// use std::alloc::Layout; /// /// // All rust types have a size that's a multiple of their alignment. @@ -353,17 +349,32 @@ impl Layout { /// // But you can manually make layouts which don't meet that rule. /// let padding_needed = Layout::from_size_align(6, 4).unwrap(); /// let repeated = padding_needed.repeat(3).unwrap(); - /// assert_eq!(repeated, (Layout::from_size_align(24, 4).unwrap(), 8)); + /// assert_eq!(repeated, (Layout::from_size_align(22, 4).unwrap(), 8)); + /// + /// // Repeating an element zero times has zero size, but keeps the alignment (like `[T; 0]`) + /// let repeated = normal.repeat(0).unwrap(); + /// assert_eq!(repeated, (Layout::from_size_align(0, 4).unwrap(), 12)); + /// let repeated = padding_needed.repeat(0).unwrap(); + /// assert_eq!(repeated, (Layout::from_size_align(0, 4).unwrap(), 8)); /// ``` - #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] #[inline] pub const fn repeat(&self, n: usize) -> Result<(Self, usize), LayoutError> { + // FIXME(const-hack): the following could be way shorter with `?` let padded = self.pad_to_align(); - if let Ok(repeated) = padded.repeat_packed(n) { - Ok((repeated, padded.size())) + let Ok(result) = (if let Some(k) = n.checked_sub(1) { + let Ok(repeated) = padded.repeat_packed(k) else { + return Err(LayoutError); + }; + repeated.extend_packed(*self) } else { - Err(LayoutError) - } + debug_assert!(n == 0); + self.repeat_packed(0) + }) else { + return Err(LayoutError); + }; + Ok((result, padded.size())) } /// Creates a layout describing the record for `self` followed by @@ -443,7 +454,8 @@ impl Layout { /// aligned. /// /// On arithmetic overflow, returns `LayoutError`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] #[inline] pub const fn repeat_packed(&self, n: usize) -> Result { if let Some(size) = self.size.checked_mul(n) { @@ -460,7 +472,8 @@ impl Layout { /// and is not incorporated *at all* into the resulting layout. /// /// On arithmetic overflow, returns `LayoutError`. - #[unstable(feature = "alloc_layout_extra", issue = "55724")] + #[stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "alloc_layout_extra", since = "CURRENT_RUSTC_VERSION")] #[inline] pub const fn extend_packed(&self, next: Self) -> Result { // SAFETY: each `size` is at most `isize::MAX == usize::MAX/2`, so the diff --git a/library/coretests/tests/alloc.rs b/library/coretests/tests/alloc.rs index a4af6fd32a11..aac1d60ce32c 100644 --- a/library/coretests/tests/alloc.rs +++ b/library/coretests/tests/alloc.rs @@ -6,7 +6,7 @@ fn const_unchecked_layout() { const SIZE: usize = 0x2000; const ALIGN: usize = 0x1000; const LAYOUT: Layout = unsafe { Layout::from_size_align_unchecked(SIZE, ALIGN) }; - const DANGLING: NonNull = LAYOUT.dangling(); + const DANGLING: NonNull = LAYOUT.dangling_ptr(); assert_eq!(LAYOUT.size(), SIZE); assert_eq!(LAYOUT.align(), ALIGN); assert_eq!(Some(DANGLING), NonNull::new(ptr::without_provenance_mut(ALIGN))); diff --git a/library/coretests/tests/lib.rs b/library/coretests/tests/lib.rs index b28e7338859c..4974f52cd9df 100644 --- a/library/coretests/tests/lib.rs +++ b/library/coretests/tests/lib.rs @@ -1,7 +1,6 @@ // tidy-alphabetical-start #![cfg_attr(target_has_atomic = "128", feature(integer_atomics))] #![cfg_attr(test, feature(cfg_select))] -#![feature(alloc_layout_extra)] #![feature(array_ptr_get)] #![feature(array_try_from_fn)] #![feature(array_try_map)] diff --git a/library/std/src/alloc.rs b/library/std/src/alloc.rs index f0d2b913bbbb..ed0322e2cf5d 100644 --- a/library/std/src/alloc.rs +++ b/library/std/src/alloc.rs @@ -142,7 +142,7 @@ impl System { #[inline] fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result, AllocError> { match layout.size() { - 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling_ptr(), 0)), // SAFETY: `layout` is non-zero in size, size => unsafe { let raw_ptr = if zeroed { @@ -266,7 +266,7 @@ unsafe impl Allocator for System { // SAFETY: conditions must be upheld by the caller 0 => unsafe { Allocator::deallocate(self, ptr, old_layout); - Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) + Ok(NonNull::slice_from_raw_parts(new_layout.dangling_ptr(), 0)) }, // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index f5b9f69a5f9f..4f48f23f44b8 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -367,7 +367,6 @@ // // Library features (alloc): // tidy-alphabetical-start -#![feature(alloc_layout_extra)] #![feature(allocator_api)] #![feature(clone_from_ref)] #![feature(get_mut_unchecked)] From 33f0f80c1ac9dd1df22082cce77e7eedbb30b4f1 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Sun, 11 Jan 2026 21:33:57 +0530 Subject: [PATCH 0189/1843] add make corresponding constructor methods in SyntaxFactory --- .../src/ast/syntax_factory/constructors.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs index 7cf9e2bf14f9..5fe419ad4eb7 100644 --- a/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs +++ b/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs @@ -1578,6 +1578,44 @@ impl SyntaxFactory { pub fn ident(&self, text: &str) -> SyntaxToken { make::tokens::ident(text) } + + pub fn mut_self_param(&self) -> ast::SelfParam { + let ast = make::mut_self_param().clone_for_update(); + + if let Some(mut mapping) = self.mappings() { + let builder = SyntaxMappingBuilder::new(ast.syntax().clone()); + builder.finish(&mut mapping); + } + + ast + } + + pub fn ret_type(&self, ty: ast::Type) -> ast::RetType { + let ast = make::ret_type(ty.clone()).clone_for_update(); + + if let Some(mut mapping) = self.mappings() { + let mut builder = SyntaxMappingBuilder::new(ast.syntax().clone()); + builder.map_node(ty.syntax().clone(), ast.ty().unwrap().syntax().clone()); + builder.finish(&mut mapping); + } + ast + } + + pub fn ty_ref(&self, ty: ast::Type, is_mut: bool) -> ast::Type { + let ast = make::ty_ref(ty.clone(), is_mut).clone_for_update(); + + if let Some(mut mapping) = self.mappings() { + let mut builder = SyntaxMappingBuilder::new(ast.syntax().clone()); + match &ast { + ast::Type::RefType(ref_ty) => { + builder.map_node(ty.syntax().clone(), ref_ty.ty().unwrap().syntax().clone()); + } + _ => unreachable!(), + } + builder.finish(&mut mapping); + } + ast + } } // `ext` constructors From 16d74e7b90fc5fe2aaa23377bf51093f0295910b Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Sun, 11 Jan 2026 21:34:44 +0530 Subject: [PATCH 0190/1843] migrate generate_mut_trait_impl to new SyntaxEditor --- .../src/handlers/generate_mut_trait_impl.rs | 191 +++++++++++------- 1 file changed, 121 insertions(+), 70 deletions(-) diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_mut_trait_impl.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_mut_trait_impl.rs index ae1ae24d1ec1..53f6f4883f4d 100644 --- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_mut_trait_impl.rs +++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_mut_trait_impl.rs @@ -1,8 +1,8 @@ use ide_db::{famous_defs::FamousDefs, traits::resolve_target_trait}; use syntax::{ - AstNode, T, - ast::{self, edit_in_place::Indent, make}, - ted, + AstNode, SyntaxElement, SyntaxNode, T, + ast::{self, edit::AstNodeEdit, edit_in_place::Indent, syntax_factory::SyntaxFactory}, + syntax_editor::{Element, Position, SyntaxEditor}, }; use crate::{AssistContext, AssistId, Assists}; @@ -45,12 +45,13 @@ use crate::{AssistContext, AssistId, Assists}; // } // ``` pub(crate) fn generate_mut_trait_impl(acc: &mut Assists, ctx: &AssistContext<'_>) -> Option<()> { - let impl_def = ctx.find_node_at_offset::()?.clone_for_update(); - let indent = impl_def.indent_level(); + let impl_def = ctx.find_node_at_offset::()?; + let indent = Indent::indent_level(&impl_def); let ast::Type::PathType(path) = impl_def.trait_()? else { return None; }; + let trait_name = path.path()?.segment()?.name_ref()?; let scope = ctx.sema.scope(impl_def.trait_()?.syntax())?; @@ -59,75 +60,133 @@ pub(crate) fn generate_mut_trait_impl(acc: &mut Assists, ctx: &AssistContext<'_> let trait_ = resolve_target_trait(&ctx.sema, &impl_def)?; let trait_new = get_trait_mut(&trait_, famous)?; - // Index -> IndexMut - ted::replace(trait_name.syntax(), make::name_ref(trait_new).clone_for_update().syntax()); - - // index -> index_mut - let (trait_method_name, new_trait_method_name) = impl_def - .syntax() - .descendants() - .filter_map(ast::Name::cast) - .find_map(process_method_name)?; - ted::replace( - trait_method_name.syntax(), - make::name(new_trait_method_name).clone_for_update().syntax(), - ); - - if let Some(type_alias) = impl_def.syntax().descendants().find_map(ast::TypeAlias::cast) { - ted::remove(type_alias.syntax()); - } - - // &self -> &mut self - let mut_self_param = make::mut_self_param(); - let self_param: ast::SelfParam = - impl_def.syntax().descendants().find_map(ast::SelfParam::cast)?; - ted::replace(self_param.syntax(), mut_self_param.clone_for_update().syntax()); - - // &Self::Output -> &mut Self::Output - let ret_type = impl_def.syntax().descendants().find_map(ast::RetType::cast)?; - let new_ret_type = process_ret_type(&ret_type)?; - ted::replace(ret_type.syntax(), make::ret_type(new_ret_type).clone_for_update().syntax()); - - let fn_ = impl_def.assoc_item_list()?.assoc_items().find_map(|it| match it { - ast::AssocItem::Fn(f) => Some(f), - _ => None, - })?; - let _ = process_ref_mut(&fn_); - - let assoc_list = make::assoc_item_list(None).clone_for_update(); - ted::replace(impl_def.assoc_item_list()?.syntax(), assoc_list.syntax()); - impl_def.get_or_create_assoc_item_list().add_item(syntax::ast::AssocItem::Fn(fn_)); - let target = impl_def.syntax().text_range(); + acc.add( AssistId::generate("generate_mut_trait_impl"), format!("Generate `{trait_new}` impl from this `{trait_name}` trait"), target, |edit| { - edit.insert( - target.start(), - if ctx.config.snippet_cap.is_some() { - format!("$0{impl_def}\n\n{indent}") - } else { - format!("{impl_def}\n\n{indent}") - }, + let impl_clone = impl_def.reset_indent().clone_subtree(); + let mut editor = SyntaxEditor::new(impl_clone.syntax().clone()); + let factory = SyntaxFactory::without_mappings(); + + apply_generate_mut_impl(&mut editor, &factory, &impl_clone, trait_new); + + let new_root = editor.finish(); + let new_root = new_root.new_root(); + + let new_impl = ast::Impl::cast(new_root.clone()).unwrap(); + + Indent::indent(&new_impl, indent); + + let mut editor = edit.make_editor(impl_def.syntax()); + editor.insert_all( + Position::before(impl_def.syntax()), + vec![ + new_impl.syntax().syntax_element(), + factory.whitespace(&format!("\n\n{indent}")).syntax_element(), + ], ); + + if let Some(cap) = ctx.config.snippet_cap { + let tabstop_before = edit.make_tabstop_before(cap); + editor.add_annotation(new_impl.syntax(), tabstop_before); + } + + edit.add_file_edits(ctx.vfs_file_id(), editor); }, ) } -fn process_ref_mut(fn_: &ast::Fn) -> Option<()> { - let expr = fn_.body()?.tail_expr()?; - match &expr { - ast::Expr::RefExpr(ref_expr) if ref_expr.mut_token().is_none() => { - ted::insert_all_raw( - ted::Position::after(ref_expr.amp_token()?), - vec![make::token(T![mut]).into(), make::tokens::whitespace(" ").into()], - ); - } - _ => {} +fn delete_with_trivia(editor: &mut SyntaxEditor, node: &SyntaxNode) { + let mut end: SyntaxElement = node.clone().into(); + + if let Some(next) = node.next_sibling_or_token() + && let SyntaxElement::Token(tok) = &next + && tok.kind().is_trivia() + { + end = next.clone(); } - None + + editor.delete_all(node.clone().into()..=end); +} + +fn apply_generate_mut_impl( + editor: &mut SyntaxEditor, + factory: &SyntaxFactory, + impl_def: &ast::Impl, + trait_new: &str, +) -> Option<()> { + let path = + impl_def.trait_().and_then(|t| t.syntax().descendants().find_map(ast::Path::cast))?; + let seg = path.segment()?; + let name_ref = seg.name_ref()?; + + let new_name_ref = factory.name_ref(trait_new); + editor.replace(name_ref.syntax(), new_name_ref.syntax()); + + if let Some((name, new_name)) = + impl_def.syntax().descendants().filter_map(ast::Name::cast).find_map(process_method_name) + { + let new_name_node = factory.name(new_name); + editor.replace(name.syntax(), new_name_node.syntax()); + } + + if let Some(type_alias) = impl_def.syntax().descendants().find_map(ast::TypeAlias::cast) { + delete_with_trivia(editor, type_alias.syntax()); + } + + if let Some(self_param) = impl_def.syntax().descendants().find_map(ast::SelfParam::cast) { + let mut_self = factory.mut_self_param(); + editor.replace(self_param.syntax(), mut_self.syntax()); + } + + if let Some(ret_type) = impl_def.syntax().descendants().find_map(ast::RetType::cast) + && let Some(new_ty) = process_ret_type(factory, &ret_type) + { + let new_ret = factory.ret_type(new_ty); + editor.replace(ret_type.syntax(), new_ret.syntax()) + } + + if let Some(fn_) = impl_def.assoc_item_list().and_then(|l| { + l.assoc_items().find_map(|it| match it { + ast::AssocItem::Fn(f) => Some(f), + _ => None, + }) + }) { + process_ref_mut(editor, factory, &fn_); + } + + Some(()) +} + +fn process_ref_mut(editor: &mut SyntaxEditor, factory: &SyntaxFactory, fn_: &ast::Fn) { + let Some(expr) = fn_.body().and_then(|b| b.tail_expr()) else { return }; + + let ast::Expr::RefExpr(ref_expr) = expr else { return }; + + if ref_expr.mut_token().is_some() { + return; + } + + let Some(amp) = ref_expr.amp_token() else { return }; + + let mut_kw = factory.token(T![mut]); + let space = factory.whitespace(" "); + + editor.insert(Position::after(amp.clone()), space.syntax_element()); + editor.insert(Position::after(amp), mut_kw.syntax_element()); +} + +fn process_ret_type(factory: &SyntaxFactory, ref_ty: &ast::RetType) -> Option { + let ty = ref_ty.ty()?; + let ast::Type::RefType(ref_type) = ty else { + return None; + }; + + let inner = ref_type.ty()?; + Some(factory.ty_ref(inner, true)) } fn get_trait_mut(apply_trait: &hir::Trait, famous: FamousDefs<'_, '_>) -> Option<&'static str> { @@ -158,14 +217,6 @@ fn process_method_name(name: ast::Name) -> Option<(ast::Name, &'static str)> { Some((name, new_name)) } -fn process_ret_type(ref_ty: &ast::RetType) -> Option { - let ty = ref_ty.ty()?; - let ast::Type::RefType(ref_type) = ty else { - return None; - }; - Some(make::ty_ref(ref_type.ty()?, true)) -} - #[cfg(test)] mod tests { use crate::{ From 8a71deb28275b847492208a325db589f2c222a49 Mon Sep 17 00:00:00 2001 From: Andre Bogus Date: Thu, 8 Jan 2026 23:26:12 +0100 Subject: [PATCH 0191/1843] add `manual_take` lint --- CHANGELOG.md | 1 + book/src/lint_configuration.md | 1 + clippy_config/src/conf.rs | 1 + clippy_lints/src/declared_lints.rs | 1 + clippy_lints/src/lib.rs | 2 + clippy_lints/src/manual_take.rs | 114 +++++++++++++++++++++++++++++ tests/ui/manual_take.fixed | 57 +++++++++++++++ tests/ui/manual_take.rs | 69 +++++++++++++++++ tests/ui/manual_take.stderr | 53 ++++++++++++++ tests/ui/manual_take_nocore.rs | 37 ++++++++++ tests/ui/manual_take_nocore.stderr | 0 tests/ui/manual_take_nostd.fixed | 10 +++ tests/ui/manual_take_nostd.rs | 22 ++++++ tests/ui/manual_take_nostd.stderr | 54 ++++++++++++++ 14 files changed, 422 insertions(+) create mode 100644 clippy_lints/src/manual_take.rs create mode 100644 tests/ui/manual_take.fixed create mode 100644 tests/ui/manual_take.rs create mode 100644 tests/ui/manual_take.stderr create mode 100644 tests/ui/manual_take_nocore.rs create mode 100644 tests/ui/manual_take_nocore.stderr create mode 100644 tests/ui/manual_take_nostd.fixed create mode 100644 tests/ui/manual_take_nostd.rs create mode 100644 tests/ui/manual_take_nostd.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cda8003b05c..b5172267fcaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6653,6 +6653,7 @@ Released 2018-09-13 [`manual_string_new`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_string_new [`manual_strip`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_strip [`manual_swap`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_swap +[`manual_take`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_take [`manual_try_fold`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_try_fold [`manual_unwrap_or`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_unwrap_or [`manual_unwrap_or_default`]: https://rust-lang.github.io/rust-clippy/master/index.html#manual_unwrap_or_default diff --git a/book/src/lint_configuration.md b/book/src/lint_configuration.md index a1c079898594..f81dd421f59b 100644 --- a/book/src/lint_configuration.md +++ b/book/src/lint_configuration.md @@ -905,6 +905,7 @@ The minimum rust version that the project supports. Defaults to the `rust-versio * [`manual_split_once`](https://rust-lang.github.io/rust-clippy/master/index.html#manual_split_once) * [`manual_str_repeat`](https://rust-lang.github.io/rust-clippy/master/index.html#manual_str_repeat) * [`manual_strip`](https://rust-lang.github.io/rust-clippy/master/index.html#manual_strip) +* [`manual_take`](https://rust-lang.github.io/rust-clippy/master/index.html#manual_take) * [`manual_try_fold`](https://rust-lang.github.io/rust-clippy/master/index.html#manual_try_fold) * [`map_clone`](https://rust-lang.github.io/rust-clippy/master/index.html#map_clone) * [`map_unwrap_or`](https://rust-lang.github.io/rust-clippy/master/index.html#map_unwrap_or) diff --git a/clippy_config/src/conf.rs b/clippy_config/src/conf.rs index e1d7c1d88eb9..849d9a613d80 100644 --- a/clippy_config/src/conf.rs +++ b/clippy_config/src/conf.rs @@ -780,6 +780,7 @@ define_Conf! { manual_split_once, manual_str_repeat, manual_strip, + manual_take, manual_try_fold, map_clone, map_unwrap_or, diff --git a/clippy_lints/src/declared_lints.rs b/clippy_lints/src/declared_lints.rs index ef9da84c4407..549fc64d50e2 100644 --- a/clippy_lints/src/declared_lints.rs +++ b/clippy_lints/src/declared_lints.rs @@ -316,6 +316,7 @@ pub static LINTS: &[&::declare_clippy_lint::LintInfo] = &[ crate::manual_slice_size_calculation::MANUAL_SLICE_SIZE_CALCULATION_INFO, crate::manual_string_new::MANUAL_STRING_NEW_INFO, crate::manual_strip::MANUAL_STRIP_INFO, + crate::manual_take::MANUAL_TAKE_INFO, crate::map_unit_fn::OPTION_MAP_UNIT_FN_INFO, crate::map_unit_fn::RESULT_MAP_UNIT_FN_INFO, crate::match_result_ok::MATCH_RESULT_OK_INFO, diff --git a/clippy_lints/src/lib.rs b/clippy_lints/src/lib.rs index ef2461f8b097..eccff4789a20 100644 --- a/clippy_lints/src/lib.rs +++ b/clippy_lints/src/lib.rs @@ -219,6 +219,7 @@ mod manual_rotate; mod manual_slice_size_calculation; mod manual_string_new; mod manual_strip; +mod manual_take; mod map_unit_fn; mod match_result_ok; mod matches; @@ -861,6 +862,7 @@ pub fn register_lint_passes(store: &mut rustc_lint::LintStore, conf: &'static Co Box::new(move |_| Box::new(manual_ilog2::ManualIlog2::new(conf))), Box::new(|_| Box::new(same_length_and_capacity::SameLengthAndCapacity)), Box::new(move |tcx| Box::new(duration_suboptimal_units::DurationSuboptimalUnits::new(tcx, conf))), + Box::new(move |_| Box::new(manual_take::ManualTake::new(conf))), // add late passes here, used by `cargo dev new_lint` ]; store.late_passes.extend(late_lints); diff --git a/clippy_lints/src/manual_take.rs b/clippy_lints/src/manual_take.rs new file mode 100644 index 000000000000..8e74d04c4556 --- /dev/null +++ b/clippy_lints/src/manual_take.rs @@ -0,0 +1,114 @@ +use clippy_config::Conf; +use clippy_utils::diagnostics::span_lint_and_then; +use clippy_utils::msrvs::{MEM_TAKE, Msrv}; +use clippy_utils::source::snippet_with_context; +use rustc_ast::LitKind; +use rustc_errors::Applicability; +use rustc_hir::{Block, Expr, ExprKind, StmtKind}; +use rustc_lint::{LateContext, LateLintPass, LintContext}; +use rustc_session::impl_lint_pass; + +declare_clippy_lint! { + /// ### What it does + /// Detects manual re-implementations of `std::mem::take`. + /// + /// ### Why is this bad? + /// Because the function call is shorter and easier to read. + /// + /// ### Known issues + /// Currently the lint only detects cases involving `bool`s. + /// + /// ### Example + /// ```no_run + /// let mut x = true; + /// let _ = if x { + /// x = false; + /// true + /// } else { + /// false + /// }; + /// ``` + /// Use instead: + /// ```no_run + /// let mut x = true; + /// let _ = std::mem::take(&mut x); + /// ``` + #[clippy::version = "1.94.0"] + pub MANUAL_TAKE, + complexity, + "manual `mem::take` implementation" +} +pub struct ManualTake { + msrv: Msrv, +} + +impl ManualTake { + pub fn new(conf: &'static Conf) -> Self { + Self { msrv: conf.msrv } + } +} + +impl_lint_pass!(ManualTake => [MANUAL_TAKE]); + +impl LateLintPass<'_> for ManualTake { + fn check_expr(&mut self, cx: &LateContext<'_>, expr: &Expr<'_>) { + if let ExprKind::If(cond, then, Some(otherwise)) = expr.kind + && let ExprKind::Path(_) = cond.kind + && let ExprKind::Block( + Block { + stmts: [stmt], + expr: Some(then_expr), + .. + }, + .., + ) = then.kind + && let ExprKind::Block( + Block { + stmts: [], + expr: Some(else_expr), + .. + }, + .., + ) = otherwise.kind + && let StmtKind::Semi(assignment) = stmt.kind + && let ExprKind::Assign(mut_c, possible_false, _) = assignment.kind + && let ExprKind::Path(_) = mut_c.kind + && !expr.span.in_external_macro(cx.sess().source_map()) + && let Some(std_or_core) = clippy_utils::std_or_core(cx) + && self.msrv.meets(cx, MEM_TAKE) + && clippy_utils::SpanlessEq::new(cx).eq_expr(cond, mut_c) + && Some(false) == as_const_bool(possible_false) + && let Some(then_bool) = as_const_bool(then_expr) + && let Some(else_bool) = as_const_bool(else_expr) + && then_bool != else_bool + { + span_lint_and_then( + cx, + MANUAL_TAKE, + expr.span, + "manual implementation of `mem::take`", + |diag| { + let mut app = Applicability::MachineApplicable; + let negate = if then_bool { "" } else { "!" }; + let taken = snippet_with_context(cx, cond.span, expr.span.ctxt(), "_", &mut app).0; + diag.span_suggestion_verbose( + expr.span, + "use", + format!("{negate}{std_or_core}::mem::take(&mut {taken})"), + app, + ); + }, + ); + } + } +} + +fn as_const_bool(e: &Expr<'_>) -> Option { + if let ExprKind::Lit(lit) = e.kind + && let LitKind::Bool(b) = lit.node + { + Some(b) + } else { + None + } +} diff --git a/tests/ui/manual_take.fixed b/tests/ui/manual_take.fixed new file mode 100644 index 000000000000..5891bd49c81c --- /dev/null +++ b/tests/ui/manual_take.fixed @@ -0,0 +1,57 @@ +#![warn(clippy::manual_take)] + +fn main() { + msrv_1_39(); + msrv_1_40(); + let mut x = true; + let mut y = false; + + let _lint_negated = !std::mem::take(&mut x); + + let _ = if x { + y = false; + true + } else { + false + }; + + let _ = if x { + x = true; + true + } else { + false + }; + + let _ = if x { + x = false; + y = true; + false + } else { + true + }; + + let _ = if x { + x = false; + false + } else { + y = true; + true + }; +} + +#[clippy::msrv = "1.39.0"] +fn msrv_1_39() -> bool { + let mut x = true; + if x { + x = false; + true + } else { + false + } +} + +#[clippy::msrv = "1.40.0"] +fn msrv_1_40() -> bool { + let mut x = true; + std::mem::take(&mut x) +} diff --git a/tests/ui/manual_take.rs b/tests/ui/manual_take.rs new file mode 100644 index 000000000000..89903fcb2416 --- /dev/null +++ b/tests/ui/manual_take.rs @@ -0,0 +1,69 @@ +#![warn(clippy::manual_take)] + +fn main() { + msrv_1_39(); + msrv_1_40(); + let mut x = true; + let mut y = false; + + let _lint_negated = if x { + //~^ manual_take + x = false; + false + } else { + true + }; + + let _ = if x { + y = false; + true + } else { + false + }; + + let _ = if x { + x = true; + true + } else { + false + }; + + let _ = if x { + x = false; + y = true; + false + } else { + true + }; + + let _ = if x { + x = false; + false + } else { + y = true; + true + }; +} + +#[clippy::msrv = "1.39.0"] +fn msrv_1_39() -> bool { + let mut x = true; + if x { + x = false; + true + } else { + false + } +} + +#[clippy::msrv = "1.40.0"] +fn msrv_1_40() -> bool { + let mut x = true; + if x { + //~^ manual_take + x = false; + true + } else { + false + } +} diff --git a/tests/ui/manual_take.stderr b/tests/ui/manual_take.stderr new file mode 100644 index 000000000000..69ba7778a275 --- /dev/null +++ b/tests/ui/manual_take.stderr @@ -0,0 +1,53 @@ +error: manual implementation of `mem::take` + --> tests/ui/manual_take.rs:9:25 + | +LL | let _lint_negated = if x { + | _________________________^ +LL | | +LL | | x = false; +LL | | false +LL | | } else { +LL | | true +LL | | }; + | |_____^ + | + = note: `-D clippy::manual-take` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::manual_take)]` +help: use + | +LL - let _lint_negated = if x { +LL - +LL - x = false; +LL - false +LL - } else { +LL - true +LL - }; +LL + let _lint_negated = !std::mem::take(&mut x); + | + +error: manual implementation of `mem::take` + --> tests/ui/manual_take.rs:62:5 + | +LL | / if x { +LL | | +LL | | x = false; +LL | | true +LL | | } else { +LL | | false +LL | | } + | |_____^ + | +help: use + | +LL - if x { +LL - +LL - x = false; +LL - true +LL - } else { +LL - false +LL - } +LL + std::mem::take(&mut x) + | + +error: aborting due to 2 previous errors + diff --git a/tests/ui/manual_take_nocore.rs b/tests/ui/manual_take_nocore.rs new file mode 100644 index 000000000000..ef4f23395028 --- /dev/null +++ b/tests/ui/manual_take_nocore.rs @@ -0,0 +1,37 @@ +//@ check-pass +#![feature(no_core, lang_items)] +#![no_core] +#![allow(clippy::missing_safety_doc)] +#![warn(clippy::manual_take)] + +#[link(name = "c")] +unsafe extern "C" {} + +#[lang = "pointee_sized"] +pub trait PointeeSized {} + +#[lang = "meta_sized"] +pub trait MetaSized: PointeeSized {} + +#[lang = "sized"] +pub trait Sized: MetaSized {} +#[lang = "copy"] +pub trait Copy {} +#[lang = "freeze"] +pub unsafe trait Freeze {} + +#[lang = "start"] +fn start(_main: fn() -> T, _argc: isize, _argv: *const *const u8, _sigpipe: u8) -> isize { + 0 +} + +fn main() { + let mut x = true; + // this should not lint because we don't have std nor core + let _manual_take = if x { + x = false; + true + } else { + false + }; +} diff --git a/tests/ui/manual_take_nocore.stderr b/tests/ui/manual_take_nocore.stderr new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/ui/manual_take_nostd.fixed b/tests/ui/manual_take_nostd.fixed new file mode 100644 index 000000000000..123c5a91998c --- /dev/null +++ b/tests/ui/manual_take_nostd.fixed @@ -0,0 +1,10 @@ +#![no_std] +#![warn(clippy::manual_take)] + +pub fn manual_mem_take_should_reference_core() { + let mut x = true; + + let _lint_negated = !core::mem::take(&mut x); + + let _lint = core::mem::take(&mut x); +} diff --git a/tests/ui/manual_take_nostd.rs b/tests/ui/manual_take_nostd.rs new file mode 100644 index 000000000000..0df352477b74 --- /dev/null +++ b/tests/ui/manual_take_nostd.rs @@ -0,0 +1,22 @@ +#![no_std] +#![warn(clippy::manual_take)] + +pub fn manual_mem_take_should_reference_core() { + let mut x = true; + + let _lint_negated = if x { + //~^ manual_take + x = false; + false + } else { + true + }; + + let _lint = if x { + //~^ manual_take + x = false; + true + } else { + false + }; +} diff --git a/tests/ui/manual_take_nostd.stderr b/tests/ui/manual_take_nostd.stderr new file mode 100644 index 000000000000..71a5066e4db3 --- /dev/null +++ b/tests/ui/manual_take_nostd.stderr @@ -0,0 +1,54 @@ +error: manual implementation of `mem::take` + --> tests/ui/manual_take_nostd.rs:7:25 + | +LL | let _lint_negated = if x { + | _________________________^ +LL | | +LL | | x = false; +LL | | false +LL | | } else { +LL | | true +LL | | }; + | |_____^ + | + = note: `-D clippy::manual-take` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::manual_take)]` +help: use + | +LL - let _lint_negated = if x { +LL - +LL - x = false; +LL - false +LL - } else { +LL - true +LL - }; +LL + let _lint_negated = !core::mem::take(&mut x); + | + +error: manual implementation of `mem::take` + --> tests/ui/manual_take_nostd.rs:15:17 + | +LL | let _lint = if x { + | _________________^ +LL | | +LL | | x = false; +LL | | true +LL | | } else { +LL | | false +LL | | }; + | |_____^ + | +help: use + | +LL - let _lint = if x { +LL - +LL - x = false; +LL - true +LL - } else { +LL - false +LL - }; +LL + let _lint = core::mem::take(&mut x); + | + +error: aborting due to 2 previous errors + From d0204a399a7f3a2d9c5ea4d2d2c897ce0536b5eb Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sun, 11 Jan 2026 21:10:17 +0100 Subject: [PATCH 0192/1843] remove `impl Neg` on s390x/powerpc vector types For other targets we don't have such instances. We specifically decided agains implementing arithmetic operators on these low-level vector types --- .../crates/core_arch/src/powerpc/altivec.rs | 51 +++++++++++++++--- .../crates/core_arch/src/powerpc/macros.rs | 14 ----- .../crates/core_arch/src/s390x/macros.rs | 14 ----- .../crates/core_arch/src/s390x/vector.rs | 54 ++++++++++++++++--- 4 files changed, 91 insertions(+), 42 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/powerpc/altivec.rs b/library/stdarch/crates/core_arch/src/powerpc/altivec.rs index 37b557c3b2af..c1881da08f86 100644 --- a/library/stdarch/crates/core_arch/src/powerpc/altivec.rs +++ b/library/stdarch/crates/core_arch/src/powerpc/altivec.rs @@ -364,15 +364,46 @@ unsafe extern "C" { fn vrfin(a: vector_float) -> vector_float; } -impl_neg! { i8x16 : 0 } -impl_neg! { i16x8 : 0 } -impl_neg! { i32x4 : 0 } -impl_neg! { f32x4 : 0f32 } - #[macro_use] mod sealed { use super::*; + #[unstable(feature = "stdarch_powerpc", issue = "111145")] + pub trait VectorNeg { + unsafe fn vec_neg(self) -> Self; + } + + macro_rules! impl_neg { + ($($v:ty)*) => { + $( + #[unstable(feature = "stdarch_powerpc", issue = "111145")] + impl VectorNeg for $v { + #[inline] + #[target_feature(enable = "altivec")] + unsafe fn vec_neg(self) -> Self { + simd_neg(self) + } + } + )* + } + } + + impl_neg! { + vector_signed_char + vector_unsigned_char + vector_bool_char + + vector_signed_short + vector_unsigned_short + vector_bool_short + + vector_signed_int + vector_unsigned_int + vector_bool_int + + vector_float + } + #[unstable(feature = "stdarch_powerpc", issue = "111145")] pub trait VectorInsert { type Scalar; @@ -1378,7 +1409,7 @@ mod sealed { #[inline] #[target_feature(enable = "altivec")] unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) { - v.vec_max(-v) + v.vec_max(simd_neg(v)) } impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) } @@ -4030,6 +4061,14 @@ pub unsafe fn vec_mfvscr() -> vector_unsigned_short { mfvscr() } +/// Vector Negate +#[inline] +#[target_feature(enable = "altivec")] +#[unstable(feature = "stdarch_powerpc", issue = "111145")] +pub unsafe fn vec_neg(a: T) -> T { + a.vec_neg() +} + /// Vector add. #[inline] #[target_feature(enable = "altivec")] diff --git a/library/stdarch/crates/core_arch/src/powerpc/macros.rs b/library/stdarch/crates/core_arch/src/powerpc/macros.rs index 6097ca31adc9..f697d4d25748 100644 --- a/library/stdarch/crates/core_arch/src/powerpc/macros.rs +++ b/library/stdarch/crates/core_arch/src/powerpc/macros.rs @@ -274,20 +274,6 @@ macro_rules! t_b { }; } -macro_rules! impl_neg { - ($s: ident : $zero: expr) => { - #[unstable(feature = "stdarch_powerpc", issue = "111145")] - impl crate::ops::Neg for s_t_l!($s) { - type Output = s_t_l!($s); - #[inline] - fn neg(self) -> Self::Output { - unsafe { simd_neg(self) } - } - } - }; -} - -pub(crate) use impl_neg; pub(crate) use impl_vec_trait; pub(crate) use s_t_l; pub(crate) use t_b; diff --git a/library/stdarch/crates/core_arch/src/s390x/macros.rs b/library/stdarch/crates/core_arch/src/s390x/macros.rs index b72560396c43..c47f242948df 100644 --- a/library/stdarch/crates/core_arch/src/s390x/macros.rs +++ b/library/stdarch/crates/core_arch/src/s390x/macros.rs @@ -431,20 +431,6 @@ macro_rules! t_b { }; } -macro_rules! impl_neg { - ($s: ident : $zero: expr) => { - #[unstable(feature = "stdarch_s390x", issue = "135681")] - impl crate::ops::Neg for s_t_l!($s) { - type Output = s_t_l!($s); - #[inline] - fn neg(self) -> Self::Output { - unsafe { simd_neg(self) } - } - } - }; -} - -pub(crate) use impl_neg; pub(crate) use impl_vec_trait; pub(crate) use l_t_t; pub(crate) use s_t_l; diff --git a/library/stdarch/crates/core_arch/src/s390x/vector.rs b/library/stdarch/crates/core_arch/src/s390x/vector.rs index d63f37eeb9a8..97bfa32f17ad 100644 --- a/library/stdarch/crates/core_arch/src/s390x/vector.rs +++ b/library/stdarch/crates/core_arch/src/s390x/vector.rs @@ -283,13 +283,6 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple; } -impl_neg! { i8x16 : 0 } -impl_neg! { i16x8 : 0 } -impl_neg! { i32x4 : 0 } -impl_neg! { i64x2 : 0 } -impl_neg! { f32x4 : 0f32 } -impl_neg! { f64x2 : 0f64 } - #[repr(simd)] struct ShuffleMask([u32; N]); @@ -437,6 +430,43 @@ enum FindImm { mod sealed { use super::*; + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorNeg { + unsafe fn vec_neg(self) -> Self; + } + + macro_rules! impl_neg { + ($($v:ty)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorNeg for $v { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_neg(self) -> Self { + simd_neg(self) + } + } + )* + } + } + + impl_neg! { + vector_signed_char + vector_unsigned_char + + vector_signed_short + vector_unsigned_short + + vector_signed_int + vector_unsigned_int + + vector_signed_long_long + vector_unsigned_long_long + + vector_float + vector_double + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorAdd { type Result; @@ -759,7 +789,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) { - v.vec_max(-v) + v.vec_max(simd_neg(v)) } impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) } @@ -4053,6 +4083,14 @@ unsafe fn __lcbb(ptr: *const u8) -> u32 { lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) }) } +/// Vector Negate +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_neg(a: T) -> T { + a.vec_neg() +} + /// Vector Add #[inline] #[target_feature(enable = "vector")] From d419bddb96f8177a3d8a7cb3c4df15902a24028a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 9 Jan 2026 10:45:26 +1100 Subject: [PATCH 0193/1843] clippy: remove `ty_has_erased_regions`. It duplicates `has_erased_regions` from the compiler. --- .../matches/significant_drop_in_scrutinee.rs | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/clippy_lints/src/matches/significant_drop_in_scrutinee.rs b/clippy_lints/src/matches/significant_drop_in_scrutinee.rs index bac35e7f8c70..14d265bfdad8 100644 --- a/clippy_lints/src/matches/significant_drop_in_scrutinee.rs +++ b/clippy_lints/src/matches/significant_drop_in_scrutinee.rs @@ -12,7 +12,7 @@ use rustc_errors::{Applicability, Diag}; use rustc_hir::intravisit::{Visitor, walk_expr}; use rustc_hir::{Arm, Expr, ExprKind, MatchSource}; use rustc_lint::{LateContext, LintContext}; -use rustc_middle::ty::{GenericArgKind, Region, RegionKind, Ty, TyCtxt, TypeVisitable, TypeVisitor}; +use rustc_middle::ty::{GenericArgKind, RegionKind, Ty, TypeVisitableExt}; use rustc_span::Span; use super::SIGNIFICANT_DROP_IN_SCRUTINEE; @@ -303,13 +303,13 @@ impl<'a, 'tcx> SigDropHelper<'a, 'tcx> { if self.sig_drop_holder != SigDropHolder::None { let parent_ty = self.cx.typeck_results().expr_ty(parent_expr); - if !ty_has_erased_regions(parent_ty) && !parent_expr.is_syntactic_place_expr() { + if !parent_ty.has_erased_regions() && !parent_expr.is_syntactic_place_expr() { self.replace_current_sig_drop(parent_expr.span, parent_ty.is_unit(), 0); self.sig_drop_holder = SigDropHolder::Moved; } let (peel_ref_ty, peel_ref_times) = ty_peel_refs(parent_ty); - if !ty_has_erased_regions(peel_ref_ty) && is_copy(self.cx, peel_ref_ty) { + if !peel_ref_ty.has_erased_regions() && is_copy(self.cx, peel_ref_ty) { self.replace_current_sig_drop(parent_expr.span, peel_ref_ty.is_unit(), peel_ref_times); self.sig_drop_holder = SigDropHolder::Moved; } @@ -399,24 +399,6 @@ fn ty_peel_refs(mut ty: Ty<'_>) -> (Ty<'_>, usize) { (ty, n) } -fn ty_has_erased_regions(ty: Ty<'_>) -> bool { - struct V; - - impl<'tcx> TypeVisitor> for V { - type Result = ControlFlow<()>; - - fn visit_region(&mut self, region: Region<'tcx>) -> Self::Result { - if region.is_erased() { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - } - } - - ty.visit_with(&mut V).is_break() -} - impl<'tcx> Visitor<'tcx> for SigDropHelper<'_, 'tcx> { fn visit_expr(&mut self, ex: &'tcx Expr<'_>) { // We've emitted a lint on some neighborhood expression. That lint will suggest to move out the From 2d2d3b0a6bb9bc88f795002e1da70e4181254d34 Mon Sep 17 00:00:00 2001 From: Nicholas Bishop Date: Sun, 11 Jan 2026 23:21:38 -0500 Subject: [PATCH 0194/1843] remote-test-server: Fix header in batch mode In https://github.com/rust-lang/rust/pull/119999, the length field of the header was changed from 4 bytes to 8. One of the headers in `batch_copy` was missed though, so it sends the wrong size. Fix by switching to `create_header` in that spot too. --- src/tools/remote-test-server/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/remote-test-server/src/main.rs b/src/tools/remote-test-server/src/main.rs index bfe8f54937f6..48b0d83e8d82 100644 --- a/src/tools/remote-test-server/src/main.rs +++ b/src/tools/remote-test-server/src/main.rs @@ -387,7 +387,7 @@ fn batch_copy(buf: &[u8], which: u8, dst: &Mutex) { if n > 0 { t!(dst.write_all(buf)); // Marking buf finished - t!(dst.write_all(&[which, 0, 0, 0, 0,])); + t!(dst.write_all(&create_header(which, 0))); } } From 594ca4b1bc47588a851265d51a5db05ed2f080b2 Mon Sep 17 00:00:00 2001 From: The rustc-josh-sync Cronjob Bot Date: Mon, 12 Jan 2026 04:26:31 +0000 Subject: [PATCH 0195/1843] Prepare for merging from rust-lang/rust This updates the rust-version file to 44a5b55557c26353f388400d7da95527256fe260. --- src/tools/rust-analyzer/rust-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rust-analyzer/rust-version b/src/tools/rust-analyzer/rust-version index 4b08b0884ca8..b53a66c66751 100644 --- a/src/tools/rust-analyzer/rust-version +++ b/src/tools/rust-analyzer/rust-version @@ -1 +1 @@ -548e586795f6b6fe089d8329aa5edbf0f5202646 +44a5b55557c26353f388400d7da95527256fe260 From 80acef153ffa6ecc4c9458b8a1169b10d7d750c7 Mon Sep 17 00:00:00 2001 From: The rustc-josh-sync Cronjob Bot Date: Mon, 12 Jan 2026 04:26:43 +0000 Subject: [PATCH 0196/1843] Format code --- src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs | 4 +++- src/tools/rust-analyzer/crates/span/src/hygiene.rs | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs b/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs index d10e122a5deb..172641227599 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/declarative.rs @@ -101,7 +101,9 @@ impl DeclarativeMacroExpander { match &*value { "transparent" => ControlFlow::Break(Transparency::Transparent), // "semitransparent" is for old rustc versions. - "semiopaque" | "semitransparent" => ControlFlow::Break(Transparency::SemiOpaque), + "semiopaque" | "semitransparent" => { + ControlFlow::Break(Transparency::SemiOpaque) + } "opaque" => ControlFlow::Break(Transparency::Opaque), _ => ControlFlow::Continue(()), } diff --git a/src/tools/rust-analyzer/crates/span/src/hygiene.rs b/src/tools/rust-analyzer/crates/span/src/hygiene.rs index 9904c562a740..fe05ef946518 100644 --- a/src/tools/rust-analyzer/crates/span/src/hygiene.rs +++ b/src/tools/rust-analyzer/crates/span/src/hygiene.rs @@ -241,9 +241,7 @@ const _: () = { edition: zalsa_::interned::Lookup::into_owned(data.2), parent: zalsa_::interned::Lookup::into_owned(data.3), opaque: opaque(zalsa_::FromId::from_id(id)), - opaque_and_semiopaque: opaque_and_semiopaque( - zalsa_::FromId::from_id(id), - ), + opaque_and_semiopaque: opaque_and_semiopaque(zalsa_::FromId::from_id(id)), }, ) } From 199aa6813eb08b32abca02745f1fb331813bb044 Mon Sep 17 00:00:00 2001 From: The Miri Cronjob Bot Date: Mon, 12 Jan 2026 05:04:54 +0000 Subject: [PATCH 0197/1843] Prepare for merging from rust-lang/rust This updates the rust-version file to 44a5b55557c26353f388400d7da95527256fe260. --- src/tools/miri/rust-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/miri/rust-version b/src/tools/miri/rust-version index c44422d758c5..b53a66c66751 100644 --- a/src/tools/miri/rust-version +++ b/src/tools/miri/rust-version @@ -1 +1 @@ -f57b9e6f565a1847e83a63f3e90faa3870536c1f +44a5b55557c26353f388400d7da95527256fe260 From 294a0afd655f22f98201c5e34e9f3b57e42a537c Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Mon, 12 Jan 2026 08:51:42 +0200 Subject: [PATCH 0198/1843] Disable `unused_variables` and `unused_mut` warnings They suffer from an unacceptable amount of false positives after #21209. Another option to disable them is to include them in `rust-analyzer.diagnostics.disable` by default, but that will mean users could override that. --- .../src/handlers/mutability_errors.rs | 24 ++++++----- .../src/handlers/unused_variables.rs | 42 ++++++++++--------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/mutability_errors.rs b/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/mutability_errors.rs index 2887a32825db..e3cfbdfb515f 100644 --- a/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/mutability_errors.rs +++ b/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/mutability_errors.rs @@ -1,3 +1,5 @@ +#![expect(unused, reason = "diagnostics is temporarily disabled due to too many false positives")] + use hir::db::ExpandDatabase; use ide_db::source_change::SourceChange; use ide_db::text_edit::TextEdit; @@ -88,16 +90,17 @@ pub(crate) fn unused_mut(ctx: &DiagnosticsContext<'_>, d: &hir::UnusedMut) -> Op )]) })(); let ast = d.local.primary_source(ctx.sema.db).syntax_ptr(); - Some( - Diagnostic::new_with_syntax_node_ptr( - ctx, - DiagnosticCode::RustcLint("unused_mut"), - "variable does not need to be mutable", - ast, - ) - // Not supporting `#[allow(unused_mut)]` in proc macros leads to false positive, hence not stable. - .with_fixes(fixes), - ) + // Some( + // Diagnostic::new_with_syntax_node_ptr( + // ctx, + // DiagnosticCode::RustcLint("unused_mut"), + // "variable does not need to be mutable", + // ast, + // ) + // // Not supporting `#[allow(unused_mut)]` in proc macros leads to false positive, hence not stable. + // .with_fixes(fixes), + // ) + None } pub(super) fn token(parent: &SyntaxNode, kind: SyntaxKind) -> Option { @@ -105,6 +108,7 @@ pub(super) fn token(parent: &SyntaxNode, kind: SyntaxKind) -> Option Date: Mon, 12 Jan 2026 08:36:32 +0100 Subject: [PATCH 0199/1843] Create a new `SymbolKind::CrateRoot` --- .../src/completions/extern_crate.rs | 6 +++--- .../crates/ide-completion/src/item.rs | 1 + .../rust-analyzer/crates/ide-db/src/lib.rs | 8 ++++--- .../crates/ide/src/navigation_target.rs | 13 ++++-------- .../crates/ide/src/references.rs | 2 +- .../rust-analyzer/crates/ide/src/runnables.rs | 8 +++---- .../ide/src/syntax_highlighting/highlight.rs | 21 +++++++++---------- .../ide/src/syntax_highlighting/inject.rs | 12 +++++------ .../ide/src/syntax_highlighting/tags.rs | 1 + .../test_data/highlight_asm.html | 8 +++---- .../test_data/highlight_attributes.html | 2 +- .../test_data/highlight_crate_root.html | 16 +++++++------- .../test_data/highlight_default_library.html | 2 +- .../test_data/highlight_deprecated.html | 4 ++-- .../test_data/highlight_doctest.html | 6 +++--- .../test_data/highlight_extern_crate.html | 14 ++++++------- .../test_data/highlight_general.html | 12 +++++------ .../test_data/highlight_injection_2.html | 2 +- .../test_data/highlight_issue_18089.html | 2 +- .../test_data/highlight_keywords_2015.html | 8 +++---- .../test_data/highlight_keywords_2018.html | 8 +++---- .../test_data/highlight_keywords_2021.html | 8 +++---- .../test_data/highlight_keywords_2024.html | 8 +++---- .../test_data/highlight_keywords_macros.html | 4 ++-- .../test_data/highlight_macros.html | 2 +- .../test_data/highlight_strings.html | 2 +- .../test_data/highlight_unsafe.html | 4 ++-- .../crates/rust-analyzer/src/lsp/to_proto.rs | 8 ++++++- 28 files changed, 98 insertions(+), 94 deletions(-) diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/completions/extern_crate.rs b/src/tools/rust-analyzer/crates/ide-completion/src/completions/extern_crate.rs index 71a3e4eb4ed6..91202e8b32fc 100644 --- a/src/tools/rust-analyzer/crates/ide-completion/src/completions/extern_crate.rs +++ b/src/tools/rust-analyzer/crates/ide-completion/src/completions/extern_crate.rs @@ -17,7 +17,7 @@ pub(crate) fn complete_extern_crate(acc: &mut Completions, ctx: &CompletionConte } let mut item = CompletionItem::new( - CompletionItemKind::SymbolKind(SymbolKind::Module), + CompletionItemKind::SymbolKind(SymbolKind::CrateRoot), ctx.source_range(), name.display_no_db(ctx.edition).to_smolstr(), ctx.edition, @@ -48,7 +48,7 @@ mod other_mod {} let completion_list = completion_list_no_kw(case); - assert_eq!("md other_crate_a\n".to_owned(), completion_list); + assert_eq!("cr other_crate_a\n".to_owned(), completion_list); } #[test] @@ -68,6 +68,6 @@ mod other_mod {} let completion_list = completion_list_no_kw(case); - assert_eq!("md other_crate_a\n".to_owned(), completion_list); + assert_eq!("cr other_crate_a\n".to_owned(), completion_list); } } diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/item.rs b/src/tools/rust-analyzer/crates/ide-completion/src/item.rs index 71d32da74710..1a9139d8553b 100644 --- a/src/tools/rust-analyzer/crates/ide-completion/src/item.rs +++ b/src/tools/rust-analyzer/crates/ide-completion/src/item.rs @@ -381,6 +381,7 @@ impl CompletionItemKind { SymbolKind::BuiltinAttr => "ba", SymbolKind::Const => "ct", SymbolKind::ConstParam => "cp", + SymbolKind::CrateRoot => "cr", SymbolKind::Derive => "de", SymbolKind::DeriveHelper => "dh", SymbolKind::Enum => "en", diff --git a/src/tools/rust-analyzer/crates/ide-db/src/lib.rs b/src/tools/rust-analyzer/crates/ide-db/src/lib.rs index 413b58bf7980..023b32b36195 100644 --- a/src/tools/rust-analyzer/crates/ide-db/src/lib.rs +++ b/src/tools/rust-analyzer/crates/ide-db/src/lib.rs @@ -65,7 +65,7 @@ use base_db::{ }; use hir::{ FilePositionWrapper, FileRangeWrapper, - db::{DefDatabase, ExpandDatabase}, + db::{DefDatabase, ExpandDatabase, HirDatabase}, }; use triomphe::Arc; @@ -269,6 +269,7 @@ pub enum SymbolKind { BuiltinAttr, Const, ConstParam, + CrateRoot, Derive, DeriveHelper, Enum, @@ -307,14 +308,15 @@ impl From for SymbolKind { } } -impl From for SymbolKind { - fn from(it: hir::ModuleDef) -> Self { +impl SymbolKind { + pub fn from_module_def(db: &dyn HirDatabase, it: hir::ModuleDef) -> Self { match it { hir::ModuleDef::Const(..) => SymbolKind::Const, hir::ModuleDef::Variant(..) => SymbolKind::Variant, hir::ModuleDef::Function(..) => SymbolKind::Function, hir::ModuleDef::Macro(mac) if mac.is_proc_macro() => SymbolKind::ProcMacro, hir::ModuleDef::Macro(..) => SymbolKind::Macro, + hir::ModuleDef::Module(m) if m.is_crate_root(db) => SymbolKind::CrateRoot, hir::ModuleDef::Module(..) => SymbolKind::Module, hir::ModuleDef::Static(..) => SymbolKind::Static, hir::ModuleDef::Adt(hir::Adt::Struct(..)) => SymbolKind::Struct, diff --git a/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs b/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs index 047df309eca6..185df92e2d39 100644 --- a/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs +++ b/src/tools/rust-analyzer/crates/ide/src/navigation_target.rs @@ -264,7 +264,7 @@ impl<'db> TryToNav for FileSymbol<'db> { .flatten() .map_or_else(|| self.name.clone(), |it| it.symbol().clone()), alias: self.is_alias.then(|| self.name.clone()), - kind: Some(self.def.into()), + kind: Some(SymbolKind::from_module_def(db, self.def)), full_range, focus_range, container_name: self.container_name.clone(), @@ -480,16 +480,11 @@ impl ToNav for hir::Module { ModuleSource::Module(node) => (node.syntax(), node.name()), ModuleSource::BlockExpr(node) => (node.syntax(), None), }; + let kind = if self.is_crate_root(db) { SymbolKind::CrateRoot } else { SymbolKind::Module }; orig_range_with_focus(db, file_id, syntax, focus).map( |(FileRange { file_id, range: full_range }, focus_range)| { - NavigationTarget::from_syntax( - file_id, - name.clone(), - focus_range, - full_range, - SymbolKind::Module, - ) + NavigationTarget::from_syntax(file_id, name.clone(), focus_range, full_range, kind) }, ) } @@ -549,7 +544,7 @@ impl TryToNav for hir::ExternCrateDecl { self.alias_or_name(db).unwrap_or_else(|| self.name(db)).symbol().clone(), focus_range, full_range, - SymbolKind::Module, + SymbolKind::CrateRoot, ); res.docs = self.docs(db).map(Documentation::into_owned); diff --git a/src/tools/rust-analyzer/crates/ide/src/references.rs b/src/tools/rust-analyzer/crates/ide/src/references.rs index 4918fe4ff9a4..5443021988d4 100644 --- a/src/tools/rust-analyzer/crates/ide/src/references.rs +++ b/src/tools/rust-analyzer/crates/ide/src/references.rs @@ -1079,7 +1079,7 @@ use self$0; use self$0; "#, expect![[r#" - _ Module FileId(0) 0..10 + _ CrateRoot FileId(0) 0..10 FileId(0) 4..8 import "#]], diff --git a/src/tools/rust-analyzer/crates/ide/src/runnables.rs b/src/tools/rust-analyzer/crates/ide/src/runnables.rs index 6cec91250351..42efa7142b50 100644 --- a/src/tools/rust-analyzer/crates/ide/src/runnables.rs +++ b/src/tools/rust-analyzer/crates/ide/src/runnables.rs @@ -815,7 +815,7 @@ mod not_a_root { "#, expect![[r#" [ - "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..331, name: \"_\", kind: Module })", + "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..331, name: \"_\", kind: CrateRoot })", "(Bin, NavigationTarget { file_id: FileId(0), full_range: 1..13, focus_range: 4..8, name: \"main\", kind: Function })", "(Bin, NavigationTarget { file_id: FileId(0), full_range: 15..76, focus_range: 42..71, name: \"__cortex_m_rt_main_trampoline\", kind: Function })", "(Bin, NavigationTarget { file_id: FileId(0), full_range: 78..154, focus_range: 113..149, name: \"__cortex_m_rt_main_trampoline_unsafe\", kind: Function })", @@ -1136,7 +1136,7 @@ fn test_foo1() {} "#, expect![[r#" [ - "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..51, name: \"_\", kind: Module })", + "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..51, name: \"_\", kind: CrateRoot })", "(Test, NavigationTarget { file_id: FileId(0), full_range: 1..50, focus_range: 36..45, name: \"test_foo1\", kind: Function }, Atom(KeyValue { key: \"feature\", value: \"foo\" }))", ] "#]], @@ -1155,7 +1155,7 @@ fn test_foo1() {} "#, expect![[r#" [ - "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..73, name: \"_\", kind: Module })", + "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..73, name: \"_\", kind: CrateRoot })", "(Test, NavigationTarget { file_id: FileId(0), full_range: 1..72, focus_range: 58..67, name: \"test_foo1\", kind: Function }, All([Atom(KeyValue { key: \"feature\", value: \"foo\" }), Atom(KeyValue { key: \"feature\", value: \"bar\" })]))", ] "#]], @@ -1234,7 +1234,7 @@ generate_main!(); "#, expect![[r#" [ - "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..345, name: \"_\", kind: Module })", + "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 0..345, name: \"_\", kind: CrateRoot })", "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 282..312, focus_range: 286..291, name: \"tests\", kind: Module, description: \"mod tests\" })", "(Test, NavigationTarget { file_id: FileId(0), full_range: 298..307, name: \"foo_test\", kind: Function })", "(TestMod, NavigationTarget { file_id: FileId(0), full_range: 313..323, name: \"tests2\", kind: Module, description: \"mod tests2\" }, true)", diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs index 044fd3f5ac9a..3795d3d4146d 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs @@ -399,7 +399,7 @@ fn highlight_name_ref( highlight_def(sema, krate, field_ref.into(), edition, true) } NameRefClass::ExternCrateShorthand { decl, krate: resolved_krate } => { - let mut h = HlTag::Symbol(SymbolKind::Module).into(); + let mut h = HlTag::Symbol(SymbolKind::CrateRoot).into(); if krate.as_ref().is_some_and(|krate| resolved_krate != *krate) { h |= HlMod::Library; @@ -417,7 +417,6 @@ fn highlight_name_ref( if is_deprecated { h |= HlMod::Deprecated; } - h |= HlMod::CrateRoot; h } }; @@ -496,15 +495,15 @@ pub(super) fn highlight_def( } Definition::TupleField(_) => (Highlight::new(HlTag::Symbol(SymbolKind::Field)), None), Definition::Crate(krate) => ( - Highlight::new(HlTag::Symbol(SymbolKind::Module)) | HlMod::CrateRoot, + Highlight::new(HlTag::Symbol(SymbolKind::CrateRoot)).into(), Some(krate.attrs(sema.db)), ), Definition::Module(module) => { - let mut h = Highlight::new(HlTag::Symbol(SymbolKind::Module)); - if module.is_crate_root(db) { - h |= HlMod::CrateRoot; - } - + let h = Highlight::new(HlTag::Symbol(if module.is_crate_root(db) { + SymbolKind::CrateRoot + } else { + SymbolKind::Module + })); (h, Some(module.attrs(sema.db))) } Definition::Function(func) => { @@ -662,8 +661,7 @@ pub(super) fn highlight_def( (h, None) } Definition::ExternCrateDecl(extern_crate) => { - let mut highlight = - Highlight::new(HlTag::Symbol(SymbolKind::Module)) | HlMod::CrateRoot; + let mut highlight = Highlight::new(HlTag::Symbol(SymbolKind::CrateRoot)).into(); if extern_crate.alias(db).is_none() { highlight |= HlMod::Library; } @@ -805,6 +803,7 @@ fn highlight_name_by_syntax(name: ast::Name) -> Highlight { TYPE_PARAM => SymbolKind::TypeParam, RECORD_FIELD => SymbolKind::Field, MODULE => SymbolKind::Module, + EXTERN_CRATE => SymbolKind::CrateRoot, FN => SymbolKind::Function, CONST => SymbolKind::Const, STATIC => SymbolKind::Static, @@ -835,7 +834,7 @@ fn highlight_name_ref_by_syntax( }; match parent.kind() { - EXTERN_CRATE => HlTag::Symbol(SymbolKind::Module) | HlMod::CrateRoot, + EXTERN_CRATE => HlTag::Symbol(SymbolKind::CrateRoot).into(), METHOD_CALL_EXPR => ast::MethodCallExpr::cast(parent) .and_then(|it| highlight_method_call(sema, krate, &it, is_unsafe_node)) .unwrap_or_else(|| SymbolKind::Method.into()), diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/inject.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/inject.rs index 26d2bb5e0288..291333f09cf8 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/inject.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/inject.rs @@ -1,6 +1,6 @@ //! "Recursive" Syntax highlighting for code in doctests and fixtures. -use hir::{EditionedFileId, HirFileId, InFile, Semantics}; +use hir::{EditionedFileId, HirFileId, InFile, Semantics, db::HirDatabase}; use ide_db::{ SymbolKind, defs::Definition, documentation::Documentation, range_mapper::RangeMapper, rust_doc::is_rust_fence, @@ -109,7 +109,7 @@ pub(super) fn doc_comment( .for_each(|(range, def)| { hl.add(HlRange { range, - highlight: module_def_to_hl_tag(def) + highlight: module_def_to_hl_tag(sema.db, def) | HlMod::Documentation | HlMod::Injected | HlMod::IntraDocLink, @@ -200,11 +200,11 @@ pub(super) fn doc_comment( } } -fn module_def_to_hl_tag(def: Definition) -> HlTag { +fn module_def_to_hl_tag(db: &dyn HirDatabase, def: Definition) -> HlTag { let symbol = match def { - Definition::Module(_) | Definition::Crate(_) | Definition::ExternCrateDecl(_) => { - SymbolKind::Module - } + Definition::Crate(_) | Definition::ExternCrateDecl(_) => SymbolKind::CrateRoot, + Definition::Module(m) if m.is_crate_root(db) => SymbolKind::CrateRoot, + Definition::Module(_) => SymbolKind::Module, Definition::Function(_) => SymbolKind::Function, Definition::Adt(hir::Adt::Struct(_)) => SymbolKind::Struct, Definition::Adt(hir::Adt::Enum(_)) => SymbolKind::Enum, diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tags.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tags.rs index ca3c3e3aaace..0c64d3de1012 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tags.rs +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tags.rs @@ -144,6 +144,7 @@ impl HlTag { SymbolKind::BuiltinAttr => "builtin_attr", SymbolKind::Const => "constant", SymbolKind::ConstParam => "const_param", + SymbolKind::CrateRoot => "crate_root", SymbolKind::Derive => "derive", SymbolKind::DeriveHelper => "derive_helper", SymbolKind::Enum => "enum", diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_asm.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_asm.html index 100fdd2155a4..1228849c5bfd 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_asm.html +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_asm.html @@ -45,7 +45,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd unsafe { let foo = 1; let mut o = 0; - core::arch::asm!( + core::arch::asm!( "%input = OpLoad _ {0}", concat!("%result = ", "bar", " _ %input"), "OpStore {1} %result", @@ -54,7 +54,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd ); let thread_id: usize; - core::arch::asm!(" + core::arch::asm!(" mov {0}, gs:[0x30] mov {0}, [{0}+0x48] ", out(reg) thread_id, options(pure, readonly, nostack)); @@ -64,7 +64,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd static VirtualFree: usize; const OffPtr: usize; const OffFn: usize; - core::arch::asm!(" + core::arch::asm!(" push {free_type} push {free_size} push {base} @@ -97,7 +97,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd // Ensure thumb mode is set. let rv = (rv as u32) | 1; let msp = msp as u32; - core::arch::asm!( + core::arch::asm!( "mrs {tmp}, CONTROL", "bics {tmp}, {spsel}", "msr CONTROL, {tmp}", diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html index b151ff42fc39..fa7f7b1cbafb 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html @@ -43,7 +43,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd 

#[allow(dead_code)]
 #[rustfmt::skip]
-#[proc_macros::identity]
+#[proc_macros::identity]
 #[derive(Default)]
 /// This is a doc comment
 // This is a normal comment
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
index a6e6b16bead5..0b32cedca5d8 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
@@ -41,25 +41,25 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
extern crate foo;
-use core::iter;
+
extern crate foo;
+use core::iter;
 
 pub const NINETY_TWO: u8 = 92;
 
-use foo as foooo;
+use foo as foooo;
 
-pub(crate) fn main() {
+pub(crate) fn main() {
     let baz = iter::repeat(92);
 }
 
 mod bar {
-    pub(in super) const FORTY_TWO: u8 = 42;
+    pub(in super) const FORTY_TWO: u8 = 42;
 
     mod baz {
-        use super::super::NINETY_TWO;
-        use crate::foooo::Point;
+        use super::super::NINETY_TWO;
+        use crate::foooo::Point;
 
-        pub(in super::super) const TWENTY_NINE: u8 = 29;
+        pub(in super::super) const TWENTY_NINE: u8 = 29;
     }
 }
 

\ No newline at end of file
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
index 2f4a2004f1de..29f78959a54f 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
@@ -41,7 +41,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
use core::iter;
+
use core::iter;
 
 fn main() {
     let foo = Some(92);
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_deprecated.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_deprecated.html
index 41d3dff8ed9e..5287affbfc5c 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_deprecated.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_deprecated.html
@@ -42,8 +42,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
#![deprecated]
-use crate as _;
-extern crate bar;
+use crate as _;
+extern crate bar;
 #[deprecated]
 macro_rules! macro_ {
     () => {};
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
index b5c3df6ee447..ce9ec7431a97 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
@@ -48,9 +48,9 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 
 //! Syntactic name ref highlighting testing
 //! ```rust
-//! extern crate self;
-//! extern crate other as otter;
-//! extern crate core;
+//! extern crate self;
+//! extern crate other as otter;
+//! extern crate core;
 //! trait T { type Assoc; }
 //! fn f<Arg>() -> use<Arg> where (): T<Assoc = ()> {}
 //! ```
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
index 3a4518236883..8f7cbddd7ffb 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
@@ -41,12 +41,12 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
extern crate self as this;
-extern crate std;
-extern crate alloc as abc;
-extern crate unresolved as definitely_unresolved;
+
extern crate self as this;
+extern crate std;
+extern crate alloc as abc;
+extern crate unresolved as definitely_unresolved;
 extern crate unresolved as _;
-extern crate test as opt_in_crate;
-extern crate test as _;
-extern crate proc_macro;
+extern crate test as opt_in_crate;
+extern crate test as _;
+extern crate proc_macro;
 

\ No newline at end of file
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
index fd652f444ffd..c6dbc435c0e8 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
@@ -72,7 +72,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     }
 }
 
-use self::FooCopy::{self as BarCopy};
+use self::FooCopy::{self as BarCopy};
 
 #[derive(Copy)]
 struct FooCopy {
@@ -110,7 +110,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     FOO
 }
 
-use core::ops::Fn;
+use core::ops::Fn;
 fn baz<F: Fn() -> ()>(f: F) {
     f()
 }
@@ -184,15 +184,15 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 }
 
 fn use_foo_items() {
-    let bob = foo::Person {
+    let bob = foo::Person {
         name: "Bob",
-        age: foo::consts::NUMBER,
+        age: foo::consts::NUMBER,
     };
 
-    let control_flow = foo::identity(foo::ControlFlow::Continue);
+    let control_flow = foo::identity(foo::ControlFlow::Continue);
 
     if control_flow.should_die() {
-        foo::die!();
+        foo::die!();
     }
 }
 
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html
index 5a5d9bd1f909..391a46f706c8 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html
@@ -47,7 +47,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     fixture(r#"
 @@- /main.rs crate:main deps:other_crate
 fn test() {
-    let x = other_crate::foo::S::thing();
+    let x = other_crate::foo::S::thing();
     x;
 } //^ i128
 
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html
index b28818e679ff..fccf34083d7f 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html
@@ -45,5 +45,5 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     template!(template);
 }
 
-#[proc_macros::issue_18089]
+#[proc_macros::issue_18089]
 fn template() {}

\ No newline at end of file
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2015.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2015.html
index d2a53b2ff9e1..6366cba1bd03 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2015.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2015.html
@@ -41,12 +41,12 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
extern crate self;
+
extern crate self;
 
-use crate;
-use self;
+use crate;
+use self;
 mod __ {
-    use super::*;
+    use super::*;
 }
 
 macro_rules! void {
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2018.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2018.html
index d309b4723238..a89e8190832e 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2018.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2018.html
@@ -41,12 +41,12 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
extern crate self;
+
extern crate self;
 
-use crate;
-use self;
+use crate;
+use self;
 mod __ {
-    use super::*;
+    use super::*;
 }
 
 macro_rules! void {
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2021.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2021.html
index d309b4723238..a89e8190832e 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2021.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2021.html
@@ -41,12 +41,12 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
extern crate self;
+
extern crate self;
 
-use crate;
-use self;
+use crate;
+use self;
 mod __ {
-    use super::*;
+    use super::*;
 }
 
 macro_rules! void {
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2024.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2024.html
index 575c9a6b0aca..aa1500b8f85b 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2024.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_2024.html
@@ -41,12 +41,12 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
extern crate self;
+
extern crate self;
 
-use crate;
-use self;
+use crate;
+use self;
 mod __ {
-    use super::*;
+    use super::*;
 }
 
 macro_rules! void {
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_macros.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_macros.html
index caf66ace7a68..484afd81ead2 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_macros.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_keywords_macros.html
@@ -41,6 +41,6 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
lib2015::void_2015!(try async await gen);
-lib2024::void_2024!(try async await gen);
+
lib2015::void_2015!(try async await gen);
+lib2024::void_2024!(try async await gen);
 

\ No newline at end of file
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
index b63d5cedc825..59612634fda3 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
@@ -41,7 +41,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
 .unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
-
use proc_macros::{mirror, identity, DeriveIdentity};
+
use proc_macros::{mirror, identity, DeriveIdentity};
 
 mirror! {
     {
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
index e178782c79c4..4e3822c3d31f 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
@@ -165,7 +165,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     toho!("{}fmt", 0);
     let i: u64 = 3;
     let o: u64;
-    core::arch::asm!(
+    core::arch::asm!(
         "mov {0}, {1}",
         "add {0}, 5",
         out(reg) o,
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
index 93513f5b575d..008987d409ad 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
@@ -91,7 +91,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 
         // unsafe fn and method calls
         unsafe_fn();
-        self::unsafe_fn();
+        self::unsafe_fn();
         (unsafe_fn as unsafe fn())();
         Struct { field: 0 }.unsafe_method();
 
@@ -120,7 +120,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
         &EXTERN_STATIC;
         &raw const EXTERN_STATIC;
 
-        core::arch::asm!(
+        core::arch::asm!(
             "push {base}",
             base = const 0
         );
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs
index 6f0f57725fc7..e5b983dcbf85 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs
@@ -70,6 +70,7 @@ pub(crate) fn symbol_kind(symbol_kind: SymbolKind) -> lsp_types::SymbolKind {
         | SymbolKind::Attribute
         | SymbolKind::Derive
         | SymbolKind::DeriveHelper => lsp_types::SymbolKind::FUNCTION,
+        SymbolKind::CrateRoot => lsp_types::SymbolKind::PACKAGE,
         SymbolKind::Module | SymbolKind::ToolModule => lsp_types::SymbolKind::MODULE,
         SymbolKind::TypeAlias | SymbolKind::TypeParam | SymbolKind::SelfType => {
             lsp_types::SymbolKind::TYPE_PARAMETER
@@ -141,6 +142,7 @@ pub(crate) fn completion_item_kind(
             SymbolKind::Method => lsp_types::CompletionItemKind::METHOD,
             SymbolKind::Const => lsp_types::CompletionItemKind::CONSTANT,
             SymbolKind::ConstParam => lsp_types::CompletionItemKind::TYPE_PARAMETER,
+            SymbolKind::CrateRoot => lsp_types::CompletionItemKind::MODULE,
             SymbolKind::Derive => lsp_types::CompletionItemKind::FUNCTION,
             SymbolKind::DeriveHelper => lsp_types::CompletionItemKind::FUNCTION,
             SymbolKind::Enum => lsp_types::CompletionItemKind::ENUM,
@@ -803,11 +805,16 @@ fn semantic_token_type_and_modifiers(
 ) -> (lsp_types::SemanticTokenType, semantic_tokens::ModifierSet) {
     use semantic_tokens::{modifiers as mods, types};
 
+    let mut mods = semantic_tokens::ModifierSet::default();
     let ty = match highlight.tag {
         HlTag::Symbol(symbol) => match symbol {
             SymbolKind::Attribute => types::DECORATOR,
             SymbolKind::Derive => types::DERIVE,
             SymbolKind::DeriveHelper => types::DERIVE_HELPER,
+            SymbolKind::CrateRoot => {
+                mods |= mods::CRATE_ROOT;
+                types::NAMESPACE
+            }
             SymbolKind::Module => types::NAMESPACE,
             SymbolKind::Impl => types::TYPE_ALIAS,
             SymbolKind::Field => types::PROPERTY,
@@ -870,7 +877,6 @@ fn semantic_token_type_and_modifiers(
         },
     };
 
-    let mut mods = semantic_tokens::ModifierSet::default();
     for modifier in highlight.mods.iter() {
         let modifier = match modifier {
             HlMod::Associated => mods::ASSOCIATED,

From 5f6d3852ee494e167eaa8128b1d5f81a0f03234d Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Mon, 12 Jan 2026 08:46:20 +0100
Subject: [PATCH 0200/1843] Fix crate root search in world symbols duplicating
 root entries

---
 .../rust-analyzer/crates/hir/src/symbols.rs   |  15 +--
 .../crates/ide-db/src/items_locator.rs        |   4 +-
 .../crates/ide-db/src/symbol_index.rs         | 102 +++++++++---------
 .../ide-db/src/test_data/test_doc_alias.txt   |  28 -----
 .../test_symbol_index_collection.txt          |  28 -----
 .../ide/src/syntax_highlighting/highlight.rs  |   9 +-
 .../rust-analyzer/tests/slow-tests/main.rs    |   4 +-
 7 files changed, 67 insertions(+), 123 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir/src/symbols.rs b/src/tools/rust-analyzer/crates/hir/src/symbols.rs
index f9002f31fd15..4461659f5c4e 100644
--- a/src/tools/rust-analyzer/crates/hir/src/symbols.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/symbols.rs
@@ -23,7 +23,7 @@ use intern::Symbol;
 use rustc_hash::FxHashMap;
 use syntax::{AstNode, AstPtr, SyntaxNode, SyntaxNodePtr, ToSmolStr, ast::HasName};
 
-use crate::{HasCrate, Module, ModuleDef, Semantics};
+use crate::{Crate, HasCrate, Module, ModuleDef, Semantics};
 
 /// The actual data that is stored in the index. It should be as compact as
 /// possible.
@@ -100,11 +100,6 @@ impl<'a> SymbolCollector<'a> {
         let _p = tracing::info_span!("SymbolCollector::collect", ?module).entered();
         tracing::info!(?module, "SymbolCollector::collect");
 
-        // If this is a crate root module, add a symbol for the crate itself
-        if module.is_crate_root(self.db) {
-            self.push_crate_root(module);
-        }
-
         // The initial work is the root module we're collecting, additional work will
         // be populated as we traverse the module's definitions.
         self.work.push(SymbolCollectorWork { module_id: module.into(), parent: None });
@@ -116,8 +111,7 @@ impl<'a> SymbolCollector<'a> {
 
     /// Push a symbol for a crate's root module.
     /// This allows crate roots to appear in the symbol index for queries like `::` or `::foo`.
-    fn push_crate_root(&mut self, module: Module) {
-        let krate = module.krate(self.db);
+    pub fn push_crate_root(&mut self, krate: Crate) {
         let Some(display_name) = krate.display_name(self.db) else { return };
         let crate_name = display_name.crate_name();
         let canonical_name = display_name.canonical_name();
@@ -131,10 +125,11 @@ impl<'a> SymbolCollector<'a> {
         let ptr = SyntaxNodePtr::new(&syntax_node);
 
         let loc = DeclarationLocation { hir_file_id, ptr, name_ptr: None };
+        let root_module = krate.root_module(self.db);
 
         self.symbols.insert(FileSymbol {
             name: crate_name.symbol().clone(),
-            def: ModuleDef::Module(module),
+            def: ModuleDef::Module(root_module),
             loc,
             container_name: None,
             is_alias: false,
@@ -147,7 +142,7 @@ impl<'a> SymbolCollector<'a> {
         if canonical_name != crate_name.symbol() {
             self.symbols.insert(FileSymbol {
                 name: canonical_name.clone(),
-                def: ModuleDef::Module(module),
+                def: ModuleDef::Module(root_module),
                 loc,
                 container_name: None,
                 is_alias: false,
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/items_locator.rs b/src/tools/rust-analyzer/crates/ide-db/src/items_locator.rs
index 0d305530d925..af0c69c6856d 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/items_locator.rs
+++ b/src/tools/rust-analyzer/crates/ide-db/src/items_locator.rs
@@ -110,7 +110,7 @@ pub fn items_with_name_in_module(
             local_query
         }
     };
-    local_query.search(&[SymbolIndex::module_symbols(db, module)], |local_candidate| {
+    local_query.search(db, &[SymbolIndex::module_symbols(db, module)], |local_candidate| {
         cb(match local_candidate.def {
             hir::ModuleDef::Macro(macro_def) => ItemInNs::Macros(macro_def),
             def => ItemInNs::from(def),
@@ -140,7 +140,7 @@ fn find_items(
 
     // Query the local crate using the symbol index.
     let mut local_results = Vec::new();
-    local_query.search(&symbol_index::crate_symbols(db, krate), |local_candidate| {
+    local_query.search(db, &symbol_index::crate_symbols(db, krate), |local_candidate| {
         let def = match local_candidate.def {
             hir::ModuleDef::Macro(macro_def) => ItemInNs::Macros(macro_def),
             def => ItemInNs::from(def),
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
index 05c3f360fa87..c95b541748ec 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
+++ b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
@@ -218,15 +218,18 @@ pub fn crate_symbols(db: &dyn HirDatabase, krate: Crate) -> Box<[&SymbolIndex<'_
 // | Editor  | Shortcut |
 // |---------|-----------|
 // | VS Code | Ctrl+T
-pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> {
+pub fn world_symbols(db: &RootDatabase, mut query: Query) -> Vec> {
     let _p = tracing::info_span!("world_symbols", query = ?query.query).entered();
 
-    if query.is_crate_search() {
-        return search_crates(db, &query);
-    }
-
-    // If we have a path filter, resolve it to target modules
-    let indices: Vec<_> = if !query.path_filter.is_empty() {
+    // Search for crates by name (handles "::" and "::foo" queries)
+    let indices: Vec<_> = if query.is_crate_search() {
+        query.only_types = false;
+        query.libs = true;
+        vec![SymbolIndex::extern_prelude_symbols(db)]
+        // If we have a path filter, resolve it to target modules
+    } else if !query.path_filter.is_empty() {
+        query.only_types = false;
+        query.libs = true;
         let target_modules = resolve_path_to_modules(
             db,
             &query.path_filter,
@@ -258,13 +261,17 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> {
         crates
             .par_iter()
             .for_each_with(db.clone(), |snap, &krate| _ = crate_symbols(snap, krate.into()));
-        crates.into_iter().flat_map(|krate| Vec::from(crate_symbols(db, krate.into()))).collect()
+        crates
+            .into_iter()
+            .flat_map(|krate| Vec::from(crate_symbols(db, krate.into())))
+            .chain(std::iter::once(SymbolIndex::extern_prelude_symbols(db)))
+            .collect()
     };
 
     let mut res = vec![];
 
     // Normal search: use FST to match item name
-    query.search::<()>(&indices, |f| {
+    query.search::<()>(db, &indices, |f| {
         res.push(f.clone());
         ControlFlow::Continue(())
     });
@@ -272,39 +279,6 @@ pub fn world_symbols(db: &RootDatabase, query: Query) -> Vec> {
     res
 }
 
-/// Search for crates by name (handles "::" and "::foo" queries)
-fn search_crates<'db>(db: &'db RootDatabase, query: &Query) -> Vec> {
-    let mut res = vec![];
-
-    for krate in Crate::all(db) {
-        let Some(display_name) = krate.display_name(db) else { continue };
-        let crate_name = display_name.crate_name().as_str();
-
-        // If query is empty (sole "::"), return all crates
-        // Otherwise, fuzzy match the crate name
-        let matches = if query.query.is_empty() {
-            true
-        } else {
-            query.mode.check(&query.query, query.case_sensitive, crate_name)
-        };
-
-        if matches {
-            // Get the crate root module's symbol index and find the root module symbol
-            let root_module = krate.root_module(db);
-            let index = SymbolIndex::module_symbols(db, root_module);
-            // Find the module symbol itself (representing the crate)
-            for symbol in index.symbols.iter() {
-                if matches!(symbol.def, hir::ModuleDef::Module(m) if m == root_module) {
-                    res.push(symbol.clone());
-                    break;
-                }
-            }
-        }
-    }
-
-    res
-}
-
 /// Resolve a path filter to the target module(s) it points to.
 /// Returns the modules whose symbol indices should be searched.
 ///
@@ -452,6 +426,33 @@ impl<'db> SymbolIndex<'db> {
 
         module_symbols(db, InternedModuleId::new(db, hir::ModuleId::from(module)))
     }
+
+    /// The symbol index for all extern prelude crates.
+    pub fn extern_prelude_symbols(db: &dyn HirDatabase) -> &SymbolIndex<'_> {
+        #[salsa::tracked(returns(ref))]
+        fn extern_prelude_symbols<'db>(db: &'db dyn HirDatabase) -> SymbolIndex<'db> {
+            let _p = tracing::info_span!("extern_prelude_symbols").entered();
+
+            // We call this without attaching because this runs in parallel, so we need to attach here.
+            hir::attach_db(db, || {
+                let mut collector = SymbolCollector::new(db, false);
+
+                for krate in Crate::all(db) {
+                    if krate
+                        .display_name(db)
+                        .is_none_or(|name| name.canonical_name().as_str() == "build-script-build")
+                    {
+                        continue;
+                    }
+                    collector.push_crate_root(krate);
+                }
+
+                SymbolIndex::new(collector.finish())
+            })
+        }
+
+        extern_prelude_symbols(db)
+    }
 }
 
 impl fmt::Debug for SymbolIndex<'_> {
@@ -555,6 +556,7 @@ impl Query {
     /// Search symbols in the given indices.
     pub(crate) fn search<'db, T>(
         &self,
+        db: &'db RootDatabase,
         indices: &[&'db SymbolIndex<'db>],
         cb: impl FnMut(&'db FileSymbol<'db>) -> ControlFlow,
     ) -> Option {
@@ -568,7 +570,7 @@ impl Query {
                 for index in indices.iter() {
                     op = op.add(index.map.search(&automaton));
                 }
-                self.search_maps(indices, op.union(), cb)
+                self.search_maps(db, indices, op.union(), cb)
             }
             SearchMode::Fuzzy => {
                 let automaton = fst::automaton::Subsequence::new(&self.lowercased);
@@ -576,7 +578,7 @@ impl Query {
                 for index in indices.iter() {
                     op = op.add(index.map.search(&automaton));
                 }
-                self.search_maps(indices, op.union(), cb)
+                self.search_maps(db, indices, op.union(), cb)
             }
             SearchMode::Prefix => {
                 let automaton = fst::automaton::Str::new(&self.lowercased).starts_with();
@@ -584,13 +586,14 @@ impl Query {
                 for index in indices.iter() {
                     op = op.add(index.map.search(&automaton));
                 }
-                self.search_maps(indices, op.union(), cb)
+                self.search_maps(db, indices, op.union(), cb)
             }
         }
     }
 
     fn search_maps<'db, T>(
         &self,
+        db: &'db RootDatabase,
         indices: &[&'db SymbolIndex<'db>],
         mut stream: fst::map::Union<'_>,
         mut cb: impl FnMut(&'db FileSymbol<'db>) -> ControlFlow,
@@ -598,18 +601,21 @@ impl Query {
         let ignore_underscore_prefixed = !self.query.starts_with("__");
         while let Some((_, indexed_values)) = stream.next() {
             for &IndexedValue { index, value } in indexed_values {
-                let symbol_index = &indices[index];
+                let symbol_index = indices[index];
                 let (start, end) = SymbolIndex::map_value_to_range(value);
 
                 for symbol in &symbol_index.symbols[start..end] {
                     let non_type_for_type_only_query = self.only_types
-                        && !matches!(
+                        && !(matches!(
                             symbol.def,
                             hir::ModuleDef::Adt(..)
                                 | hir::ModuleDef::TypeAlias(..)
                                 | hir::ModuleDef::BuiltinType(..)
                                 | hir::ModuleDef::Trait(..)
-                        );
+                        ) || matches!(
+                            symbol.def,
+                            hir::ModuleDef::Module(module) if module.is_crate_root(db)
+                        ));
                     if non_type_for_type_only_query || !self.matches_assoc_mode(symbol.is_assoc) {
                         continue;
                     }
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt
index 71680699b739..0c28c312f83b 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt
+++ b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_doc_alias.txt
@@ -154,34 +154,6 @@
                 do_not_complete: Yes,
                 _marker: PhantomData<&()>,
             },
-            FileSymbol {
-                name: "ra_test_fixture",
-                def: Module(
-                    Module {
-                        id: ModuleIdLt {
-                            [salsa id]: Id(3800),
-                        },
-                    },
-                ),
-                loc: DeclarationLocation {
-                    hir_file_id: FileId(
-                        EditionedFileId(
-                            Id(3000),
-                        ),
-                    ),
-                    ptr: SyntaxNodePtr {
-                        kind: SOURCE_FILE,
-                        range: 0..128,
-                    },
-                    name_ptr: None,
-                },
-                container_name: None,
-                is_alias: false,
-                is_assoc: false,
-                is_import: false,
-                do_not_complete: Yes,
-                _marker: PhantomData<&()>,
-            },
             FileSymbol {
                 name: "s1",
                 def: Adt(
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt
index 2d62a56fe22d..4b588572d328 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt
+++ b/src/tools/rust-analyzer/crates/ide-db/src/test_data/test_symbol_index_collection.txt
@@ -919,34 +919,6 @@
                 do_not_complete: Yes,
                 _marker: PhantomData<&()>,
             },
-            FileSymbol {
-                name: "ra_test_fixture",
-                def: Module(
-                    Module {
-                        id: ModuleIdLt {
-                            [salsa id]: Id(3800),
-                        },
-                    },
-                ),
-                loc: DeclarationLocation {
-                    hir_file_id: FileId(
-                        EditionedFileId(
-                            Id(3000),
-                        ),
-                    ),
-                    ptr: SyntaxNodePtr {
-                        kind: SOURCE_FILE,
-                        range: 0..793,
-                    },
-                    name_ptr: None,
-                },
-                container_name: None,
-                is_alias: false,
-                is_assoc: false,
-                is_import: false,
-                do_not_complete: Yes,
-                _marker: PhantomData<&()>,
-            },
             FileSymbol {
                 name: "really_define_struct",
                 def: Macro(
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs
index 3795d3d4146d..dcc9a8c0d5f7 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/highlight.rs
@@ -494,10 +494,9 @@ pub(super) fn highlight_def(
             (Highlight::new(HlTag::Symbol(SymbolKind::Field)), Some(field.attrs(sema.db)))
         }
         Definition::TupleField(_) => (Highlight::new(HlTag::Symbol(SymbolKind::Field)), None),
-        Definition::Crate(krate) => (
-            Highlight::new(HlTag::Symbol(SymbolKind::CrateRoot)).into(),
-            Some(krate.attrs(sema.db)),
-        ),
+        Definition::Crate(krate) => {
+            (Highlight::new(HlTag::Symbol(SymbolKind::CrateRoot)), Some(krate.attrs(sema.db)))
+        }
         Definition::Module(module) => {
             let h = Highlight::new(HlTag::Symbol(if module.is_crate_root(db) {
                 SymbolKind::CrateRoot
@@ -661,7 +660,7 @@ pub(super) fn highlight_def(
             (h, None)
         }
         Definition::ExternCrateDecl(extern_crate) => {
-            let mut highlight = Highlight::new(HlTag::Symbol(SymbolKind::CrateRoot)).into();
+            let mut highlight = Highlight::new(HlTag::Symbol(SymbolKind::CrateRoot));
             if extern_crate.alias(db).is_none() {
                 highlight |= HlMod::Library;
             }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs
index 9f3c6742d651..b4a7b44d165a 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs
@@ -1452,7 +1452,7 @@ foo = { path = "../foo" }
         json!([
         {
           "name": "bar",
-          "kind": 2,
+          "kind": 4,
           "location": {
             "uri": "file:///[..]bar/src/lib.rs",
             "range": {
@@ -1511,7 +1511,7 @@ version = "0.0.0"
         json!([
         {
           "name": "baz",
-          "kind": 2,
+          "kind": 4,
           "location": {
             "uri": "file:///[..]baz/src/lib.rs",
             "range": {

From 756e1bac6e1939dcd80125934fc6409846f02dac Mon Sep 17 00:00:00 2001
From: Jason Newcomb 
Date: Sat, 11 Oct 2025 23:48:20 -0400
Subject: [PATCH 0201/1843] `clippy_dev`: Move lint uplifting into its own
 command.

---
 clippy_dev/src/deprecate_lint.rs | 54 ++++++++++++++++++++++++++++-
 clippy_dev/src/main.rs           | 36 +++++++++-----------
 clippy_dev/src/rename_lint.rs    | 58 +++++---------------------------
 3 files changed, 78 insertions(+), 70 deletions(-)

diff --git a/clippy_dev/src/deprecate_lint.rs b/clippy_dev/src/deprecate_lint.rs
index 0401cfda7080..bee7508dabb9 100644
--- a/clippy_dev/src/deprecate_lint.rs
+++ b/clippy_dev/src/deprecate_lint.rs
@@ -1,4 +1,4 @@
-use crate::parse::{DeprecatedLint, Lint, ParseCx};
+use crate::parse::{DeprecatedLint, Lint, ParseCx, RenamedLint};
 use crate::update_lints::generate_lint_files;
 use crate::utils::{UpdateMode, Version};
 use std::ffi::OsStr;
@@ -61,6 +61,58 @@ pub fn deprecate<'cx>(cx: ParseCx<'cx>, clippy_version: Version, name: &'cx str,
     }
 }
 
+pub fn uplift<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'env str, new_name: &'env str) {
+    let mut lints = cx.find_lint_decls();
+    let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints();
+
+    let Some(lint) = lints.iter().find(|l| l.name == old_name) else {
+        eprintln!("error: failed to find lint `{old_name}`");
+        return;
+    };
+
+    let old_name_prefixed = cx.str_buf.with(|buf| {
+        buf.extend(["clippy::", old_name]);
+        cx.arena.alloc_str(buf)
+    });
+    for lint in &mut renamed_lints {
+        if lint.new_name == old_name_prefixed {
+            lint.new_name = new_name;
+        }
+    }
+    match renamed_lints.binary_search_by(|x| x.old_name.cmp(old_name_prefixed)) {
+        Ok(_) => {
+            println!("`{old_name}` is already deprecated");
+            return;
+        },
+        Err(idx) => renamed_lints.insert(
+            idx,
+            RenamedLint {
+                old_name: old_name_prefixed,
+                new_name,
+                version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()),
+            },
+        ),
+    }
+
+    let mod_path = {
+        let mut mod_path = PathBuf::from(format!("clippy_lints/src/{}", lint.module));
+        if mod_path.is_dir() {
+            mod_path = mod_path.join("mod");
+        }
+
+        mod_path.set_extension("rs");
+        mod_path
+    };
+
+    if remove_lint_declaration(old_name, &mod_path, &mut lints).unwrap_or(false) {
+        generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
+        println!("info: `{old_name}` has successfully been uplifted");
+        println!("note: you must run `cargo uitest` to update the test results");
+    } else {
+        eprintln!("error: lint not found");
+    }
+}
+
 fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec>) -> io::Result {
     fn remove_lint(name: &str, lints: &mut Vec>) {
         lints.iter().position(|l| l.name == name).map(|pos| lints.remove(pos));
diff --git a/clippy_dev/src/main.rs b/clippy_dev/src/main.rs
index 392c3aabf193..9571dfde1877 100644
--- a/clippy_dev/src/main.rs
+++ b/clippy_dev/src/main.rs
@@ -74,18 +74,11 @@ fn main() {
         },
         DevCommand::Serve { port, lint } => serve::run(port, lint),
         DevCommand::Lint { path, edition, args } => lint::run(&path, &edition, args.iter()),
-        DevCommand::RenameLint {
-            old_name,
-            new_name,
-            uplift,
-        } => new_parse_cx(|cx| {
-            rename_lint::rename(
-                cx,
-                clippy.version,
-                &old_name,
-                new_name.as_ref().unwrap_or(&old_name),
-                uplift,
-            );
+        DevCommand::RenameLint { old_name, new_name } => new_parse_cx(|cx| {
+            rename_lint::rename(cx, clippy.version, &old_name, &new_name);
+        }),
+        DevCommand::Uplift { old_name, new_name } => new_parse_cx(|cx| {
+            deprecate_lint::uplift(cx, clippy.version, &old_name, new_name.as_deref().unwrap_or(&old_name));
         }),
         DevCommand::Deprecate { name, reason } => {
             new_parse_cx(|cx| deprecate_lint::deprecate(cx, clippy.version, &name, &reason));
@@ -243,15 +236,9 @@ enum DevCommand {
         /// The name of the lint to rename
         #[arg(value_parser = lint_name)]
         old_name: String,
-        #[arg(
-            required_unless_present = "uplift",
-            value_parser = lint_name,
-        )]
+        #[arg(value_parser = lint_name)]
         /// The new name of the lint
-        new_name: Option,
-        #[arg(long)]
-        /// This lint will be uplifted into rustc
-        uplift: bool,
+        new_name: String,
     },
     /// Deprecate the given lint
     Deprecate {
@@ -266,6 +253,15 @@ enum DevCommand {
     Sync(SyncCommand),
     /// Manage Clippy releases
     Release(ReleaseCommand),
+    /// Marks a lint as uplifted into rustc and removes its code
+    Uplift {
+        /// The name of the lint to uplift
+        #[arg(value_parser = lint_name)]
+        old_name: String,
+        /// The name of the lint in rustc
+        #[arg(value_parser = lint_name)]
+        new_name: Option,
+    },
 }
 
 #[derive(Args)]
diff --git a/clippy_dev/src/rename_lint.rs b/clippy_dev/src/rename_lint.rs
index 8e30eb7ce95b..9be4f2bdc970 100644
--- a/clippy_dev/src/rename_lint.rs
+++ b/clippy_dev/src/rename_lint.rs
@@ -25,8 +25,7 @@ use std::path::Path;
 /// * If either lint name has a prefix
 /// * If `old_name` doesn't name an existing lint.
 /// * If `old_name` names a deprecated or renamed lint.
-#[expect(clippy::too_many_lines)]
-pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str, new_name: &'cx str, uplift: bool) {
+pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str, new_name: &'cx str) {
     let mut updater = FileUpdater::default();
     let mut lints = cx.find_lint_decls();
     let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints();
@@ -34,20 +33,15 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str
     let Ok(lint_idx) = lints.binary_search_by(|x| x.name.cmp(old_name)) else {
         panic!("could not find lint `{old_name}`");
     };
-    let lint = &lints[lint_idx];
 
     let old_name_prefixed = cx.str_buf.with(|buf| {
         buf.extend(["clippy::", old_name]);
         cx.arena.alloc_str(buf)
     });
-    let new_name_prefixed = if uplift {
-        new_name
-    } else {
-        cx.str_buf.with(|buf| {
-            buf.extend(["clippy::", new_name]);
-            cx.arena.alloc_str(buf)
-        })
-    };
+    let new_name_prefixed = cx.str_buf.with(|buf| {
+        buf.extend(["clippy::", new_name]);
+        cx.arena.alloc_str(buf)
+    });
 
     for lint in &mut renamed_lints {
         if lint.new_name == old_name_prefixed {
@@ -77,31 +71,7 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str
     let change_prefixed_tests = lints.get(lint_idx + 1).is_none_or(|l| !l.name.starts_with(old_name));
 
     let mut mod_edit = ModEdit::None;
-    if uplift {
-        let is_unique_mod = lints[..lint_idx].iter().any(|l| l.module == lint.module)
-            || lints[lint_idx + 1..].iter().any(|l| l.module == lint.module);
-        if is_unique_mod {
-            if delete_file_if_exists(lint.path.as_ref()) {
-                mod_edit = ModEdit::Delete;
-            }
-        } else {
-            updater.update_file(&lint.path, &mut |_, src, dst| -> UpdateStatus {
-                let mut start = &src[..lint.declaration_range.start];
-                if start.ends_with("\n\n") {
-                    start = &start[..start.len() - 1];
-                }
-                let mut end = &src[lint.declaration_range.end..];
-                if end.starts_with("\n\n") {
-                    end = &end[1..];
-                }
-                dst.push_str(start);
-                dst.push_str(end);
-                UpdateStatus::Changed
-            });
-        }
-        delete_test_files(old_name, change_prefixed_tests);
-        lints.remove(lint_idx);
-    } else if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() {
+    if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() {
         let lint = &mut lints[lint_idx];
         if lint.module.ends_with(old_name)
             && lint
@@ -139,19 +109,9 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str
     }
     generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
 
-    if uplift {
-        println!("Uplifted `clippy::{old_name}` as `{new_name}`");
-        if matches!(mod_edit, ModEdit::None) {
-            println!("Only the rename has been registered, the code will need to be edited manually");
-        } else {
-            println!("All the lint's code has been deleted");
-            println!("Make sure to inspect the results as some things may have been missed");
-        }
-    } else {
-        println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
-        println!("All code referencing the old name has been updated");
-        println!("Make sure to inspect the results as some things may have been missed");
-    }
+    println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
+    println!("All code referencing the old name has been updated");
+    println!("Make sure to inspect the results as some things may have been missed");
     println!("note: `cargo uibless` still needs to be run to update the test results");
 }
 

From d9fa6d95151ff267094da06781ef655dd0f693dd Mon Sep 17 00:00:00 2001
From: Jason Newcomb 
Date: Sun, 12 Oct 2025 14:43:21 -0400
Subject: [PATCH 0202/1843] `clippy_dev`: Rename `deprecate_lint` module to
 `edit_lints`.

---
 clippy_dev/src/edit_lints.rs | 213 +++++++++++++++++++++++++++++++++++
 clippy_dev/src/lib.rs        |   2 +-
 clippy_dev/src/main.rs       |   8 +-
 3 files changed, 218 insertions(+), 5 deletions(-)
 create mode 100644 clippy_dev/src/edit_lints.rs

diff --git a/clippy_dev/src/edit_lints.rs b/clippy_dev/src/edit_lints.rs
new file mode 100644
index 000000000000..573bd1e44535
--- /dev/null
+++ b/clippy_dev/src/edit_lints.rs
@@ -0,0 +1,213 @@
+use crate::parse::{DeprecatedLint, Lint, ParseCx, RenamedLint};
+use crate::update_lints::generate_lint_files;
+use crate::utils::{UpdateMode, Version};
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+use std::{fs, io};
+
+/// Runs the `deprecate` command
+///
+/// This does the following:
+/// * Adds an entry to `deprecated_lints.rs`.
+/// * Removes the lint declaration (and the entire file if applicable)
+///
+/// # Panics
+///
+/// If a file path could not read from or written to
+pub fn deprecate<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, name: &'env str, reason: &'env str) {
+    let mut lints = cx.find_lint_decls();
+    let (mut deprecated_lints, renamed_lints) = cx.read_deprecated_lints();
+
+    let Some(lint) = lints.iter().find(|l| l.name == name) else {
+        eprintln!("error: failed to find lint `{name}`");
+        return;
+    };
+
+    let prefixed_name = cx.str_buf.with(|buf| {
+        buf.extend(["clippy::", name]);
+        cx.arena.alloc_str(buf)
+    });
+    match deprecated_lints.binary_search_by(|x| x.name.cmp(prefixed_name)) {
+        Ok(_) => {
+            println!("`{name}` is already deprecated");
+            return;
+        },
+        Err(idx) => deprecated_lints.insert(
+            idx,
+            DeprecatedLint {
+                name: prefixed_name,
+                reason,
+                version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()),
+            },
+        ),
+    }
+
+    let mod_path = {
+        let mut mod_path = PathBuf::from(format!("clippy_lints/src/{}", lint.module));
+        if mod_path.is_dir() {
+            mod_path = mod_path.join("mod");
+        }
+
+        mod_path.set_extension("rs");
+        mod_path
+    };
+
+    if remove_lint_declaration(name, &mod_path, &mut lints).unwrap_or(false) {
+        generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
+        println!("info: `{name}` has successfully been deprecated");
+        println!("note: you must run `cargo uitest` to update the test results");
+    } else {
+        eprintln!("error: lint not found");
+    }
+}
+
+pub fn uplift<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'env str, new_name: &'env str) {
+    let mut lints = cx.find_lint_decls();
+    let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints();
+
+    let Some(lint) = lints.iter().find(|l| l.name == old_name) else {
+        eprintln!("error: failed to find lint `{old_name}`");
+        return;
+    };
+
+    let old_name_prefixed = cx.str_buf.with(|buf| {
+        buf.extend(["clippy::", old_name]);
+        cx.arena.alloc_str(buf)
+    });
+    for lint in &mut renamed_lints {
+        if lint.new_name == old_name_prefixed {
+            lint.new_name = new_name;
+        }
+    }
+    match renamed_lints.binary_search_by(|x| x.old_name.cmp(old_name_prefixed)) {
+        Ok(_) => {
+            println!("`{old_name}` is already deprecated");
+            return;
+        },
+        Err(idx) => renamed_lints.insert(
+            idx,
+            RenamedLint {
+                old_name: old_name_prefixed,
+                new_name,
+                version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()),
+            },
+        ),
+    }
+
+    let mod_path = {
+        let mut mod_path = PathBuf::from(format!("clippy_lints/src/{}", lint.module));
+        if mod_path.is_dir() {
+            mod_path = mod_path.join("mod");
+        }
+
+        mod_path.set_extension("rs");
+        mod_path
+    };
+
+    if remove_lint_declaration(old_name, &mod_path, &mut lints).unwrap_or(false) {
+        generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
+        println!("info: `{old_name}` has successfully been uplifted");
+        println!("note: you must run `cargo uitest` to update the test results");
+    } else {
+        eprintln!("error: lint not found");
+    }
+}
+
+fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec>) -> io::Result {
+    fn remove_lint(name: &str, lints: &mut Vec>) {
+        lints.iter().position(|l| l.name == name).map(|pos| lints.remove(pos));
+    }
+
+    fn remove_test_assets(name: &str) {
+        let test_file_stem = format!("tests/ui/{name}");
+        let path = Path::new(&test_file_stem);
+
+        // Some lints have their own directories, delete them
+        if path.is_dir() {
+            let _ = fs::remove_dir_all(path);
+            return;
+        }
+
+        // Remove all related test files
+        let _ = fs::remove_file(path.with_extension("rs"));
+        let _ = fs::remove_file(path.with_extension("stderr"));
+        let _ = fs::remove_file(path.with_extension("fixed"));
+    }
+
+    fn remove_impl_lint_pass(lint_name_upper: &str, content: &mut String) {
+        let impl_lint_pass_start = content.find("impl_lint_pass!").unwrap_or_else(|| {
+            content
+                .find("declare_lint_pass!")
+                .unwrap_or_else(|| panic!("failed to find `impl_lint_pass`"))
+        });
+        let mut impl_lint_pass_end = content[impl_lint_pass_start..]
+            .find(']')
+            .expect("failed to find `impl_lint_pass` terminator");
+
+        impl_lint_pass_end += impl_lint_pass_start;
+        if let Some(lint_name_pos) = content[impl_lint_pass_start..impl_lint_pass_end].find(lint_name_upper) {
+            let mut lint_name_end = impl_lint_pass_start + (lint_name_pos + lint_name_upper.len());
+            for c in content[lint_name_end..impl_lint_pass_end].chars() {
+                // Remove trailing whitespace
+                if c == ',' || c.is_whitespace() {
+                    lint_name_end += 1;
+                } else {
+                    break;
+                }
+            }
+
+            content.replace_range(impl_lint_pass_start + lint_name_pos..lint_name_end, "");
+        }
+    }
+
+    if path.exists()
+        && let Some(lint) = lints.iter().find(|l| l.name == name)
+    {
+        if lint.module == name {
+            // The lint name is the same as the file, we can just delete the entire file
+            fs::remove_file(path)?;
+        } else {
+            // We can't delete the entire file, just remove the declaration
+
+            if let Some(Some("mod.rs")) = path.file_name().map(OsStr::to_str) {
+                // Remove clippy_lints/src/some_mod/some_lint.rs
+                let mut lint_mod_path = path.to_path_buf();
+                lint_mod_path.set_file_name(name);
+                lint_mod_path.set_extension("rs");
+
+                let _ = fs::remove_file(lint_mod_path);
+            }
+
+            let mut content =
+                fs::read_to_string(path).unwrap_or_else(|_| panic!("failed to read `{}`", path.to_string_lossy()));
+
+            eprintln!(
+                "warn: you will have to manually remove any code related to `{name}` from `{}`",
+                path.display()
+            );
+
+            assert!(
+                content[lint.declaration_range].contains(&name.to_uppercase()),
+                "error: `{}` does not contain lint `{}`'s declaration",
+                path.display(),
+                lint.name
+            );
+
+            // Remove lint declaration (declare_clippy_lint!)
+            content.replace_range(lint.declaration_range, "");
+
+            // Remove the module declaration (mod xyz;)
+            let mod_decl = format!("\nmod {name};");
+            content = content.replacen(&mod_decl, "", 1);
+
+            remove_impl_lint_pass(&lint.name.to_uppercase(), &mut content);
+            fs::write(path, content).unwrap_or_else(|_| panic!("failed to write to `{}`", path.to_string_lossy()));
+        }
+
+        remove_test_assets(name);
+        remove_lint(name, lints);
+        return Ok(true);
+    }
+
+    Ok(false)
+}
diff --git a/clippy_dev/src/lib.rs b/clippy_dev/src/lib.rs
index cd103908be03..db9b230f6e59 100644
--- a/clippy_dev/src/lib.rs
+++ b/clippy_dev/src/lib.rs
@@ -23,8 +23,8 @@ extern crate rustc_arena;
 extern crate rustc_driver;
 extern crate rustc_lexer;
 
-pub mod deprecate_lint;
 pub mod dogfood;
+pub mod edit_lints;
 pub mod fmt;
 pub mod lint;
 pub mod new_lint;
diff --git a/clippy_dev/src/main.rs b/clippy_dev/src/main.rs
index 9571dfde1877..e4e19a100c6e 100644
--- a/clippy_dev/src/main.rs
+++ b/clippy_dev/src/main.rs
@@ -4,8 +4,8 @@
 
 use clap::{Args, Parser, Subcommand};
 use clippy_dev::{
-    ClippyInfo, UpdateMode, deprecate_lint, dogfood, fmt, lint, new_lint, new_parse_cx, release, rename_lint, serve,
-    setup, sync, update_lints,
+    ClippyInfo, UpdateMode, dogfood, edit_lints, fmt, lint, new_lint, new_parse_cx, release, rename_lint, serve, setup,
+    sync, update_lints,
 };
 use std::env;
 
@@ -78,10 +78,10 @@ fn main() {
             rename_lint::rename(cx, clippy.version, &old_name, &new_name);
         }),
         DevCommand::Uplift { old_name, new_name } => new_parse_cx(|cx| {
-            deprecate_lint::uplift(cx, clippy.version, &old_name, new_name.as_deref().unwrap_or(&old_name));
+            edit_lints::uplift(cx, clippy.version, &old_name, new_name.as_deref().unwrap_or(&old_name));
         }),
         DevCommand::Deprecate { name, reason } => {
-            new_parse_cx(|cx| deprecate_lint::deprecate(cx, clippy.version, &name, &reason));
+            new_parse_cx(|cx| edit_lints::deprecate(cx, clippy.version, &name, &reason));
         },
         DevCommand::Sync(SyncCommand { subcommand }) => match subcommand {
             SyncSubcommand::UpdateNightly => sync::update_nightly(),

From 8d4e435f492a54c14c8620ef665cdec4710abd19 Mon Sep 17 00:00:00 2001
From: Jason Newcomb 
Date: Sun, 12 Oct 2025 14:44:25 -0400
Subject: [PATCH 0203/1843] `clippy_dev`: Move `rename_lint` command to
 `edit_lints` module.

---
 clippy_dev/src/edit_lints.rs  | 351 ++++++++++++++++++++++++++++++++-
 clippy_dev/src/lib.rs         |   1 -
 clippy_dev/src/main.rs        |   6 +-
 clippy_dev/src/rename_lint.rs | 353 ----------------------------------
 4 files changed, 352 insertions(+), 359 deletions(-)
 delete mode 100644 clippy_dev/src/rename_lint.rs

diff --git a/clippy_dev/src/edit_lints.rs b/clippy_dev/src/edit_lints.rs
index 573bd1e44535..8a5ba26fd215 100644
--- a/clippy_dev/src/edit_lints.rs
+++ b/clippy_dev/src/edit_lints.rs
@@ -1,7 +1,12 @@
+use crate::parse::cursor::{self, Capture, Cursor};
 use crate::parse::{DeprecatedLint, Lint, ParseCx, RenamedLint};
 use crate::update_lints::generate_lint_files;
-use crate::utils::{UpdateMode, Version};
-use std::ffi::OsStr;
+use crate::utils::{
+    ErrAction, FileUpdater, UpdateMode, UpdateStatus, Version, delete_dir_if_exists, delete_file_if_exists,
+    expect_action, try_rename_dir, try_rename_file, walk_dir_no_dot_or_target,
+};
+use rustc_lexer::TokenKind;
+use std::ffi::{OsStr, OsString};
 use std::path::{Path, PathBuf};
 use std::{fs, io};
 
@@ -113,6 +118,111 @@ pub fn uplift<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
     }
 }
 
+/// Runs the `rename_lint` command.
+///
+/// This does the following:
+/// * Adds an entry to `renamed_lints.rs`.
+/// * Renames all lint attributes to the new name (e.g. `#[allow(clippy::lint_name)]`).
+/// * Renames the lint struct to the new name.
+/// * Renames the module containing the lint struct to the new name if it shares a name with the
+///   lint.
+///
+/// # Panics
+/// Panics for the following conditions:
+/// * If a file path could not read from or then written to
+/// * If either lint name has a prefix
+/// * If `old_name` doesn't name an existing lint.
+/// * If `old_name` names a deprecated or renamed lint.
+pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'env str, new_name: &'env str) {
+    let mut updater = FileUpdater::default();
+    let mut lints = cx.find_lint_decls();
+    let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints();
+
+    let Ok(lint_idx) = lints.binary_search_by(|x| x.name.cmp(old_name)) else {
+        panic!("could not find lint `{old_name}`");
+    };
+
+    let old_name_prefixed = cx.str_buf.with(|buf| {
+        buf.extend(["clippy::", old_name]);
+        cx.arena.alloc_str(buf)
+    });
+    let new_name_prefixed = cx.str_buf.with(|buf| {
+        buf.extend(["clippy::", new_name]);
+        cx.arena.alloc_str(buf)
+    });
+
+    for lint in &mut renamed_lints {
+        if lint.new_name == old_name_prefixed {
+            lint.new_name = new_name_prefixed;
+        }
+    }
+    match renamed_lints.binary_search_by(|x| x.old_name.cmp(old_name_prefixed)) {
+        Ok(_) => {
+            println!("`{old_name}` already has a rename registered");
+            return;
+        },
+        Err(idx) => {
+            renamed_lints.insert(
+                idx,
+                RenamedLint {
+                    old_name: old_name_prefixed,
+                    new_name: new_name_prefixed,
+                    version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()),
+                },
+            );
+        },
+    }
+
+    // Some tests are named `lint_name_suffix` which should also be renamed,
+    // but we can't do that if the renamed lint's name overlaps with another
+    // lint. e.g. renaming 'foo' to 'bar' when a lint 'foo_bar' also exists.
+    let change_prefixed_tests = lints.get(lint_idx + 1).is_none_or(|l| !l.name.starts_with(old_name));
+
+    let mut mod_edit = ModEdit::None;
+    if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() {
+        let lint = &mut lints[lint_idx];
+        if lint.module.ends_with(old_name)
+            && lint
+                .path
+                .file_stem()
+                .is_some_and(|x| x.as_encoded_bytes() == old_name.as_bytes())
+        {
+            let mut new_path = lint.path.with_file_name(new_name).into_os_string();
+            new_path.push(".rs");
+            if try_rename_file(lint.path.as_ref(), new_path.as_ref()) {
+                mod_edit = ModEdit::Rename;
+            }
+
+            lint.module = cx.str_buf.with(|buf| {
+                buf.push_str(&lint.module[..lint.module.len() - old_name.len()]);
+                buf.push_str(new_name);
+                cx.arena.alloc_str(buf)
+            });
+        }
+        rename_test_files(old_name, new_name, change_prefixed_tests);
+        lints[lint_idx].name = new_name;
+        lints.sort_by(|lhs, rhs| lhs.name.cmp(rhs.name));
+    } else {
+        println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
+        println!("Since `{new_name}` already exists the existing code has not been changed");
+        return;
+    }
+
+    let mut update_fn = file_update_fn(old_name, new_name, mod_edit);
+    for e in walk_dir_no_dot_or_target(".") {
+        let e = expect_action(e, ErrAction::Read, ".");
+        if e.path().as_os_str().as_encoded_bytes().ends_with(b".rs") {
+            updater.update_file(e.path(), &mut update_fn);
+        }
+    }
+    generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
+
+    println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
+    println!("All code referencing the old name has been updated");
+    println!("Make sure to inspect the results as some things may have been missed");
+    println!("note: `cargo uibless` still needs to be run to update the test results");
+}
+
 fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec>) -> io::Result {
     fn remove_lint(name: &str, lints: &mut Vec>) {
         lints.iter().position(|l| l.name == name).map(|pos| lints.remove(pos));
@@ -211,3 +321,240 @@ fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec>) -
 
     Ok(false)
 }
+
+#[derive(Clone, Copy)]
+enum ModEdit {
+    None,
+    Delete,
+    Rename,
+}
+
+fn collect_ui_test_names(lint: &str, rename_prefixed: bool, dst: &mut Vec<(OsString, bool)>) {
+    for e in fs::read_dir("tests/ui").expect("error reading `tests/ui`") {
+        let e = e.expect("error reading `tests/ui`");
+        let name = e.file_name();
+        if let Some((name_only, _)) = name.as_encoded_bytes().split_once(|&x| x == b'.') {
+            if name_only.starts_with(lint.as_bytes()) && (rename_prefixed || name_only.len() == lint.len()) {
+                dst.push((name, true));
+            }
+        } else if name.as_encoded_bytes().starts_with(lint.as_bytes()) && (rename_prefixed || name.len() == lint.len())
+        {
+            dst.push((name, false));
+        }
+    }
+}
+
+fn collect_ui_toml_test_names(lint: &str, rename_prefixed: bool, dst: &mut Vec<(OsString, bool)>) {
+    if rename_prefixed {
+        for e in fs::read_dir("tests/ui-toml").expect("error reading `tests/ui-toml`") {
+            let e = e.expect("error reading `tests/ui-toml`");
+            let name = e.file_name();
+            if name.as_encoded_bytes().starts_with(lint.as_bytes()) && e.file_type().is_ok_and(|ty| ty.is_dir()) {
+                dst.push((name, false));
+            }
+        }
+    } else {
+        dst.push((lint.into(), false));
+    }
+}
+
+/// Renames all test files for the given lint.
+///
+/// If `rename_prefixed` is `true` this will also rename tests which have the lint name as a prefix.
+fn rename_test_files(old_name: &str, new_name: &str, rename_prefixed: bool) {
+    let mut tests = Vec::new();
+
+    let mut old_buf = OsString::from("tests/ui/");
+    let mut new_buf = OsString::from("tests/ui/");
+    collect_ui_test_names(old_name, rename_prefixed, &mut tests);
+    for &(ref name, is_file) in &tests {
+        old_buf.push(name);
+        new_buf.extend([new_name.as_ref(), name.slice_encoded_bytes(old_name.len()..)]);
+        if is_file {
+            try_rename_file(old_buf.as_ref(), new_buf.as_ref());
+        } else {
+            try_rename_dir(old_buf.as_ref(), new_buf.as_ref());
+        }
+        old_buf.truncate("tests/ui/".len());
+        new_buf.truncate("tests/ui/".len());
+    }
+
+    tests.clear();
+    old_buf.truncate("tests/ui".len());
+    new_buf.truncate("tests/ui".len());
+    old_buf.push("-toml/");
+    new_buf.push("-toml/");
+    collect_ui_toml_test_names(old_name, rename_prefixed, &mut tests);
+    for (name, _) in &tests {
+        old_buf.push(name);
+        new_buf.extend([new_name.as_ref(), name.slice_encoded_bytes(old_name.len()..)]);
+        try_rename_dir(old_buf.as_ref(), new_buf.as_ref());
+        old_buf.truncate("tests/ui/".len());
+        new_buf.truncate("tests/ui/".len());
+    }
+}
+
+fn delete_test_files(lint: &str, rename_prefixed: bool) {
+    let mut tests = Vec::new();
+
+    let mut buf = OsString::from("tests/ui/");
+    collect_ui_test_names(lint, rename_prefixed, &mut tests);
+    for &(ref name, is_file) in &tests {
+        buf.push(name);
+        if is_file {
+            delete_file_if_exists(buf.as_ref());
+        } else {
+            delete_dir_if_exists(buf.as_ref());
+        }
+        buf.truncate("tests/ui/".len());
+    }
+
+    buf.truncate("tests/ui".len());
+    buf.push("-toml/");
+
+    tests.clear();
+    collect_ui_toml_test_names(lint, rename_prefixed, &mut tests);
+    for (name, _) in &tests {
+        buf.push(name);
+        delete_dir_if_exists(buf.as_ref());
+        buf.truncate("tests/ui/".len());
+    }
+}
+
+fn snake_to_pascal(s: &str) -> String {
+    let mut dst = Vec::with_capacity(s.len());
+    let mut iter = s.bytes();
+    || -> Option<()> {
+        dst.push(iter.next()?.to_ascii_uppercase());
+        while let Some(c) = iter.next() {
+            if c == b'_' {
+                dst.push(iter.next()?.to_ascii_uppercase());
+            } else {
+                dst.push(c);
+            }
+        }
+        Some(())
+    }();
+    String::from_utf8(dst).unwrap()
+}
+
+#[expect(clippy::too_many_lines)]
+fn file_update_fn<'a, 'b>(
+    old_name: &'a str,
+    new_name: &'b str,
+    mod_edit: ModEdit,
+) -> impl use<'a, 'b> + FnMut(&Path, &str, &mut String) -> UpdateStatus {
+    let old_name_pascal = snake_to_pascal(old_name);
+    let new_name_pascal = snake_to_pascal(new_name);
+    let old_name_upper = old_name.to_ascii_uppercase();
+    let new_name_upper = new_name.to_ascii_uppercase();
+    move |_, src, dst| {
+        let mut copy_pos = 0u32;
+        let mut changed = false;
+        let mut cursor = Cursor::new(src);
+        let mut captures = [Capture::EMPTY];
+        loop {
+            match cursor.peek() {
+                TokenKind::Eof => break,
+                TokenKind::Ident => {
+                    let match_start = cursor.pos();
+                    let text = cursor.peek_text();
+                    cursor.step();
+                    match text {
+                        // clippy::line_name or clippy::lint-name
+                        "clippy" => {
+                            if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent], &mut captures)
+                                && cursor.get_text(captures[0]) == old_name
+                            {
+                                dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
+                                dst.push_str(new_name);
+                                copy_pos = cursor.pos();
+                                changed = true;
+                            }
+                        },
+                        // mod lint_name
+                        "mod" => {
+                            if !matches!(mod_edit, ModEdit::None)
+                                && let Some(pos) = cursor.find_ident(old_name)
+                            {
+                                match mod_edit {
+                                    ModEdit::Rename => {
+                                        dst.push_str(&src[copy_pos as usize..pos as usize]);
+                                        dst.push_str(new_name);
+                                        copy_pos = cursor.pos();
+                                        changed = true;
+                                    },
+                                    ModEdit::Delete if cursor.match_pat(cursor::Pat::Semi) => {
+                                        let mut start = &src[copy_pos as usize..match_start as usize];
+                                        if start.ends_with("\n\n") {
+                                            start = &start[..start.len() - 1];
+                                        }
+                                        dst.push_str(start);
+                                        copy_pos = cursor.pos();
+                                        if src[copy_pos as usize..].starts_with("\n\n") {
+                                            copy_pos += 1;
+                                        }
+                                        changed = true;
+                                    },
+                                    ModEdit::Delete | ModEdit::None => {},
+                                }
+                            }
+                        },
+                        // lint_name::
+                        name if matches!(mod_edit, ModEdit::Rename) && name == old_name => {
+                            let name_end = cursor.pos();
+                            if cursor.match_pat(cursor::Pat::DoubleColon) {
+                                dst.push_str(&src[copy_pos as usize..match_start as usize]);
+                                dst.push_str(new_name);
+                                copy_pos = name_end;
+                                changed = true;
+                            }
+                        },
+                        // LINT_NAME or LintName
+                        name => {
+                            let replacement = if name == old_name_upper {
+                                &new_name_upper
+                            } else if name == old_name_pascal {
+                                &new_name_pascal
+                            } else {
+                                continue;
+                            };
+                            dst.push_str(&src[copy_pos as usize..match_start as usize]);
+                            dst.push_str(replacement);
+                            copy_pos = cursor.pos();
+                            changed = true;
+                        },
+                    }
+                },
+                // //~ lint_name
+                TokenKind::LineComment { doc_style: None } => {
+                    let text = cursor.peek_text();
+                    if text.starts_with("//~")
+                        && let Some(text) = text.strip_suffix(old_name)
+                        && !text.ends_with(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '_'))
+                    {
+                        dst.push_str(&src[copy_pos as usize..cursor.pos() as usize + text.len()]);
+                        dst.push_str(new_name);
+                        copy_pos = cursor.pos() + cursor.peek_len();
+                        changed = true;
+                    }
+                    cursor.step();
+                },
+                // ::lint_name
+                TokenKind::Colon
+                    if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent], &mut captures)
+                        && cursor.get_text(captures[0]) == old_name =>
+                {
+                    dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
+                    dst.push_str(new_name);
+                    copy_pos = cursor.pos();
+                    changed = true;
+                },
+                _ => cursor.step(),
+            }
+        }
+
+        dst.push_str(&src[copy_pos as usize..]);
+        UpdateStatus::from_changed(changed)
+    }
+}
diff --git a/clippy_dev/src/lib.rs b/clippy_dev/src/lib.rs
index db9b230f6e59..69309403c8d0 100644
--- a/clippy_dev/src/lib.rs
+++ b/clippy_dev/src/lib.rs
@@ -29,7 +29,6 @@ pub mod fmt;
 pub mod lint;
 pub mod new_lint;
 pub mod release;
-pub mod rename_lint;
 pub mod serve;
 pub mod setup;
 pub mod sync;
diff --git a/clippy_dev/src/main.rs b/clippy_dev/src/main.rs
index e4e19a100c6e..8dc2290df8e4 100644
--- a/clippy_dev/src/main.rs
+++ b/clippy_dev/src/main.rs
@@ -4,8 +4,8 @@
 
 use clap::{Args, Parser, Subcommand};
 use clippy_dev::{
-    ClippyInfo, UpdateMode, dogfood, edit_lints, fmt, lint, new_lint, new_parse_cx, release, rename_lint, serve, setup,
-    sync, update_lints,
+    ClippyInfo, UpdateMode, dogfood, edit_lints, fmt, lint, new_lint, new_parse_cx, release, serve, setup, sync,
+    update_lints,
 };
 use std::env;
 
@@ -75,7 +75,7 @@ fn main() {
         DevCommand::Serve { port, lint } => serve::run(port, lint),
         DevCommand::Lint { path, edition, args } => lint::run(&path, &edition, args.iter()),
         DevCommand::RenameLint { old_name, new_name } => new_parse_cx(|cx| {
-            rename_lint::rename(cx, clippy.version, &old_name, &new_name);
+            edit_lints::rename(cx, clippy.version, &old_name, &new_name);
         }),
         DevCommand::Uplift { old_name, new_name } => new_parse_cx(|cx| {
             edit_lints::uplift(cx, clippy.version, &old_name, new_name.as_deref().unwrap_or(&old_name));
diff --git a/clippy_dev/src/rename_lint.rs b/clippy_dev/src/rename_lint.rs
deleted file mode 100644
index 9be4f2bdc970..000000000000
--- a/clippy_dev/src/rename_lint.rs
+++ /dev/null
@@ -1,353 +0,0 @@
-use crate::parse::cursor::{self, Capture, Cursor};
-use crate::parse::{ParseCx, RenamedLint};
-use crate::update_lints::generate_lint_files;
-use crate::utils::{
-    ErrAction, FileUpdater, UpdateMode, UpdateStatus, Version, delete_dir_if_exists, delete_file_if_exists,
-    expect_action, try_rename_dir, try_rename_file, walk_dir_no_dot_or_target,
-};
-use rustc_lexer::TokenKind;
-use std::ffi::OsString;
-use std::fs;
-use std::path::Path;
-
-/// Runs the `rename_lint` command.
-///
-/// This does the following:
-/// * Adds an entry to `renamed_lints.rs`.
-/// * Renames all lint attributes to the new name (e.g. `#[allow(clippy::lint_name)]`).
-/// * Renames the lint struct to the new name.
-/// * Renames the module containing the lint struct to the new name if it shares a name with the
-///   lint.
-///
-/// # Panics
-/// Panics for the following conditions:
-/// * If a file path could not read from or then written to
-/// * If either lint name has a prefix
-/// * If `old_name` doesn't name an existing lint.
-/// * If `old_name` names a deprecated or renamed lint.
-pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str, new_name: &'cx str) {
-    let mut updater = FileUpdater::default();
-    let mut lints = cx.find_lint_decls();
-    let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints();
-
-    let Ok(lint_idx) = lints.binary_search_by(|x| x.name.cmp(old_name)) else {
-        panic!("could not find lint `{old_name}`");
-    };
-
-    let old_name_prefixed = cx.str_buf.with(|buf| {
-        buf.extend(["clippy::", old_name]);
-        cx.arena.alloc_str(buf)
-    });
-    let new_name_prefixed = cx.str_buf.with(|buf| {
-        buf.extend(["clippy::", new_name]);
-        cx.arena.alloc_str(buf)
-    });
-
-    for lint in &mut renamed_lints {
-        if lint.new_name == old_name_prefixed {
-            lint.new_name = new_name_prefixed;
-        }
-    }
-    match renamed_lints.binary_search_by(|x| x.old_name.cmp(old_name_prefixed)) {
-        Ok(_) => {
-            println!("`{old_name}` already has a rename registered");
-            return;
-        },
-        Err(idx) => {
-            renamed_lints.insert(
-                idx,
-                RenamedLint {
-                    old_name: old_name_prefixed,
-                    new_name: new_name_prefixed,
-                    version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()),
-                },
-            );
-        },
-    }
-
-    // Some tests are named `lint_name_suffix` which should also be renamed,
-    // but we can't do that if the renamed lint's name overlaps with another
-    // lint. e.g. renaming 'foo' to 'bar' when a lint 'foo_bar' also exists.
-    let change_prefixed_tests = lints.get(lint_idx + 1).is_none_or(|l| !l.name.starts_with(old_name));
-
-    let mut mod_edit = ModEdit::None;
-    if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() {
-        let lint = &mut lints[lint_idx];
-        if lint.module.ends_with(old_name)
-            && lint
-                .path
-                .file_stem()
-                .is_some_and(|x| x.as_encoded_bytes() == old_name.as_bytes())
-        {
-            let mut new_path = lint.path.with_file_name(new_name).into_os_string();
-            new_path.push(".rs");
-            if try_rename_file(lint.path.as_ref(), new_path.as_ref()) {
-                mod_edit = ModEdit::Rename;
-            }
-
-            lint.module = cx.str_buf.with(|buf| {
-                buf.push_str(&lint.module[..lint.module.len() - old_name.len()]);
-                buf.push_str(new_name);
-                cx.arena.alloc_str(buf)
-            });
-        }
-        rename_test_files(old_name, new_name, change_prefixed_tests);
-        lints[lint_idx].name = new_name;
-        lints.sort_by(|lhs, rhs| lhs.name.cmp(rhs.name));
-    } else {
-        println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
-        println!("Since `{new_name}` already exists the existing code has not been changed");
-        return;
-    }
-
-    let mut update_fn = file_update_fn(old_name, new_name, mod_edit);
-    for e in walk_dir_no_dot_or_target(".") {
-        let e = expect_action(e, ErrAction::Read, ".");
-        if e.path().as_os_str().as_encoded_bytes().ends_with(b".rs") {
-            updater.update_file(e.path(), &mut update_fn);
-        }
-    }
-    generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
-
-    println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
-    println!("All code referencing the old name has been updated");
-    println!("Make sure to inspect the results as some things may have been missed");
-    println!("note: `cargo uibless` still needs to be run to update the test results");
-}
-
-#[derive(Clone, Copy)]
-enum ModEdit {
-    None,
-    Delete,
-    Rename,
-}
-
-fn collect_ui_test_names(lint: &str, rename_prefixed: bool, dst: &mut Vec<(OsString, bool)>) {
-    for e in fs::read_dir("tests/ui").expect("error reading `tests/ui`") {
-        let e = e.expect("error reading `tests/ui`");
-        let name = e.file_name();
-        if let Some((name_only, _)) = name.as_encoded_bytes().split_once(|&x| x == b'.') {
-            if name_only.starts_with(lint.as_bytes()) && (rename_prefixed || name_only.len() == lint.len()) {
-                dst.push((name, true));
-            }
-        } else if name.as_encoded_bytes().starts_with(lint.as_bytes()) && (rename_prefixed || name.len() == lint.len())
-        {
-            dst.push((name, false));
-        }
-    }
-}
-
-fn collect_ui_toml_test_names(lint: &str, rename_prefixed: bool, dst: &mut Vec<(OsString, bool)>) {
-    if rename_prefixed {
-        for e in fs::read_dir("tests/ui-toml").expect("error reading `tests/ui-toml`") {
-            let e = e.expect("error reading `tests/ui-toml`");
-            let name = e.file_name();
-            if name.as_encoded_bytes().starts_with(lint.as_bytes()) && e.file_type().is_ok_and(|ty| ty.is_dir()) {
-                dst.push((name, false));
-            }
-        }
-    } else {
-        dst.push((lint.into(), false));
-    }
-}
-
-/// Renames all test files for the given lint.
-///
-/// If `rename_prefixed` is `true` this will also rename tests which have the lint name as a prefix.
-fn rename_test_files(old_name: &str, new_name: &str, rename_prefixed: bool) {
-    let mut tests = Vec::new();
-
-    let mut old_buf = OsString::from("tests/ui/");
-    let mut new_buf = OsString::from("tests/ui/");
-    collect_ui_test_names(old_name, rename_prefixed, &mut tests);
-    for &(ref name, is_file) in &tests {
-        old_buf.push(name);
-        new_buf.extend([new_name.as_ref(), name.slice_encoded_bytes(old_name.len()..)]);
-        if is_file {
-            try_rename_file(old_buf.as_ref(), new_buf.as_ref());
-        } else {
-            try_rename_dir(old_buf.as_ref(), new_buf.as_ref());
-        }
-        old_buf.truncate("tests/ui/".len());
-        new_buf.truncate("tests/ui/".len());
-    }
-
-    tests.clear();
-    old_buf.truncate("tests/ui".len());
-    new_buf.truncate("tests/ui".len());
-    old_buf.push("-toml/");
-    new_buf.push("-toml/");
-    collect_ui_toml_test_names(old_name, rename_prefixed, &mut tests);
-    for (name, _) in &tests {
-        old_buf.push(name);
-        new_buf.extend([new_name.as_ref(), name.slice_encoded_bytes(old_name.len()..)]);
-        try_rename_dir(old_buf.as_ref(), new_buf.as_ref());
-        old_buf.truncate("tests/ui/".len());
-        new_buf.truncate("tests/ui/".len());
-    }
-}
-
-fn delete_test_files(lint: &str, rename_prefixed: bool) {
-    let mut tests = Vec::new();
-
-    let mut buf = OsString::from("tests/ui/");
-    collect_ui_test_names(lint, rename_prefixed, &mut tests);
-    for &(ref name, is_file) in &tests {
-        buf.push(name);
-        if is_file {
-            delete_file_if_exists(buf.as_ref());
-        } else {
-            delete_dir_if_exists(buf.as_ref());
-        }
-        buf.truncate("tests/ui/".len());
-    }
-
-    buf.truncate("tests/ui".len());
-    buf.push("-toml/");
-
-    tests.clear();
-    collect_ui_toml_test_names(lint, rename_prefixed, &mut tests);
-    for (name, _) in &tests {
-        buf.push(name);
-        delete_dir_if_exists(buf.as_ref());
-        buf.truncate("tests/ui/".len());
-    }
-}
-
-fn snake_to_pascal(s: &str) -> String {
-    let mut dst = Vec::with_capacity(s.len());
-    let mut iter = s.bytes();
-    || -> Option<()> {
-        dst.push(iter.next()?.to_ascii_uppercase());
-        while let Some(c) = iter.next() {
-            if c == b'_' {
-                dst.push(iter.next()?.to_ascii_uppercase());
-            } else {
-                dst.push(c);
-            }
-        }
-        Some(())
-    }();
-    String::from_utf8(dst).unwrap()
-}
-
-#[expect(clippy::too_many_lines)]
-fn file_update_fn<'a, 'b>(
-    old_name: &'a str,
-    new_name: &'b str,
-    mod_edit: ModEdit,
-) -> impl use<'a, 'b> + FnMut(&Path, &str, &mut String) -> UpdateStatus {
-    let old_name_pascal = snake_to_pascal(old_name);
-    let new_name_pascal = snake_to_pascal(new_name);
-    let old_name_upper = old_name.to_ascii_uppercase();
-    let new_name_upper = new_name.to_ascii_uppercase();
-    move |_, src, dst| {
-        let mut copy_pos = 0u32;
-        let mut changed = false;
-        let mut cursor = Cursor::new(src);
-        let mut captures = [Capture::EMPTY];
-        loop {
-            match cursor.peek() {
-                TokenKind::Eof => break,
-                TokenKind::Ident => {
-                    let match_start = cursor.pos();
-                    let text = cursor.peek_text();
-                    cursor.step();
-                    match text {
-                        // clippy::line_name or clippy::lint-name
-                        "clippy" => {
-                            if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent], &mut captures)
-                                && cursor.get_text(captures[0]) == old_name
-                            {
-                                dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
-                                dst.push_str(new_name);
-                                copy_pos = cursor.pos();
-                                changed = true;
-                            }
-                        },
-                        // mod lint_name
-                        "mod" => {
-                            if !matches!(mod_edit, ModEdit::None)
-                                && let Some(pos) = cursor.find_ident(old_name)
-                            {
-                                match mod_edit {
-                                    ModEdit::Rename => {
-                                        dst.push_str(&src[copy_pos as usize..pos as usize]);
-                                        dst.push_str(new_name);
-                                        copy_pos = cursor.pos();
-                                        changed = true;
-                                    },
-                                    ModEdit::Delete if cursor.match_pat(cursor::Pat::Semi) => {
-                                        let mut start = &src[copy_pos as usize..match_start as usize];
-                                        if start.ends_with("\n\n") {
-                                            start = &start[..start.len() - 1];
-                                        }
-                                        dst.push_str(start);
-                                        copy_pos = cursor.pos();
-                                        if src[copy_pos as usize..].starts_with("\n\n") {
-                                            copy_pos += 1;
-                                        }
-                                        changed = true;
-                                    },
-                                    ModEdit::Delete | ModEdit::None => {},
-                                }
-                            }
-                        },
-                        // lint_name::
-                        name if matches!(mod_edit, ModEdit::Rename) && name == old_name => {
-                            let name_end = cursor.pos();
-                            if cursor.match_pat(cursor::Pat::DoubleColon) {
-                                dst.push_str(&src[copy_pos as usize..match_start as usize]);
-                                dst.push_str(new_name);
-                                copy_pos = name_end;
-                                changed = true;
-                            }
-                        },
-                        // LINT_NAME or LintName
-                        name => {
-                            let replacement = if name == old_name_upper {
-                                &new_name_upper
-                            } else if name == old_name_pascal {
-                                &new_name_pascal
-                            } else {
-                                continue;
-                            };
-                            dst.push_str(&src[copy_pos as usize..match_start as usize]);
-                            dst.push_str(replacement);
-                            copy_pos = cursor.pos();
-                            changed = true;
-                        },
-                    }
-                },
-                // //~ lint_name
-                TokenKind::LineComment { doc_style: None } => {
-                    let text = cursor.peek_text();
-                    if text.starts_with("//~")
-                        && let Some(text) = text.strip_suffix(old_name)
-                        && !text.ends_with(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '_'))
-                    {
-                        dst.push_str(&src[copy_pos as usize..cursor.pos() as usize + text.len()]);
-                        dst.push_str(new_name);
-                        copy_pos = cursor.pos() + cursor.peek_len();
-                        changed = true;
-                    }
-                    cursor.step();
-                },
-                // ::lint_name
-                TokenKind::Colon
-                    if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::CaptureIdent], &mut captures)
-                        && cursor.get_text(captures[0]) == old_name =>
-                {
-                    dst.push_str(&src[copy_pos as usize..captures[0].pos as usize]);
-                    dst.push_str(new_name);
-                    copy_pos = cursor.pos();
-                    changed = true;
-                },
-                _ => cursor.step(),
-            }
-        }
-
-        dst.push_str(&src[copy_pos as usize..]);
-        UpdateStatus::from_changed(changed)
-    }
-}

From 31db6aa5b24d9bf1751bf1017450abd4158bc06d Mon Sep 17 00:00:00 2001
From: Jason Newcomb 
Date: Sun, 12 Oct 2025 22:42:33 -0400
Subject: [PATCH 0204/1843] `clippy_dev`: When renaming a lint better handle
 the case where the renamed lint is a prefix of another lint name.

---
 clippy_dev/src/edit_lints.rs | 79 ++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 36 deletions(-)

diff --git a/clippy_dev/src/edit_lints.rs b/clippy_dev/src/edit_lints.rs
index 8a5ba26fd215..14dd8b21df9a 100644
--- a/clippy_dev/src/edit_lints.rs
+++ b/clippy_dev/src/edit_lints.rs
@@ -173,11 +173,6 @@ pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
         },
     }
 
-    // Some tests are named `lint_name_suffix` which should also be renamed,
-    // but we can't do that if the renamed lint's name overlaps with another
-    // lint. e.g. renaming 'foo' to 'bar' when a lint 'foo_bar' also exists.
-    let change_prefixed_tests = lints.get(lint_idx + 1).is_none_or(|l| !l.name.starts_with(old_name));
-
     let mut mod_edit = ModEdit::None;
     if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() {
         let lint = &mut lints[lint_idx];
@@ -199,7 +194,16 @@ pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
                 cx.arena.alloc_str(buf)
             });
         }
-        rename_test_files(old_name, new_name, change_prefixed_tests);
+
+        rename_test_files(
+            old_name,
+            new_name,
+            &lints[lint_idx + 1..]
+                .iter()
+                .map(|l| l.name)
+                .take_while(|&n| n.starts_with(old_name))
+                .collect::>(),
+        );
         lints[lint_idx].name = new_name;
         lints.sort_by(|lhs, rhs| lhs.name.cmp(rhs.name));
     } else {
@@ -329,44 +333,45 @@ enum ModEdit {
     Rename,
 }
 
-fn collect_ui_test_names(lint: &str, rename_prefixed: bool, dst: &mut Vec<(OsString, bool)>) {
+fn collect_ui_test_names(lint: &str, ignored_prefixes: &[&str], dst: &mut Vec<(OsString, bool)>) {
     for e in fs::read_dir("tests/ui").expect("error reading `tests/ui`") {
         let e = e.expect("error reading `tests/ui`");
         let name = e.file_name();
-        if let Some((name_only, _)) = name.as_encoded_bytes().split_once(|&x| x == b'.') {
-            if name_only.starts_with(lint.as_bytes()) && (rename_prefixed || name_only.len() == lint.len()) {
-                dst.push((name, true));
-            }
-        } else if name.as_encoded_bytes().starts_with(lint.as_bytes()) && (rename_prefixed || name.len() == lint.len())
+        if name.as_encoded_bytes().starts_with(lint.as_bytes())
+            && !ignored_prefixes
+                .iter()
+                .any(|&pre| name.as_encoded_bytes().starts_with(pre.as_bytes()))
+            && let Ok(ty) = e.file_type()
+            && (ty.is_file() || ty.is_dir())
+        {
+            dst.push((name, ty.is_file()));
+        }
+    }
+}
+
+fn collect_ui_toml_test_names(lint: &str, ignored_prefixes: &[&str], dst: &mut Vec<(OsString, bool)>) {
+    for e in fs::read_dir("tests/ui-toml").expect("error reading `tests/ui-toml`") {
+        let e = e.expect("error reading `tests/ui-toml`");
+        let name = e.file_name();
+        if name.as_encoded_bytes().starts_with(lint.as_bytes())
+            && !ignored_prefixes
+                .iter()
+                .any(|&pre| name.as_encoded_bytes().starts_with(pre.as_bytes()))
+            && e.file_type().is_ok_and(|ty| ty.is_dir())
         {
             dst.push((name, false));
         }
     }
 }
 
-fn collect_ui_toml_test_names(lint: &str, rename_prefixed: bool, dst: &mut Vec<(OsString, bool)>) {
-    if rename_prefixed {
-        for e in fs::read_dir("tests/ui-toml").expect("error reading `tests/ui-toml`") {
-            let e = e.expect("error reading `tests/ui-toml`");
-            let name = e.file_name();
-            if name.as_encoded_bytes().starts_with(lint.as_bytes()) && e.file_type().is_ok_and(|ty| ty.is_dir()) {
-                dst.push((name, false));
-            }
-        }
-    } else {
-        dst.push((lint.into(), false));
-    }
-}
-
-/// Renames all test files for the given lint.
-///
-/// If `rename_prefixed` is `true` this will also rename tests which have the lint name as a prefix.
-fn rename_test_files(old_name: &str, new_name: &str, rename_prefixed: bool) {
-    let mut tests = Vec::new();
+/// Renames all test files for the given lint where the file name does not start with any
+/// of the given prefixes.
+fn rename_test_files(old_name: &str, new_name: &str, ignored_prefixes: &[&str]) {
+    let mut tests: Vec<(OsString, bool)> = Vec::new();
 
     let mut old_buf = OsString::from("tests/ui/");
     let mut new_buf = OsString::from("tests/ui/");
-    collect_ui_test_names(old_name, rename_prefixed, &mut tests);
+    collect_ui_test_names(old_name, ignored_prefixes, &mut tests);
     for &(ref name, is_file) in &tests {
         old_buf.push(name);
         new_buf.extend([new_name.as_ref(), name.slice_encoded_bytes(old_name.len()..)]);
@@ -384,7 +389,7 @@ fn rename_test_files(old_name: &str, new_name: &str, rename_prefixed: bool) {
     new_buf.truncate("tests/ui".len());
     old_buf.push("-toml/");
     new_buf.push("-toml/");
-    collect_ui_toml_test_names(old_name, rename_prefixed, &mut tests);
+    collect_ui_toml_test_names(old_name, ignored_prefixes, &mut tests);
     for (name, _) in &tests {
         old_buf.push(name);
         new_buf.extend([new_name.as_ref(), name.slice_encoded_bytes(old_name.len()..)]);
@@ -394,11 +399,13 @@ fn rename_test_files(old_name: &str, new_name: &str, rename_prefixed: bool) {
     }
 }
 
-fn delete_test_files(lint: &str, rename_prefixed: bool) {
+/// Deletes all test files for the given lint where the file name does not start with any
+/// of the given prefixes.
+fn delete_test_files(lint: &str, ignored_prefixes: &[&str]) {
     let mut tests = Vec::new();
 
     let mut buf = OsString::from("tests/ui/");
-    collect_ui_test_names(lint, rename_prefixed, &mut tests);
+    collect_ui_test_names(lint, ignored_prefixes, &mut tests);
     for &(ref name, is_file) in &tests {
         buf.push(name);
         if is_file {
@@ -413,7 +420,7 @@ fn delete_test_files(lint: &str, rename_prefixed: bool) {
     buf.push("-toml/");
 
     tests.clear();
-    collect_ui_toml_test_names(lint, rename_prefixed, &mut tests);
+    collect_ui_toml_test_names(lint, ignored_prefixes, &mut tests);
     for (name, _) in &tests {
         buf.push(name);
         delete_dir_if_exists(buf.as_ref());

From 5e8d1b541c3e89587e8b07cd75208ea5e029af6a Mon Sep 17 00:00:00 2001
From: Jason Newcomb 
Date: Mon, 13 Oct 2025 17:31:51 -0400
Subject: [PATCH 0205/1843] `clippy_dev`: Re-enable deleting the module
 declaration on deprecation and uplifting.

---
 clippy_dev/src/edit_lints.rs   | 273 ++++++++++++---------------------
 clippy_dev/src/parse/cursor.rs |  16 ++
 2 files changed, 117 insertions(+), 172 deletions(-)

diff --git a/clippy_dev/src/edit_lints.rs b/clippy_dev/src/edit_lints.rs
index 14dd8b21df9a..fb1c1458c50c 100644
--- a/clippy_dev/src/edit_lints.rs
+++ b/clippy_dev/src/edit_lints.rs
@@ -6,9 +6,9 @@ use crate::utils::{
     expect_action, try_rename_dir, try_rename_file, walk_dir_no_dot_or_target,
 };
 use rustc_lexer::TokenKind;
-use std::ffi::{OsStr, OsString};
-use std::path::{Path, PathBuf};
-use std::{fs, io};
+use std::ffi::OsString;
+use std::fs;
+use std::path::Path;
 
 /// Runs the `deprecate` command
 ///
@@ -23,7 +23,7 @@ pub fn deprecate<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, name
     let mut lints = cx.find_lint_decls();
     let (mut deprecated_lints, renamed_lints) = cx.read_deprecated_lints();
 
-    let Some(lint) = lints.iter().find(|l| l.name == name) else {
+    let Some(lint_idx) = lints.iter().position(|l| l.name == name) else {
         eprintln!("error: failed to find lint `{name}`");
         return;
     };
@@ -47,30 +47,17 @@ pub fn deprecate<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, name
         ),
     }
 
-    let mod_path = {
-        let mut mod_path = PathBuf::from(format!("clippy_lints/src/{}", lint.module));
-        if mod_path.is_dir() {
-            mod_path = mod_path.join("mod");
-        }
-
-        mod_path.set_extension("rs");
-        mod_path
-    };
-
-    if remove_lint_declaration(name, &mod_path, &mut lints).unwrap_or(false) {
-        generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
-        println!("info: `{name}` has successfully been deprecated");
-        println!("note: you must run `cargo uitest` to update the test results");
-    } else {
-        eprintln!("error: lint not found");
-    }
+    remove_lint_declaration(lint_idx, &mut lints, &mut FileUpdater::default());
+    generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
+    println!("info: `{name}` has successfully been deprecated");
+    println!("note: you must run `cargo uitest` to update the test results");
 }
 
 pub fn uplift<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'env str, new_name: &'env str) {
     let mut lints = cx.find_lint_decls();
     let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints();
 
-    let Some(lint) = lints.iter().find(|l| l.name == old_name) else {
+    let Some(lint_idx) = lints.iter().position(|l| l.name == old_name) else {
         eprintln!("error: failed to find lint `{old_name}`");
         return;
     };
@@ -99,23 +86,18 @@ pub fn uplift<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
         ),
     }
 
-    let mod_path = {
-        let mut mod_path = PathBuf::from(format!("clippy_lints/src/{}", lint.module));
-        if mod_path.is_dir() {
-            mod_path = mod_path.join("mod");
+    let mut updater = FileUpdater::default();
+    let remove_mod = remove_lint_declaration(lint_idx, &mut lints, &mut updater);
+    let mut update_fn = uplift_update_fn(old_name, new_name, remove_mod);
+    for e in walk_dir_no_dot_or_target(".") {
+        let e = expect_action(e, ErrAction::Read, ".");
+        if e.path().as_os_str().as_encoded_bytes().ends_with(b".rs") {
+            updater.update_file(e.path(), &mut update_fn);
         }
-
-        mod_path.set_extension("rs");
-        mod_path
-    };
-
-    if remove_lint_declaration(old_name, &mod_path, &mut lints).unwrap_or(false) {
-        generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
-        println!("info: `{old_name}` has successfully been uplifted");
-        println!("note: you must run `cargo uitest` to update the test results");
-    } else {
-        eprintln!("error: lint not found");
     }
+    generate_lint_files(UpdateMode::Change, &lints, &deprecated_lints, &renamed_lints);
+    println!("info: `{old_name}` has successfully been uplifted as `{new_name}`");
+    println!("note: you must run `cargo uitest` to update the test results");
 }
 
 /// Runs the `rename_lint` command.
@@ -173,7 +155,7 @@ pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
         },
     }
 
-    let mut mod_edit = ModEdit::None;
+    let mut rename_mod = false;
     if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() {
         let lint = &mut lints[lint_idx];
         if lint.module.ends_with(old_name)
@@ -185,7 +167,7 @@ pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
             let mut new_path = lint.path.with_file_name(new_name).into_os_string();
             new_path.push(".rs");
             if try_rename_file(lint.path.as_ref(), new_path.as_ref()) {
-                mod_edit = ModEdit::Rename;
+                rename_mod = true;
             }
 
             lint.module = cx.str_buf.with(|buf| {
@@ -212,7 +194,7 @@ pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
         return;
     }
 
-    let mut update_fn = file_update_fn(old_name, new_name, mod_edit);
+    let mut update_fn = rename_update_fn(old_name, new_name, rename_mod);
     for e in walk_dir_no_dot_or_target(".") {
         let e = expect_action(e, ErrAction::Read, ".");
         if e.path().as_os_str().as_encoded_bytes().ends_with(b".rs") {
@@ -227,110 +209,38 @@ pub fn rename<'cx, 'env: 'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_nam
     println!("note: `cargo uibless` still needs to be run to update the test results");
 }
 
-fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec>) -> io::Result {
-    fn remove_lint(name: &str, lints: &mut Vec>) {
-        lints.iter().position(|l| l.name == name).map(|pos| lints.remove(pos));
-    }
-
-    fn remove_test_assets(name: &str) {
-        let test_file_stem = format!("tests/ui/{name}");
-        let path = Path::new(&test_file_stem);
-
-        // Some lints have their own directories, delete them
-        if path.is_dir() {
-            let _ = fs::remove_dir_all(path);
-            return;
-        }
-
-        // Remove all related test files
-        let _ = fs::remove_file(path.with_extension("rs"));
-        let _ = fs::remove_file(path.with_extension("stderr"));
-        let _ = fs::remove_file(path.with_extension("fixed"));
-    }
-
-    fn remove_impl_lint_pass(lint_name_upper: &str, content: &mut String) {
-        let impl_lint_pass_start = content.find("impl_lint_pass!").unwrap_or_else(|| {
-            content
-                .find("declare_lint_pass!")
-                .unwrap_or_else(|| panic!("failed to find `impl_lint_pass`"))
+/// Removes a lint's declaration and test files. Returns whether the module containing the
+/// lint was deleted.
+fn remove_lint_declaration(lint_idx: usize, lints: &mut Vec>, updater: &mut FileUpdater) -> bool {
+    let lint = lints.remove(lint_idx);
+    let delete_mod = if lints.iter().all(|l| l.module != lint.module) {
+        delete_file_if_exists(lint.path.as_ref())
+    } else {
+        updater.update_file(&lint.path, &mut |_, src, dst| -> UpdateStatus {
+            let mut start = &src[..lint.declaration_range.start];
+            if start.ends_with("\n\n") {
+                start = &start[..start.len() - 1];
+            }
+            let mut end = &src[lint.declaration_range.end..];
+            if end.starts_with("\n\n") {
+                end = &end[1..];
+            }
+            dst.push_str(start);
+            dst.push_str(end);
+            UpdateStatus::Changed
         });
-        let mut impl_lint_pass_end = content[impl_lint_pass_start..]
-            .find(']')
-            .expect("failed to find `impl_lint_pass` terminator");
+        false
+    };
+    delete_test_files(
+        lint.name,
+        &lints[lint_idx..]
+            .iter()
+            .map(|l| l.name)
+            .take_while(|&n| n.starts_with(lint.name))
+            .collect::>(),
+    );
 
-        impl_lint_pass_end += impl_lint_pass_start;
-        if let Some(lint_name_pos) = content[impl_lint_pass_start..impl_lint_pass_end].find(lint_name_upper) {
-            let mut lint_name_end = impl_lint_pass_start + (lint_name_pos + lint_name_upper.len());
-            for c in content[lint_name_end..impl_lint_pass_end].chars() {
-                // Remove trailing whitespace
-                if c == ',' || c.is_whitespace() {
-                    lint_name_end += 1;
-                } else {
-                    break;
-                }
-            }
-
-            content.replace_range(impl_lint_pass_start + lint_name_pos..lint_name_end, "");
-        }
-    }
-
-    if path.exists()
-        && let Some(lint) = lints.iter().find(|l| l.name == name)
-    {
-        if lint.module == name {
-            // The lint name is the same as the file, we can just delete the entire file
-            fs::remove_file(path)?;
-        } else {
-            // We can't delete the entire file, just remove the declaration
-
-            if let Some(Some("mod.rs")) = path.file_name().map(OsStr::to_str) {
-                // Remove clippy_lints/src/some_mod/some_lint.rs
-                let mut lint_mod_path = path.to_path_buf();
-                lint_mod_path.set_file_name(name);
-                lint_mod_path.set_extension("rs");
-
-                let _ = fs::remove_file(lint_mod_path);
-            }
-
-            let mut content =
-                fs::read_to_string(path).unwrap_or_else(|_| panic!("failed to read `{}`", path.to_string_lossy()));
-
-            eprintln!(
-                "warn: you will have to manually remove any code related to `{name}` from `{}`",
-                path.display()
-            );
-
-            assert!(
-                content[lint.declaration_range].contains(&name.to_uppercase()),
-                "error: `{}` does not contain lint `{}`'s declaration",
-                path.display(),
-                lint.name
-            );
-
-            // Remove lint declaration (declare_clippy_lint!)
-            content.replace_range(lint.declaration_range, "");
-
-            // Remove the module declaration (mod xyz;)
-            let mod_decl = format!("\nmod {name};");
-            content = content.replacen(&mod_decl, "", 1);
-
-            remove_impl_lint_pass(&lint.name.to_uppercase(), &mut content);
-            fs::write(path, content).unwrap_or_else(|_| panic!("failed to write to `{}`", path.to_string_lossy()));
-        }
-
-        remove_test_assets(name);
-        remove_lint(name, lints);
-        return Ok(true);
-    }
-
-    Ok(false)
-}
-
-#[derive(Clone, Copy)]
-enum ModEdit {
-    None,
-    Delete,
-    Rename,
+    delete_mod
 }
 
 fn collect_ui_test_names(lint: &str, ignored_prefixes: &[&str], dst: &mut Vec<(OsString, bool)>) {
@@ -445,12 +355,50 @@ fn snake_to_pascal(s: &str) -> String {
     String::from_utf8(dst).unwrap()
 }
 
-#[expect(clippy::too_many_lines)]
-fn file_update_fn<'a, 'b>(
+/// Creates an update function which replaces all instances of `clippy::old_name` with
+/// `new_name`.
+fn uplift_update_fn<'a>(
     old_name: &'a str,
-    new_name: &'b str,
-    mod_edit: ModEdit,
-) -> impl use<'a, 'b> + FnMut(&Path, &str, &mut String) -> UpdateStatus {
+    new_name: &'a str,
+    remove_mod: bool,
+) -> impl use<'a> + FnMut(&Path, &str, &mut String) -> UpdateStatus {
+    move |_, src, dst| {
+        let mut copy_pos = 0u32;
+        let mut changed = false;
+        let mut cursor = Cursor::new(src);
+        while let Some(ident) = cursor.find_any_ident() {
+            match cursor.get_text(ident) {
+                "mod"
+                    if remove_mod && cursor.match_all(&[cursor::Pat::Ident(old_name), cursor::Pat::Semi], &mut []) =>
+                {
+                    dst.push_str(&src[copy_pos as usize..ident.pos as usize]);
+                    dst.push_str(new_name);
+                    copy_pos = cursor.pos();
+                    if src[copy_pos as usize..].starts_with('\n') {
+                        copy_pos += 1;
+                    }
+                    changed = true;
+                },
+                "clippy" if cursor.match_all(&[cursor::Pat::DoubleColon, cursor::Pat::Ident(old_name)], &mut []) => {
+                    dst.push_str(&src[copy_pos as usize..ident.pos as usize]);
+                    dst.push_str(new_name);
+                    copy_pos = cursor.pos();
+                    changed = true;
+                },
+
+                _ => {},
+            }
+        }
+        dst.push_str(&src[copy_pos as usize..]);
+        UpdateStatus::from_changed(changed)
+    }
+}
+
+fn rename_update_fn<'a>(
+    old_name: &'a str,
+    new_name: &'a str,
+    rename_mod: bool,
+) -> impl use<'a> + FnMut(&Path, &str, &mut String) -> UpdateStatus {
     let old_name_pascal = snake_to_pascal(old_name);
     let new_name_pascal = snake_to_pascal(new_name);
     let old_name_upper = old_name.to_ascii_uppercase();
@@ -481,34 +429,15 @@ fn file_update_fn<'a, 'b>(
                         },
                         // mod lint_name
                         "mod" => {
-                            if !matches!(mod_edit, ModEdit::None)
-                                && let Some(pos) = cursor.find_ident(old_name)
-                            {
-                                match mod_edit {
-                                    ModEdit::Rename => {
-                                        dst.push_str(&src[copy_pos as usize..pos as usize]);
-                                        dst.push_str(new_name);
-                                        copy_pos = cursor.pos();
-                                        changed = true;
-                                    },
-                                    ModEdit::Delete if cursor.match_pat(cursor::Pat::Semi) => {
-                                        let mut start = &src[copy_pos as usize..match_start as usize];
-                                        if start.ends_with("\n\n") {
-                                            start = &start[..start.len() - 1];
-                                        }
-                                        dst.push_str(start);
-                                        copy_pos = cursor.pos();
-                                        if src[copy_pos as usize..].starts_with("\n\n") {
-                                            copy_pos += 1;
-                                        }
-                                        changed = true;
-                                    },
-                                    ModEdit::Delete | ModEdit::None => {},
-                                }
+                            if rename_mod && let Some(pos) = cursor.match_ident(old_name) {
+                                dst.push_str(&src[copy_pos as usize..pos as usize]);
+                                dst.push_str(new_name);
+                                copy_pos = cursor.pos();
+                                changed = true;
                             }
                         },
                         // lint_name::
-                        name if matches!(mod_edit, ModEdit::Rename) && name == old_name => {
+                        name if rename_mod && name == old_name => {
                             let name_end = cursor.pos();
                             if cursor.match_pat(cursor::Pat::DoubleColon) {
                                 dst.push_str(&src[copy_pos as usize..match_start as usize]);
diff --git a/clippy_dev/src/parse/cursor.rs b/clippy_dev/src/parse/cursor.rs
index 6dc003f326de..2c142af4883a 100644
--- a/clippy_dev/src/parse/cursor.rs
+++ b/clippy_dev/src/parse/cursor.rs
@@ -219,6 +219,22 @@ impl<'txt> Cursor<'txt> {
         }
     }
 
+    /// Consume the returns the position of the next non-whitespace token if it's an
+    /// identifier. Returns `None` otherwise.
+    pub fn match_ident(&mut self, s: &str) -> Option {
+        loop {
+            match self.next_token.kind {
+                TokenKind::Ident if s == self.peek_text() => {
+                    let pos = self.pos;
+                    self.step();
+                    return Some(pos);
+                },
+                TokenKind::Whitespace => self.step(),
+                _ => return None,
+            }
+        }
+    }
+
     /// Continually attempt to match the pattern on subsequent tokens until a match is
     /// found. Returns whether the pattern was successfully matched.
     ///

From 55f6901e967dda86ac15cf836777f1b32855feb7 Mon Sep 17 00:00:00 2001
From: Wilfred Hughes 
Date: Mon, 29 Dec 2025 03:23:48 -0800
Subject: [PATCH 0206/1843] Fix lowering with supertrait predicates

Previously both valid and invalid Rust code could crash r-a due to a
cyclic query during lowering.
---
 .../rust-analyzer/crates/hir-ty/src/lower.rs  | 38 +++++++++++----
 .../crates/hir-ty/src/tests/regression.rs     | 48 +++++++++++++++++++
 2 files changed, 78 insertions(+), 8 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs b/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs
index 9befca11b3e5..5789bf02a42e 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/lower.rs
@@ -53,7 +53,7 @@ use tracing::debug;
 use triomphe::{Arc, ThinArc};
 
 use crate::{
-    FnAbi, ImplTraitId, TyLoweringDiagnostic, TyLoweringDiagnosticKind,
+    FnAbi, ImplTraitId, TyLoweringDiagnostic, TyLoweringDiagnosticKind, all_super_traits,
     consteval::intern_const_ref,
     db::{HirDatabase, InternedOpaqueTyId},
     generics::{Generics, generics, trait_self_param_idx},
@@ -1624,11 +1624,16 @@ pub(crate) fn field_types_with_diagnostics_query<'db>(
     (res, create_diagnostics(ctx.diagnostics))
 }
 
+/// Predicates for `param_id` of the form `P: SomeTrait`. If
+/// `assoc_name` is provided, only return predicates referencing traits
+/// that have an associated type of that name.
+///
 /// This query exists only to be used when resolving short-hand associated types
 /// like `T::Item`.
 ///
 /// See the analogous query in rustc and its comment:
 /// 
+///
 /// This is a query mostly to handle cycles somewhat gracefully; e.g. the
 /// following bounds are disallowed: `T: Foo, U: Foo`, but
 /// these are fine: `T: Foo, U: Foo<()>`.
@@ -1652,7 +1657,7 @@ pub(crate) fn generic_predicates_for_param<'db>(
     );
 
     // we have to filter out all other predicates *first*, before attempting to lower them
-    let predicate = |pred: &_, ctx: &mut TyLoweringContext<'_, '_>| match pred {
+    let has_relevant_bound = |pred: &_, ctx: &mut TyLoweringContext<'_, '_>| match pred {
         WherePredicate::ForLifetime { target, bound, .. }
         | WherePredicate::TypeBound { target, bound, .. } => {
             let invalid_target = { ctx.lower_ty_only_param(*target) != Some(param_id) };
@@ -1700,11 +1705,7 @@ pub(crate) fn generic_predicates_for_param<'db>(
                         return false;
                     };
 
-                    rustc_type_ir::elaborate::supertrait_def_ids(interner, tr.into()).any(|tr| {
-                        tr.0.trait_items(db).items.iter().any(|(name, item)| {
-                            matches!(item, AssocItemId::TypeAliasId(_)) && name == assoc_name
-                        })
-                    })
+                    trait_or_supertrait_has_assoc_type(db, tr, assoc_name)
                 }
                 TypeBound::Use(_) | TypeBound::Lifetime(_) | TypeBound::Error => false,
             }
@@ -1717,7 +1718,7 @@ pub(crate) fn generic_predicates_for_param<'db>(
     {
         ctx.store = maybe_parent_generics.store();
         for pred in maybe_parent_generics.where_predicates() {
-            if predicate(pred, &mut ctx) {
+            if has_relevant_bound(pred, &mut ctx) {
                 predicates.extend(
                     ctx.lower_where_predicate(
                         pred,
@@ -1757,6 +1758,27 @@ pub(crate) fn generic_predicates_for_param_cycle_result(
     StoredEarlyBinder::bind(Clauses::empty(DbInterner::new_no_crate(db)).store())
 }
 
+/// Check if this trait or any of its supertraits define an associated
+/// type with the given name.
+fn trait_or_supertrait_has_assoc_type(
+    db: &dyn HirDatabase,
+    tr: TraitId,
+    assoc_name: &Name,
+) -> bool {
+    for trait_id in all_super_traits(db, tr) {
+        if trait_id
+            .trait_items(db)
+            .items
+            .iter()
+            .any(|(name, item)| matches!(item, AssocItemId::TypeAliasId(_)) && name == assoc_name)
+        {
+            return true;
+        }
+    }
+
+    false
+}
+
 #[inline]
 pub(crate) fn type_alias_bounds<'db>(
     db: &'db dyn HirDatabase,
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
index df49d7999fee..a04c46f8eabd 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
@@ -2598,3 +2598,51 @@ trait ColumnLike {
     "#,
     );
 }
+
+#[test]
+fn issue_21006_generic_predicates_for_param_supertrait_cycle() {
+    check_no_mismatches(
+        r#"
+trait VCipherSuite {}
+
+trait CipherSuite
+where
+    OprfHash: Hash,
+{
+}
+
+type Bar = ::Hash;
+
+type OprfHash = ::Hash;
+
+impl Foo {
+    fn seal() {}
+}
+        "#,
+    );
+}
+
+#[test]
+fn issue_21006_self_assoc_trait() {
+    check_types(
+        r#"
+trait Baz {
+    fn baz(&self);
+}
+
+trait Foo {
+    type Assoc;
+}
+
+trait Bar: Foo
+where
+    Self::Assoc: Baz,
+{
+    fn bar(v: Self::Assoc) {
+        let _ = v.baz();
+        //  ^ ()
+    }
+}
+        "#,
+    );
+}

From 2e556c490b67a4ea09cb39285b753bfc06a13604 Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Sat, 10 Jan 2026 15:31:52 +0100
Subject: [PATCH 0207/1843] fix(unnecessary_map_or): respect reduced
 applicability

---
 clippy_lints/src/methods/unnecessary_map_or.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clippy_lints/src/methods/unnecessary_map_or.rs b/clippy_lints/src/methods/unnecessary_map_or.rs
index 0c01be4b1875..7377b43571ce 100644
--- a/clippy_lints/src/methods/unnecessary_map_or.rs
+++ b/clippy_lints/src/methods/unnecessary_map_or.rs
@@ -97,12 +97,12 @@ pub(super) fn check<'a>(
         // being converted to `Some(5) == Some(5).then(|| 1)` isn't
         // the same thing
 
+        let mut app = Applicability::MachineApplicable;
         let inner_non_binding = Sugg::NonParen(Cow::Owned(format!(
             "{wrap}({})",
-            Sugg::hir(cx, non_binding_location, "")
+            Sugg::hir_with_applicability(cx, non_binding_location, "", &mut app)
         )));
 
-        let mut app = Applicability::MachineApplicable;
         let binop = make_binop(
             op.node,
             &Sugg::hir_with_applicability(cx, recv, "..", &mut app),

From c8271c1daffb9f847147f852a6325304269d064e Mon Sep 17 00:00:00 2001
From: reddevilmidzy 
Date: Sat, 10 Jan 2026 02:07:15 +0900
Subject: [PATCH 0208/1843] Added mGCA tests that were resolved

---
 .../mgca/assoc-const-projection-in-bound.rs   | 27 ++++++++++++++++
 .../mgca/cast-with-type-mismatched.rs         | 14 ++++++++
 .../mgca/cast-with-type-mismatched.stderr     | 20 ++++++++++++
 .../mgca/const-eval-with-invalid-args.rs      | 26 +++++++++++++++
 .../mgca/const-eval-with-invalid-args.stderr  | 32 +++++++++++++++++++
 .../recursive-self-referencing-const-param.rs | 11 +++++++
 ...ursive-self-referencing-const-param.stderr | 23 +++++++++++++
 .../mgca/size-of-generic-ptr-in-array-len.rs  | 10 ++++++
 .../size-of-generic-ptr-in-array-len.stderr   |  8 +++++
 9 files changed, 171 insertions(+)
 create mode 100644 tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs
 create mode 100644 tests/ui/const-generics/mgca/cast-with-type-mismatched.rs
 create mode 100644 tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr
 create mode 100644 tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs
 create mode 100644 tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr
 create mode 100644 tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs
 create mode 100644 tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr
 create mode 100644 tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs
 create mode 100644 tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr

diff --git a/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs b/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs
new file mode 100644
index 000000000000..460e14359483
--- /dev/null
+++ b/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs
@@ -0,0 +1,27 @@
+//! regression test for 
+//@ run-pass
+#![expect(incomplete_features)]
+#![feature(min_generic_const_args)]
+#![allow(dead_code)]
+
+trait Abc {}
+
+trait A {
+    #[type_const]
+    const VALUE: usize;
+}
+
+impl A for T {
+    #[type_const]
+    const VALUE: usize = 0;
+}
+
+trait S {}
+
+trait Handler
+where
+    (): S<{ ::VALUE }>,
+{
+}
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/cast-with-type-mismatched.rs b/tests/ui/const-generics/mgca/cast-with-type-mismatched.rs
new file mode 100644
index 000000000000..1f499c222e7f
--- /dev/null
+++ b/tests/ui/const-generics/mgca/cast-with-type-mismatched.rs
@@ -0,0 +1,14 @@
+//! regression test for 
+#![expect(incomplete_features)]
+#![feature(generic_const_exprs)]
+#![feature(min_generic_const_args)]
+
+fn foo(a: [(); N as usize]) {}
+//~^ ERROR: complex const arguments must be placed inside of a `const` block
+
+const C: f32 = 1.0;
+
+fn main() {
+    foo::([]);
+    //~^ ERROR: the constant `C` is not of type `u32`
+}
diff --git a/tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr b/tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr
new file mode 100644
index 000000000000..7e1b21138ec8
--- /dev/null
+++ b/tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr
@@ -0,0 +1,20 @@
+error: complex const arguments must be placed inside of a `const` block
+  --> $DIR/cast-with-type-mismatched.rs:6:30
+   |
+LL | fn foo(a: [(); N as usize]) {}
+   |                              ^^^^^^^^^^
+
+error: the constant `C` is not of type `u32`
+  --> $DIR/cast-with-type-mismatched.rs:12:11
+   |
+LL |     foo::([]);
+   |           ^ expected `u32`, found `f32`
+   |
+note: required by a const generic parameter in `foo`
+  --> $DIR/cast-with-type-mismatched.rs:6:8
+   |
+LL | fn foo(a: [(); N as usize]) {}
+   |        ^^^^^^^^^^^^ required by this const generic parameter in `foo`
+
+error: aborting due to 2 previous errors
+
diff --git a/tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs b/tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs
new file mode 100644
index 000000000000..3bf951e0c212
--- /dev/null
+++ b/tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs
@@ -0,0 +1,26 @@
+//! regression test for 
+#![expect(incomplete_features)]
+#![feature(generic_const_exprs)]
+#![feature(min_generic_const_args)]
+
+// The previous ICE was an "invalid field access on immediate".
+// If we remove `val: i32` from the field, another ICE occurs.
+// "assertion `left == right` failed: invalid field type in
+// Immediate::offset: scalar value has wrong size"
+struct A {
+    arr: usize,
+    val: i32,
+}
+
+struct B {
+    //~^ ERROR: `A` is forbidden as the type of a const generic parameter
+    arr: [u8; N.arr],
+    //~^ ERROR: complex const arguments must be placed inside of a `const` block
+}
+
+const C: u32 = 1;
+fn main() {
+    let b = B:: {arr: [1]};
+    //~^ ERROR: the constant `C` is not of type `A`
+    let _ = b.arr.len();
+}
diff --git a/tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr b/tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr
new file mode 100644
index 000000000000..a226718ccf1b
--- /dev/null
+++ b/tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr
@@ -0,0 +1,32 @@
+error: complex const arguments must be placed inside of a `const` block
+  --> $DIR/const-eval-with-invalid-args.rs:17:15
+   |
+LL |     arr: [u8; N.arr],
+   |               ^^^^^
+
+error: `A` is forbidden as the type of a const generic parameter
+  --> $DIR/const-eval-with-invalid-args.rs:15:19
+   |
+LL | struct B {
+   |                   ^
+   |
+   = note: the only supported types are integers, `bool`, and `char`
+help: add `#![feature(adt_const_params)]` to the crate attributes to enable more complex and user defined types
+   |
+LL + #![feature(adt_const_params)]
+   |
+
+error: the constant `C` is not of type `A`
+  --> $DIR/const-eval-with-invalid-args.rs:23:17
+   |
+LL |     let b = B:: {arr: [1]};
+   |                 ^ expected `A`, found `u32`
+   |
+note: required by a const generic parameter in `B`
+  --> $DIR/const-eval-with-invalid-args.rs:15:10
+   |
+LL | struct B {
+   |          ^^^^^^^^^^ required by this const generic parameter in `B`
+
+error: aborting due to 3 previous errors
+
diff --git a/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs b/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs
new file mode 100644
index 000000000000..37dcc58ff468
--- /dev/null
+++ b/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs
@@ -0,0 +1,11 @@
+//! regression test for 
+#![expect(incomplete_features)]
+#![feature(generic_const_exprs)]
+#![feature(min_generic_const_args)]
+
+fn identity }>() }>>();
+//~^ ERROR: free function without a body
+//~| ERROR: expected type, found function `identity`
+//~| ERROR: complex const arguments must be placed inside of a `const` block
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr b/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr
new file mode 100644
index 000000000000..d1899c476ec6
--- /dev/null
+++ b/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr
@@ -0,0 +1,23 @@
+error: free function without a body
+  --> $DIR/recursive-self-referencing-const-param.rs:6:1
+   |
+LL | fn identity }>() }>>();
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-
+   |                                                                       |
+   |                                                                       help: provide a definition for the function: `{  }`
+
+error[E0573]: expected type, found function `identity`
+  --> $DIR/recursive-self-referencing-const-param.rs:6:22
+   |
+LL | fn identity }>() }>>();
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ not a type
+
+error: complex const arguments must be placed inside of a `const` block
+  --> $DIR/recursive-self-referencing-const-param.rs:6:57
+   |
+LL | fn identity }>() }>>();
+   |                                                         ^^
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0573`.
diff --git a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs
new file mode 100644
index 000000000000..2a7c23929845
--- /dev/null
+++ b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs
@@ -0,0 +1,10 @@
+//! regression test for 
+#![expect(incomplete_features)]
+#![feature(min_generic_const_args)]
+
+fn foo() {
+    [0; size_of::<*mut T>()];
+    //~^ ERROR: tuple constructor with invalid base path
+}
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
new file mode 100644
index 000000000000..dcdc56a1cf47
--- /dev/null
+++ b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
@@ -0,0 +1,8 @@
+error: tuple constructor with invalid base path
+  --> $DIR/size-of-generic-ptr-in-array-len.rs:6:9
+   |
+LL |     [0; size_of::<*mut T>()];
+   |         ^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

From 01ebc285e1300de9f43a02b80a5bc408dab7bb3c Mon Sep 17 00:00:00 2001
From: Alex Butler 
Date: Mon, 12 Jan 2026 12:28:17 +0000
Subject: [PATCH 0209/1843] smol_str: update changelog 0.3.5

---
 src/tools/rust-analyzer/lib/smol_str/CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rust-analyzer/lib/smol_str/CHANGELOG.md b/src/tools/rust-analyzer/lib/smol_str/CHANGELOG.md
index b7da6d18a440..4aa25fa13446 100644
--- a/src/tools/rust-analyzer/lib/smol_str/CHANGELOG.md
+++ b/src/tools/rust-analyzer/lib/smol_str/CHANGELOG.md
@@ -1,6 +1,6 @@
 # Changelog
 
-## Unreleased
+## 0.3.5 - 2026-01-08
 - Optimise `SmolStr::clone` 4-5x speedup inline, 0.5x heap (slow down).
 
 ## 0.3.4 - 2025-10-23

From c20e6a12269678cf4012b5962ccfb73db18c940f Mon Sep 17 00:00:00 2001
From: A4-Tacks 
Date: Mon, 12 Jan 2026 20:26:24 +0800
Subject: [PATCH 0210/1843] Fix not complete `mut` and `raw` in `&x.foo()`

Example
---
```rust
fn main() {
    let _ = &$0x.foo();
}
```

**Before this PR**

```rust
...
kw loop
kw match
kw return
kw self::
...
```

**After this PR**

```rust
...
kw loop
kw match
kw mut
kw raw
kw return
kw self::
...
```
---
 .../ide-completion/src/context/analysis.rs    | 10 ++++----
 .../ide-completion/src/tests/expression.rs    | 25 ++++++++++++++++++-
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/context/analysis.rs b/src/tools/rust-analyzer/crates/ide-completion/src/context/analysis.rs
index 65bae5b66e17..0db93b0837cd 100644
--- a/src/tools/rust-analyzer/crates/ide-completion/src/context/analysis.rs
+++ b/src/tools/rust-analyzer/crates/ide-completion/src/context/analysis.rs
@@ -1305,14 +1305,14 @@ fn classify_name_ref<'db>(
 
     let make_path_kind_expr = |expr: ast::Expr| {
         let it = expr.syntax();
+        let prev_token = iter::successors(it.first_token(), |it| it.prev_token())
+            .skip(1)
+            .find(|it| !it.kind().is_trivia());
         let in_block_expr = is_in_block(it);
         let (in_loop_body, innermost_breakable) = is_in_breakable(it).unzip();
         let after_if_expr = is_after_if_expr(it.clone());
-        let ref_expr_parent =
-            path.as_single_name_ref().and_then(|_| it.parent()).and_then(ast::RefExpr::cast);
-        let after_amp = non_trivia_sibling(it.clone().into(), Direction::Prev)
-            .map(|it| it.kind() == SyntaxKind::AMP)
-            .unwrap_or(false);
+        let after_amp = prev_token.as_ref().is_some_and(|it| it.kind() == SyntaxKind::AMP);
+        let ref_expr_parent = prev_token.and_then(|it| it.parent()).and_then(ast::RefExpr::cast);
         let (innermost_ret_ty, self_param) = {
             let find_ret_ty = |it: SyntaxNode| {
                 if let Some(item) = ast::Item::cast(it.clone()) {
diff --git a/src/tools/rust-analyzer/crates/ide-completion/src/tests/expression.rs b/src/tools/rust-analyzer/crates/ide-completion/src/tests/expression.rs
index 78f003dd210b..ff005a29218b 100644
--- a/src/tools/rust-analyzer/crates/ide-completion/src/tests/expression.rs
+++ b/src/tools/rust-analyzer/crates/ide-completion/src/tests/expression.rs
@@ -706,7 +706,30 @@ fn completes_after_ref_expr() {
             kw while
             kw while let
         "#]],
-    )
+    );
+    check(
+        r#"fn main() { let _ = &$0x.foo() }"#,
+        expect![[r#"
+            fn main() fn()
+            bt u32     u32
+            kw const
+            kw crate::
+            kw false
+            kw for
+            kw if
+            kw if let
+            kw loop
+            kw match
+            kw mut
+            kw raw
+            kw return
+            kw self::
+            kw true
+            kw unsafe
+            kw while
+            kw while let
+        "#]],
+    );
 }
 
 #[test]

From 6fa61dfd995fee8ec2f766964899b55c6f304fa1 Mon Sep 17 00:00:00 2001
From: Samuel Tardieu 
Date: Sun, 11 Jan 2026 09:06:44 +0100
Subject: [PATCH 0211/1843] Do not ignore statements before a `break` when
 simplifying loop

The previous tests ignored statements in a block and only looked at the
final expression to determine if the block was made of a single `break`.
---
 clippy_lints/src/loops/while_let_loop.rs | 30 ++++++++++++++----------
 tests/ui/while_let_loop.rs               | 21 +++++++++++++++++
 2 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/clippy_lints/src/loops/while_let_loop.rs b/clippy_lints/src/loops/while_let_loop.rs
index d4285db0abfc..e2ea24c39980 100644
--- a/clippy_lints/src/loops/while_let_loop.rs
+++ b/clippy_lints/src/loops/while_let_loop.rs
@@ -40,7 +40,7 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, loop_blo
             let_info,
             Some(if_let.if_then),
         );
-    } else if els.and_then(|x| x.expr).is_some_and(is_simple_break_expr)
+    } else if els.is_some_and(is_simple_break_block)
         && let Some((pat, _)) = let_info
     {
         could_be_while_let(cx, expr, pat, init, has_trailing_exprs, let_info, None);
@@ -61,17 +61,23 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, loop_blo
     }
 }
 
-/// Returns `true` if expr contains a single break expression without a label or sub-expression,
-/// possibly embedded in blocks.
-fn is_simple_break_expr(e: &Expr<'_>) -> bool {
-    if let ExprKind::Block(b, _) = e.kind {
-        match (b.stmts, b.expr) {
-            ([s], None) => matches!(s.kind, StmtKind::Expr(e) | StmtKind::Semi(e) if is_simple_break_expr(e)),
-            ([], Some(e)) => is_simple_break_expr(e),
-            _ => false,
-        }
-    } else {
-        matches!(e.kind, ExprKind::Break(dest, None) if dest.label.is_none())
+/// Checks if `block` contains a single unlabeled `break` expression or statement, possibly embedded
+/// inside other blocks.
+fn is_simple_break_block(block: &Block<'_>) -> bool {
+    match (block.stmts, block.expr) {
+        ([s], None) => matches!(s.kind, StmtKind::Expr(e) | StmtKind::Semi(e) if is_simple_break_expr(e)),
+        ([], Some(e)) => is_simple_break_expr(e),
+        _ => false,
+    }
+}
+
+/// Checks if `expr` contains a single unlabeled `break` expression or statement, possibly embedded
+/// inside other blocks.
+fn is_simple_break_expr(expr: &Expr<'_>) -> bool {
+    match expr.kind {
+        ExprKind::Block(b, _) => is_simple_break_block(b),
+        ExprKind::Break(dest, None) => dest.label.is_none(),
+        _ => false,
     }
 }
 
diff --git a/tests/ui/while_let_loop.rs b/tests/ui/while_let_loop.rs
index f28c504742fd..b5a362669ce2 100644
--- a/tests/ui/while_let_loop.rs
+++ b/tests/ui/while_let_loop.rs
@@ -253,3 +253,24 @@ fn let_assign() {
         }
     }
 }
+
+fn issue16378() {
+    // This does not lint today because of the extra statement(s)
+    // before the `break`.
+    // TODO: When the `break` statement/expr in the `let`/`else` is the
+    // only way to leave the loop, the lint could trigger and move
+    // the statements preceeding the `break` after the loop, as in:
+    // ```rust
+    // while let Some(x) = std::hint::black_box(None::) {
+    //     println!("x = {x}");
+    // }
+    // println!("fail");
+    // ```
+    loop {
+        let Some(x) = std::hint::black_box(None::) else {
+            println!("fail");
+            break;
+        };
+        println!("x = {x}");
+    }
+}

From 2b50456eb4b3b5abdb13a36b62a92a268b740d4e Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Sun, 23 Nov 2025 20:48:47 +0100
Subject: [PATCH 0212/1843] clean-up `unnecessary_map_or` and
 `manual_is_variant_and`

- manual_is_variant_and: create `Flavor` using a function
  - removes one level of `for`
---
 .../src/methods/manual_is_variant_and.rs      | 129 ++++++++++--------
 clippy_lints/src/methods/mod.rs               |   5 +-
 .../src/methods/unnecessary_map_or.rs         |  34 ++---
 tests/ui/unnecessary_map_or.fixed             |   2 +-
 tests/ui/unnecessary_map_or.rs                |   2 +-
 tests/ui/unnecessary_map_or.stderr            |  38 +++---
 6 files changed, 111 insertions(+), 99 deletions(-)

diff --git a/clippy_lints/src/methods/manual_is_variant_and.rs b/clippy_lints/src/methods/manual_is_variant_and.rs
index 8f65858987b9..5ce9d364cdd8 100644
--- a/clippy_lints/src/methods/manual_is_variant_and.rs
+++ b/clippy_lints/src/methods/manual_is_variant_and.rs
@@ -1,12 +1,13 @@
-use clippy_utils::diagnostics::span_lint_and_sugg;
+use clippy_utils::diagnostics::{span_lint_and_sugg, span_lint_and_then};
 use clippy_utils::msrvs::{self, Msrv};
 use clippy_utils::res::MaybeDef;
-use clippy_utils::source::{snippet, snippet_with_applicability};
+use clippy_utils::source::snippet_with_applicability;
 use clippy_utils::sugg::Sugg;
 use clippy_utils::{get_parent_expr, sym};
 use rustc_ast::LitKind;
 use rustc_errors::Applicability;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
+use rustc_hir::def_id::DefId;
 use rustc_hir::{BinOpKind, Closure, Expr, ExprKind, QPath};
 use rustc_lint::LateContext;
 use rustc_middle::ty;
@@ -14,7 +15,37 @@ use rustc_span::{Span, Symbol};
 
 use super::MANUAL_IS_VARIANT_AND;
 
-pub(super) fn check(
+#[derive(Clone, Copy, PartialEq)]
+enum Flavor {
+    Option,
+    Result,
+}
+
+impl Flavor {
+    fn new(cx: &LateContext<'_>, def_id: DefId) -> Option {
+        match cx.tcx.get_diagnostic_name(def_id)? {
+            sym::Option => Some(Self::Option),
+            sym::Result => Some(Self::Result),
+            _ => None,
+        }
+    }
+
+    const fn diag_sym(self) -> Symbol {
+        match self {
+            Self::Option => sym::Option,
+            Self::Result => sym::Result,
+        }
+    }
+
+    const fn positive_variant_name(self) -> Symbol {
+        match self {
+            Self::Option => sym::Some,
+            Self::Result => sym::Ok,
+        }
+    }
+}
+
+pub(super) fn check_map_unwrap_or_default(
     cx: &LateContext<'_>,
     expr: &Expr<'_>,
     map_recv: &Expr<'_>,
@@ -30,11 +61,13 @@ pub(super) fn check(
     }
 
     // 2. the caller of `map()` is neither `Option` nor `Result`
-    let is_option = cx.typeck_results().expr_ty(map_recv).is_diag_item(cx, sym::Option);
-    let is_result = cx.typeck_results().expr_ty(map_recv).is_diag_item(cx, sym::Result);
-    if !is_option && !is_result {
+    let Some(flavor) = (cx.typeck_results())
+        .expr_ty(map_recv)
+        .opt_def_id()
+        .and_then(|did| Flavor::new(cx, did))
+    else {
         return;
-    }
+    };
 
     // 3. the caller of `unwrap_or_default` is neither `Option` nor `Result`
     if !cx.typeck_results().expr_ty(expr).is_bool() {
@@ -46,44 +79,23 @@ pub(super) fn check(
         return;
     }
 
-    let lint_msg = if is_option {
-        "called `map().unwrap_or_default()` on an `Option` value"
-    } else {
-        "called `map().unwrap_or_default()` on a `Result` value"
+    let lint_span = expr.span.with_lo(map_span.lo());
+    let lint_msg = match flavor {
+        Flavor::Option => "called `map().unwrap_or_default()` on an `Option` value",
+        Flavor::Result => "called `map().unwrap_or_default()` on a `Result` value",
     };
-    let suggestion = if is_option { "is_some_and" } else { "is_ok_and" };
 
-    span_lint_and_sugg(
-        cx,
-        MANUAL_IS_VARIANT_AND,
-        expr.span.with_lo(map_span.lo()),
-        lint_msg,
-        "use",
-        format!("{}({})", suggestion, snippet(cx, map_arg.span, "..")),
-        Applicability::MachineApplicable,
-    );
-}
+    span_lint_and_then(cx, MANUAL_IS_VARIANT_AND, lint_span, lint_msg, |diag| {
+        let method = match flavor {
+            Flavor::Option => "is_some_and",
+            Flavor::Result => "is_ok_and",
+        };
 
-#[derive(Clone, Copy, PartialEq)]
-enum Flavor {
-    Option,
-    Result,
-}
+        let mut app = Applicability::MachineApplicable;
+        let map_arg_snippet = snippet_with_applicability(cx, map_arg.span, "_", &mut app);
 
-impl Flavor {
-    const fn symbol(self) -> Symbol {
-        match self {
-            Self::Option => sym::Option,
-            Self::Result => sym::Result,
-        }
-    }
-
-    const fn positive(self) -> Symbol {
-        match self {
-            Self::Option => sym::Some,
-            Self::Result => sym::Ok,
-        }
-    }
+        diag.span_suggestion(lint_span, "use", format!("{method}({map_arg_snippet})"), app);
+    });
 }
 
 #[derive(Clone, Copy, PartialEq)]
@@ -178,7 +190,7 @@ fn emit_lint<'tcx>(
         cx,
         MANUAL_IS_VARIANT_AND,
         span,
-        format!("called `.map() {op} {pos}()`", pos = flavor.positive(),),
+        format!("called `.map() {op} {pos}()`", pos = flavor.positive_variant_name()),
         "use",
         format!(
             "{inversion}{recv}.{method}({body})",
@@ -195,24 +207,23 @@ pub(super) fn check_map(cx: &LateContext<'_>, expr: &Expr<'_>) {
         && op.span.eq_ctxt(expr.span)
         && let Ok(op) = Op::try_from(op.node)
     {
-        // Check `left` and `right` expression in any order, and for `Option` and `Result`
+        // Check `left` and `right` expression in any order
         for (expr1, expr2) in [(left, right), (right, left)] {
-            for flavor in [Flavor::Option, Flavor::Result] {
-                if let ExprKind::Call(call, [arg]) = expr1.kind
-                    && let ExprKind::Lit(lit) = arg.kind
-                    && let LitKind::Bool(bool_cst) = lit.node
-                    && let ExprKind::Path(QPath::Resolved(_, path)) = call.kind
-                    && let Res::Def(DefKind::Ctor(CtorOf::Variant, CtorKind::Fn), _) = path.res
-                    && let ty = cx.typeck_results().expr_ty(expr1)
-                    && let ty::Adt(adt, args) = ty.kind()
-                    && cx.tcx.is_diagnostic_item(flavor.symbol(), adt.did())
-                    && args.type_at(0).is_bool()
-                    && let ExprKind::MethodCall(_, recv, [map_expr], _) = expr2.kind
-                    && cx.typeck_results().expr_ty(recv).is_diag_item(cx, flavor.symbol())
-                    && let Ok(map_func) = MapFunc::try_from(map_expr)
-                {
-                    return emit_lint(cx, parent_expr.span, op, flavor, bool_cst, map_func, recv);
-                }
+            if let ExprKind::Call(call, [arg]) = expr1.kind
+                && let ExprKind::Lit(lit) = arg.kind
+                && let LitKind::Bool(bool_cst) = lit.node
+                && let ExprKind::Path(QPath::Resolved(_, path)) = call.kind
+                && let Res::Def(DefKind::Ctor(CtorOf::Variant, CtorKind::Fn), _) = path.res
+                && let ExprKind::MethodCall(_, recv, [map_expr], _) = expr2.kind
+                && let ty = cx.typeck_results().expr_ty(expr1)
+                && let ty::Adt(adt, args) = ty.kind()
+                && let Some(flavor) = Flavor::new(cx, adt.did())
+                && args.type_at(0).is_bool()
+                && cx.typeck_results().expr_ty(recv).is_diag_item(cx, flavor.diag_sym())
+                && let Ok(map_func) = MapFunc::try_from(map_expr)
+            {
+                emit_lint(cx, parent_expr.span, op, flavor, bool_cst, map_func, recv);
+                return;
             }
         }
     }
diff --git a/clippy_lints/src/methods/mod.rs b/clippy_lints/src/methods/mod.rs
index ee0cd95a5021..659a704a1ecb 100644
--- a/clippy_lints/src/methods/mod.rs
+++ b/clippy_lints/src/methods/mod.rs
@@ -3933,7 +3933,8 @@ declare_clippy_lint! {
 
 declare_clippy_lint! {
     /// ### What it does
-    /// Checks for usage of `option.map(f).unwrap_or_default()` and `result.map(f).unwrap_or_default()` where f is a function or closure that returns the `bool` type.
+    /// Checks for usage of `option.map(f).unwrap_or_default()` and `result.map(f).unwrap_or_default()` where `f` is a function or closure that returns the `bool` type.
+    ///
     /// Also checks for equality comparisons like `option.map(f) == Some(true)` and `result.map(f) == Ok(true)`.
     ///
     /// ### Why is this bad?
@@ -5629,7 +5630,7 @@ impl Methods {
                             manual_saturating_arithmetic::check_sub_unwrap_or_default(cx, expr, lhs, rhs);
                         },
                         Some((sym::map, m_recv, [arg], span, _)) => {
-                            manual_is_variant_and::check(cx, expr, m_recv, arg, span, self.msrv);
+                            manual_is_variant_and::check_map_unwrap_or_default(cx, expr, m_recv, arg, span, self.msrv);
                         },
                         Some((then_method @ (sym::then | sym::then_some), t_recv, [t_arg], _, _)) => {
                             obfuscated_if_else::check(
diff --git a/clippy_lints/src/methods/unnecessary_map_or.rs b/clippy_lints/src/methods/unnecessary_map_or.rs
index 7377b43571ce..21e112360aaf 100644
--- a/clippy_lints/src/methods/unnecessary_map_or.rs
+++ b/clippy_lints/src/methods/unnecessary_map_or.rs
@@ -8,7 +8,7 @@ use clippy_utils::sugg::{Sugg, make_binop};
 use clippy_utils::ty::{implements_trait, is_copy};
 use clippy_utils::visitors::is_local_used;
 use clippy_utils::{get_parent_expr, is_from_proc_macro};
-use rustc_ast::LitKind::Bool;
+use rustc_ast::LitKind;
 use rustc_errors::Applicability;
 use rustc_hir::{BinOpKind, Expr, ExprKind, PatKind};
 use rustc_lint::LateContext;
@@ -48,13 +48,14 @@ pub(super) fn check<'a>(
     let ExprKind::Lit(def_kind) = def.kind else {
         return;
     };
-
-    let recv_ty = cx.typeck_results().expr_ty_adjusted(recv);
-
-    let Bool(def_bool) = def_kind.node else {
+    let LitKind::Bool(def_bool) = def_kind.node else {
         return;
     };
 
+    let typeck = cx.typeck_results();
+
+    let recv_ty = typeck.expr_ty_adjusted(recv);
+
     let variant = match recv_ty.opt_diag_name(cx) {
         Some(sym::Option) => Variant::Some,
         Some(sym::Result) => Variant::Ok,
@@ -63,12 +64,12 @@ pub(super) fn check<'a>(
 
     let ext_def_span = def.span.until(map.span);
 
-    let (sugg, method, applicability) = if cx.typeck_results().expr_adjustments(recv).is_empty()
+    let (sugg, method, applicability): (_, Cow<'_, _>, _) = if typeck.expr_adjustments(recv).is_empty()
             && let ExprKind::Closure(map_closure) = map.kind
             && let closure_body = cx.tcx.hir_body(map_closure.body)
             && let closure_body_value = closure_body.value.peel_blocks()
             && let ExprKind::Binary(op, l, r) = closure_body_value.kind
-            && let Some(param) = closure_body.params.first()
+            && let [param] = closure_body.params
             && let PatKind::Binding(_, hir_id, _, _) = param.pat.kind
             // checking that map_or is one of the following:
             // .map_or(false, |x| x == y)
@@ -78,14 +79,13 @@ pub(super) fn check<'a>(
             && ((BinOpKind::Eq == op.node && !def_bool) || (BinOpKind::Ne == op.node && def_bool))
             && let non_binding_location = if l.res_local_id() == Some(hir_id) { r } else { l }
             && switch_to_eager_eval(cx, non_binding_location)
-            // if its both then that's a strange edge case and
+            // if it's both then that's a strange edge case and
             // we can just ignore it, since by default clippy will error on this
             && (l.res_local_id() == Some(hir_id)) != (r.res_local_id() == Some(hir_id))
             && !is_local_used(cx, non_binding_location, hir_id)
-            && let typeck_results = cx.typeck_results()
-            && let l_ty = typeck_results.expr_ty(l)
-            && l_ty == typeck_results.expr_ty(r)
-            && let Some(partial_eq) = cx.tcx.get_diagnostic_item(sym::PartialEq)
+            && let l_ty = typeck.expr_ty(l)
+            && l_ty == typeck.expr_ty(r)
+            && let Some(partial_eq) = cx.tcx.lang_items().eq_trait()
             && implements_trait(cx, recv_ty, partial_eq, &[recv_ty.into()])
             && is_copy(cx, l_ty)
     {
@@ -126,18 +126,18 @@ pub(super) fn check<'a>(
         }
         .into_string();
 
-        (vec![(expr.span, sugg)], "a standard comparison", app)
+        (vec![(expr.span, sugg)], "a standard comparison".into(), app)
     } else if !def_bool && msrv.meets(cx, msrvs::OPTION_RESULT_IS_VARIANT_AND) {
         let suggested_name = variant.method_name();
         (
-            vec![(method_span, suggested_name.into()), (ext_def_span, String::default())],
-            suggested_name,
+            vec![(method_span, suggested_name.into()), (ext_def_span, String::new())],
+            format!("`{suggested_name}`").into(),
             Applicability::MachineApplicable,
         )
     } else if def_bool && matches!(variant, Variant::Some) && msrv.meets(cx, msrvs::IS_NONE_OR) {
         (
-            vec![(method_span, "is_none_or".into()), (ext_def_span, String::default())],
-            "is_none_or",
+            vec![(method_span, "is_none_or".into()), (ext_def_span, String::new())],
+            "`is_none_or`".into(),
             Applicability::MachineApplicable,
         )
     } else {
diff --git a/tests/ui/unnecessary_map_or.fixed b/tests/ui/unnecessary_map_or.fixed
index 10552431d65d..52c114339292 100644
--- a/tests/ui/unnecessary_map_or.fixed
+++ b/tests/ui/unnecessary_map_or.fixed
@@ -70,7 +70,7 @@ fn main() {
     let _ = r.is_ok_and(|x| x == 7);
     //~^ unnecessary_map_or
 
-    // lint constructs that are not comparaisons as well
+    // lint constructs that are not comparisons as well
     let func = |_x| true;
     let r: Result = Ok(3);
     let _ = r.is_ok_and(func);
diff --git a/tests/ui/unnecessary_map_or.rs b/tests/ui/unnecessary_map_or.rs
index 4b406ec2998b..dd2e1a569469 100644
--- a/tests/ui/unnecessary_map_or.rs
+++ b/tests/ui/unnecessary_map_or.rs
@@ -74,7 +74,7 @@ fn main() {
     let _ = r.map_or(false, |x| x == 7);
     //~^ unnecessary_map_or
 
-    // lint constructs that are not comparaisons as well
+    // lint constructs that are not comparisons as well
     let func = |_x| true;
     let r: Result = Ok(3);
     let _ = r.map_or(false, func);
diff --git a/tests/ui/unnecessary_map_or.stderr b/tests/ui/unnecessary_map_or.stderr
index b8a22346c378..d11e7179f921 100644
--- a/tests/ui/unnecessary_map_or.stderr
+++ b/tests/ui/unnecessary_map_or.stderr
@@ -56,7 +56,7 @@ LL | |         6 >= 5
 LL | |     });
    | |______^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let _ = Some(5).map_or(false, |n| {
 LL +     let _ = Some(5).is_some_and(|n| {
@@ -68,7 +68,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(vec![5]).map_or(false, |n| n == [5]);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let _ = Some(vec![5]).map_or(false, |n| n == [5]);
 LL +     let _ = Some(vec![5]).is_some_and(|n| n == [5]);
@@ -80,7 +80,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(vec![1]).map_or(false, |n| vec![2] == n);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let _ = Some(vec![1]).map_or(false, |n| vec![2] == n);
 LL +     let _ = Some(vec![1]).is_some_and(|n| vec![2] == n);
@@ -92,7 +92,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(5).map_or(false, |n| n == n);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let _ = Some(5).map_or(false, |n| n == n);
 LL +     let _ = Some(5).is_some_and(|n| n == n);
@@ -104,7 +104,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(5).map_or(false, |n| n == if 2 > 1 { n } else { 0 });
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let _ = Some(5).map_or(false, |n| n == if 2 > 1 { n } else { 0 });
 LL +     let _ = Some(5).is_some_and(|n| n == if 2 > 1 { n } else { 0 });
@@ -116,7 +116,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Ok::, i32>(vec![5]).map_or(false, |n| n == [5]);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_ok_and instead
+help: use `is_ok_and` instead
    |
 LL -     let _ = Ok::, i32>(vec![5]).map_or(false, |n| n == [5]);
 LL +     let _ = Ok::, i32>(vec![5]).is_ok_and(|n| n == [5]);
@@ -152,7 +152,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(5).map_or(true, |n| n == 5);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     let _ = Some(5).map_or(true, |n| n == 5);
 LL +     let _ = Some(5).is_none_or(|n| n == 5);
@@ -164,7 +164,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(5).map_or(true, |n| 5 == n);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     let _ = Some(5).map_or(true, |n| 5 == n);
 LL +     let _ = Some(5).is_none_or(|n| 5 == n);
@@ -212,7 +212,7 @@ error: this `map_or` can be simplified
 LL |     let _ = r.map_or(false, |x| x == 7);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_ok_and instead
+help: use `is_ok_and` instead
    |
 LL -     let _ = r.map_or(false, |x| x == 7);
 LL +     let _ = r.is_ok_and(|x| x == 7);
@@ -224,7 +224,7 @@ error: this `map_or` can be simplified
 LL |     let _ = r.map_or(false, func);
    |             ^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_ok_and instead
+help: use `is_ok_and` instead
    |
 LL -     let _ = r.map_or(false, func);
 LL +     let _ = r.is_ok_and(func);
@@ -236,7 +236,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(5).map_or(false, func);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let _ = Some(5).map_or(false, func);
 LL +     let _ = Some(5).is_some_and(func);
@@ -248,7 +248,7 @@ error: this `map_or` can be simplified
 LL |     let _ = Some(5).map_or(true, func);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     let _ = Some(5).map_or(true, func);
 LL +     let _ = Some(5).is_none_or(func);
@@ -272,7 +272,7 @@ error: this `map_or` can be simplified
 LL |     o.map_or(true, |n| n > 5) || (o as &Option).map_or(true, |n| n < 5)
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     o.map_or(true, |n| n > 5) || (o as &Option).map_or(true, |n| n < 5)
 LL +     o.is_none_or(|n| n > 5) || (o as &Option).map_or(true, |n| n < 5)
@@ -284,7 +284,7 @@ error: this `map_or` can be simplified
 LL |     o.map_or(true, |n| n > 5) || (o as &Option).map_or(true, |n| n < 5)
    |                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     o.map_or(true, |n| n > 5) || (o as &Option).map_or(true, |n| n < 5)
 LL +     o.map_or(true, |n| n > 5) || (o as &Option).is_none_or(|n| n < 5)
@@ -296,7 +296,7 @@ error: this `map_or` can be simplified
 LL |     o.map_or(true, |n| n > 5)
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     o.map_or(true, |n| n > 5)
 LL +     o.is_none_or(|n| n > 5)
@@ -308,7 +308,7 @@ error: this `map_or` can be simplified
 LL |     let x = a.map_or(false, |a| a == *s);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     let x = a.map_or(false, |a| a == *s);
 LL +     let x = a.is_some_and(|a| a == *s);
@@ -320,7 +320,7 @@ error: this `map_or` can be simplified
 LL |     let y = b.map_or(true, |b| b == *s);
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_none_or instead
+help: use `is_none_or` instead
    |
 LL -     let y = b.map_or(true, |b| b == *s);
 LL +     let y = b.is_none_or(|b| b == *s);
@@ -356,7 +356,7 @@ error: this `map_or` can be simplified
 LL |     _ = s.lock().unwrap().map_or(false, |s| s == "foo");
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     _ = s.lock().unwrap().map_or(false, |s| s == "foo");
 LL +     _ = s.lock().unwrap().is_some_and(|s| s == "foo");
@@ -368,7 +368,7 @@ error: this `map_or` can be simplified
 LL |     _ = s.map_or(false, |s| s == "foo");
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-help: use is_some_and instead
+help: use `is_some_and` instead
    |
 LL -     _ = s.map_or(false, |s| s == "foo");
 LL +     _ = s.is_some_and(|s| s == "foo");

From 62e777ca76c62d34993dbf576957a6a9e224f034 Mon Sep 17 00:00:00 2001
From: Roberto Aloi 
Date: Mon, 12 Jan 2026 14:11:12 +0100
Subject: [PATCH 0213/1843] Fix overlapping cfg attributes for
 wasm32-unknown-emscripten target

---
 src/tools/rust-analyzer/crates/stdx/src/process.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rust-analyzer/crates/stdx/src/process.rs b/src/tools/rust-analyzer/crates/stdx/src/process.rs
index 2efeed45e44e..7c4ae978b04a 100644
--- a/src/tools/rust-analyzer/crates/stdx/src/process.rs
+++ b/src/tools/rust-analyzer/crates/stdx/src/process.rs
@@ -76,7 +76,7 @@ pub fn spawn_with_streaming_output(
     Ok(Output { status, stdout, stderr })
 }
 
-#[cfg(unix)]
+#[cfg(all(unix, not(target_arch = "wasm32")))]
 mod imp {
     use std::{
         io::{self, prelude::*},

From 41939ae3b31848a9d854c25ce2c1973de5e64e6c Mon Sep 17 00:00:00 2001
From: Ralf Jung 
Date: Mon, 12 Jan 2026 11:48:16 +0100
Subject: [PATCH 0214/1843] test closing std streams separately, and in two
 configurations

---
 .../miri/tests/pass-dep/libc/close-std-streams.rs   | 13 +++++++++++++
 src/tools/miri/tests/pass-dep/libc/libc-fs.rs       | 11 +----------
 2 files changed, 14 insertions(+), 10 deletions(-)
 create mode 100644 src/tools/miri/tests/pass-dep/libc/close-std-streams.rs

diff --git a/src/tools/miri/tests/pass-dep/libc/close-std-streams.rs b/src/tools/miri/tests/pass-dep/libc/close-std-streams.rs
new file mode 100644
index 000000000000..cc30d9557fc6
--- /dev/null
+++ b/src/tools/miri/tests/pass-dep/libc/close-std-streams.rs
@@ -0,0 +1,13 @@
+//@ignore-target: windows # no libc
+//@ revisions: default null
+//@[null] compile-flags: -Zmiri-mute-stdout-stderr
+
+fn main() {
+    // This is std library UB, but that's not relevant since we're
+    // only interacting with libc here.
+    unsafe {
+        libc::close(0);
+        libc::close(1);
+        libc::close(2);
+    }
+}
diff --git a/src/tools/miri/tests/pass-dep/libc/libc-fs.rs b/src/tools/miri/tests/pass-dep/libc/libc-fs.rs
index 8c860b5db7ba..00d5f7d97e28 100644
--- a/src/tools/miri/tests/pass-dep/libc/libc-fs.rs
+++ b/src/tools/miri/tests/pass-dep/libc/libc-fs.rs
@@ -1,4 +1,4 @@
-//@ignore-target: windows # File handling is not implemented yet
+//@ignore-target: windows # no libc
 //@compile-flags: -Zmiri-disable-isolation
 
 #![feature(io_error_more)]
@@ -48,7 +48,6 @@ fn main() {
     test_nofollow_not_symlink();
     #[cfg(target_os = "macos")]
     test_ioctl();
-    test_close_stdout();
 }
 
 fn test_file_open_unix_allow_two_args() {
@@ -580,11 +579,3 @@ fn test_ioctl() {
         assert_eq!(libc::ioctl(fd, libc::FIOCLEX), 0);
     }
 }
-
-fn test_close_stdout() {
-    // This is std library UB, but that's not relevant since we're
-    // only interacting with libc here.
-    unsafe {
-        libc::close(1);
-    }
-}

From d2e8aaa42fd3a2107fecbfc700863469909fc996 Mon Sep 17 00:00:00 2001
From: joboet 
Date: Mon, 12 Jan 2026 17:16:49 +0100
Subject: [PATCH 0215/1843] std: use `ByteStr`'s `Display` for `OsStr`

---
 library/std/src/sys/os_str/bytes.rs | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/library/std/src/sys/os_str/bytes.rs b/library/std/src/sys/os_str/bytes.rs
index 7ba6c46eaef4..5482663ef007 100644
--- a/library/std/src/sys/os_str/bytes.rs
+++ b/library/std/src/sys/os_str/bytes.rs
@@ -4,8 +4,8 @@
 use core::clone::CloneToUninit;
 
 use crate::borrow::Cow;
+use crate::bstr::ByteStr;
 use crate::collections::TryReserveError;
-use crate::fmt::Write;
 use crate::rc::Rc;
 use crate::sync::Arc;
 use crate::sys::{AsInner, FromInner, IntoInner};
@@ -64,25 +64,7 @@ impl fmt::Debug for Slice {
 
 impl fmt::Display for Slice {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // If we're the empty string then our iterator won't actually yield
-        // anything, so perform the formatting manually
-        if self.inner.is_empty() {
-            return "".fmt(f);
-        }
-
-        for chunk in self.inner.utf8_chunks() {
-            let valid = chunk.valid();
-            // If we successfully decoded the whole chunk as a valid string then
-            // we can return a direct formatting of the string which will also
-            // respect various formatting flags if possible.
-            if chunk.invalid().is_empty() {
-                return valid.fmt(f);
-            }
-
-            f.write_str(valid)?;
-            f.write_char(char::REPLACEMENT_CHARACTER)?;
-        }
-        Ok(())
+        fmt::Display::fmt(ByteStr::new(&self.inner), f)
     }
 }
 

From 3badf708c41402254349861d1a49cd3d9e128e99 Mon Sep 17 00:00:00 2001
From: Dmitry Marakasov 
Date: Mon, 8 Dec 2025 18:22:33 +0300
Subject: [PATCH 0216/1843] Mention an extra argument required to check tests

Add a note to tests-related lint descriptions that clippy needs to be
run with e.g. `--tests` argument to actually check tests, which is not
quite obvious.
---
 clippy_lints/src/attrs/mod.rs             | 4 ++++
 clippy_lints/src/redundant_test_prefix.rs | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/clippy_lints/src/attrs/mod.rs b/clippy_lints/src/attrs/mod.rs
index 42c321df61c1..fa2951d91934 100644
--- a/clippy_lints/src/attrs/mod.rs
+++ b/clippy_lints/src/attrs/mod.rs
@@ -468,6 +468,10 @@ declare_clippy_lint! {
     /// #[ignore = "Some good reason"]
     /// fn test() {}
     /// ```
+    ///
+    /// ### Note
+    /// Clippy can only lint compiled code. For this lint to trigger, you must configure `cargo clippy`
+    /// to include test compilation, for instance, by using flags such as `--tests` or `--all-targets`.
     #[clippy::version = "1.88.0"]
     pub IGNORE_WITHOUT_REASON,
     pedantic,
diff --git a/clippy_lints/src/redundant_test_prefix.rs b/clippy_lints/src/redundant_test_prefix.rs
index 84276e321657..602093259eae 100644
--- a/clippy_lints/src/redundant_test_prefix.rs
+++ b/clippy_lints/src/redundant_test_prefix.rs
@@ -47,6 +47,10 @@ declare_clippy_lint! {
     ///   }
     /// }
     /// ```
+    ///
+    /// ### Note
+    /// Clippy can only lint compiled code. For this lint to trigger, you must configure `cargo clippy`
+    /// to include test compilation, for instance, by using flags such as `--tests` or `--all-targets`.
     #[clippy::version = "1.88.0"]
     pub REDUNDANT_TEST_PREFIX,
     restriction,

From 78952f6aaa56544d1ab9ce3c18a97e34ea1f390b Mon Sep 17 00:00:00 2001
From: Jamie Cunliffe 
Date: Tue, 9 Dec 2025 16:33:40 +0000
Subject: [PATCH 0217/1843] Mark the neon intrinsics as inline(always).

The inline(always) attribute is now applied to the call site when a
function has target features enabled so that it can determine that the
call is safe to inline.
---
 .../core_arch/src/aarch64/neon/generated.rs   | 3892 +++++-----
 .../src/arm_shared/neon/generated.rs          | 6530 ++++++++---------
 library/stdarch/crates/core_arch/src/lib.rs   |    1 +
 .../crates/stdarch-gen-arm/src/intrinsic.rs   |    2 +-
 4 files changed, 5213 insertions(+), 5212 deletions(-)

diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
index 7743e52f4fe1..23b44294aa8f 100644
--- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
+++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
@@ -14,7 +14,7 @@ use super::*;
 
 #[doc = "CRC32-C single round checksum for quad words (64 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cd)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(test, assert_instr(crc32cx))]
 #[stable(feature = "stdarch_aarch64_crc32", since = "1.80.0")]
@@ -30,7 +30,7 @@ pub fn __crc32cd(crc: u32, data: u64) -> u32 {
 }
 #[doc = "CRC32 single round checksum for quad words (64 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32d)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(test, assert_instr(crc32x))]
 #[stable(feature = "stdarch_aarch64_crc32", since = "1.80.0")]
@@ -46,7 +46,7 @@ pub fn __crc32d(crc: u32, data: u64) -> u32 {
 }
 #[doc = "Floating-point JavaScript convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__jcvt)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "jsconv")]
 #[cfg_attr(test, assert_instr(fjcvtzs))]
 #[unstable(feature = "stdarch_aarch64_jscvt", issue = "147555")]
@@ -62,7 +62,7 @@ pub fn __jcvt(a: f64) -> i32 {
 }
 #[doc = "Signed Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sabal2))]
@@ -77,7 +77,7 @@ pub fn vabal_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sabal2))]
@@ -92,7 +92,7 @@ pub fn vabal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sabal2))]
@@ -107,7 +107,7 @@ pub fn vabal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uabal2))]
@@ -121,7 +121,7 @@ pub fn vabal_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t
 }
 #[doc = "Unsigned Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uabal2))]
@@ -135,7 +135,7 @@ pub fn vabal_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t
 }
 #[doc = "Unsigned Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uabal2))]
@@ -149,7 +149,7 @@ pub fn vabal_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t
 }
 #[doc = "Absolute difference between the arguments of Floating"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fabd))]
@@ -165,7 +165,7 @@ pub fn vabd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Absolute difference between the arguments of Floating"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fabd))]
@@ -181,7 +181,7 @@ pub fn vabdq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point absolute difference"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fabd))]
@@ -190,7 +190,7 @@ pub fn vabdd_f64(a: f64, b: f64) -> f64 {
 }
 #[doc = "Floating-point absolute difference"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabds_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fabd))]
@@ -199,7 +199,7 @@ pub fn vabds_f32(a: f32, b: f32) -> f32 {
 }
 #[doc = "Floating-point absolute difference"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -209,7 +209,7 @@ pub fn vabdh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Signed Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sabdl2))]
@@ -223,7 +223,7 @@ pub fn vabdl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sabdl2))]
@@ -237,7 +237,7 @@ pub fn vabdl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Signed Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sabdl2))]
@@ -251,7 +251,7 @@ pub fn vabdl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Unsigned Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uabdl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -264,7 +264,7 @@ pub fn vabdl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uabdl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -277,7 +277,7 @@ pub fn vabdl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uabdl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -290,7 +290,7 @@ pub fn vabdl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fabs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -299,7 +299,7 @@ pub fn vabs_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fabs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -308,7 +308,7 @@ pub fn vabsq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Absolute Value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(abs))]
@@ -321,7 +321,7 @@ pub fn vabs_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Absolute Value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(abs))]
@@ -334,7 +334,7 @@ pub fn vabsq_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Absolute Value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(abs))]
@@ -350,7 +350,7 @@ pub fn vabsd_s64(a: i64) -> i64 {
 }
 #[doc = "Add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -359,7 +359,7 @@ pub fn vaddd_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -368,7 +368,7 @@ pub fn vaddd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Signed Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(saddlv))]
@@ -384,7 +384,7 @@ pub fn vaddlv_s16(a: int16x4_t) -> i32 {
 }
 #[doc = "Signed Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(saddlv))]
@@ -400,7 +400,7 @@ pub fn vaddlvq_s16(a: int16x8_t) -> i32 {
 }
 #[doc = "Signed Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(saddlv))]
@@ -416,7 +416,7 @@ pub fn vaddlvq_s32(a: int32x4_t) -> i64 {
 }
 #[doc = "Signed Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(saddlp))]
@@ -432,7 +432,7 @@ pub fn vaddlv_s32(a: int32x2_t) -> i64 {
 }
 #[doc = "Signed Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(saddlv))]
@@ -448,7 +448,7 @@ pub fn vaddlv_s8(a: int8x8_t) -> i16 {
 }
 #[doc = "Signed Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(saddlv))]
@@ -464,7 +464,7 @@ pub fn vaddlvq_s8(a: int8x16_t) -> i16 {
 }
 #[doc = "Unsigned Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uaddlv))]
@@ -480,7 +480,7 @@ pub fn vaddlv_u16(a: uint16x4_t) -> u32 {
 }
 #[doc = "Unsigned Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uaddlv))]
@@ -496,7 +496,7 @@ pub fn vaddlvq_u16(a: uint16x8_t) -> u32 {
 }
 #[doc = "Unsigned Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uaddlv))]
@@ -512,7 +512,7 @@ pub fn vaddlvq_u32(a: uint32x4_t) -> u64 {
 }
 #[doc = "Unsigned Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uaddlp))]
@@ -528,7 +528,7 @@ pub fn vaddlv_u32(a: uint32x2_t) -> u64 {
 }
 #[doc = "Unsigned Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlv_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uaddlv))]
@@ -544,7 +544,7 @@ pub fn vaddlv_u8(a: uint8x8_t) -> u16 {
 }
 #[doc = "Unsigned Add Long across Vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddlvq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uaddlv))]
@@ -560,7 +560,7 @@ pub fn vaddlvq_u8(a: uint8x16_t) -> u16 {
 }
 #[doc = "Floating-point add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(faddp))]
@@ -576,7 +576,7 @@ pub fn vaddv_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(faddp))]
@@ -592,7 +592,7 @@ pub fn vaddvq_f32(a: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(faddp))]
@@ -608,7 +608,7 @@ pub fn vaddvq_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -617,7 +617,7 @@ pub fn vaddv_s32(a: int32x2_t) -> i32 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -626,7 +626,7 @@ pub fn vaddv_s8(a: int8x8_t) -> i8 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -635,7 +635,7 @@ pub fn vaddvq_s8(a: int8x16_t) -> i8 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -644,7 +644,7 @@ pub fn vaddv_s16(a: int16x4_t) -> i16 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -653,7 +653,7 @@ pub fn vaddvq_s16(a: int16x8_t) -> i16 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -662,7 +662,7 @@ pub fn vaddvq_s32(a: int32x4_t) -> i32 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -671,7 +671,7 @@ pub fn vaddv_u32(a: uint32x2_t) -> u32 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -680,7 +680,7 @@ pub fn vaddv_u8(a: uint8x8_t) -> u8 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -689,7 +689,7 @@ pub fn vaddvq_u8(a: uint8x16_t) -> u8 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddv_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -698,7 +698,7 @@ pub fn vaddv_u16(a: uint16x4_t) -> u16 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -707,7 +707,7 @@ pub fn vaddvq_u16(a: uint16x8_t) -> u16 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addv))]
@@ -716,7 +716,7 @@ pub fn vaddvq_u32(a: uint32x4_t) -> u32 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -725,7 +725,7 @@ pub fn vaddvq_s64(a: int64x2_t) -> i64 {
 }
 #[doc = "Add across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddvq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -734,7 +734,7 @@ pub fn vaddvq_u64(a: uint64x2_t) -> u64 {
 }
 #[doc = "Multi-vector floating-point absolute maximum"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vamax_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,faminmax")]
 #[cfg_attr(test, assert_instr(nop))]
 #[unstable(feature = "faminmax", issue = "137933")]
@@ -750,7 +750,7 @@ pub fn vamax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Multi-vector floating-point absolute maximum"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vamaxq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,faminmax")]
 #[cfg_attr(test, assert_instr(nop))]
 #[unstable(feature = "faminmax", issue = "137933")]
@@ -766,7 +766,7 @@ pub fn vamaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Multi-vector floating-point absolute maximum"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vamaxq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,faminmax")]
 #[cfg_attr(test, assert_instr(nop))]
 #[unstable(feature = "faminmax", issue = "137933")]
@@ -782,7 +782,7 @@ pub fn vamaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Multi-vector floating-point absolute minimum"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vamin_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,faminmax")]
 #[cfg_attr(test, assert_instr(nop))]
 #[unstable(feature = "faminmax", issue = "137933")]
@@ -798,7 +798,7 @@ pub fn vamin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Multi-vector floating-point absolute minimum"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaminq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,faminmax")]
 #[cfg_attr(test, assert_instr(nop))]
 #[unstable(feature = "faminmax", issue = "137933")]
@@ -814,7 +814,7 @@ pub fn vaminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Multi-vector floating-point absolute minimum"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaminq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,faminmax")]
 #[cfg_attr(test, assert_instr(nop))]
 #[unstable(feature = "faminmax", issue = "137933")]
@@ -830,7 +830,7 @@ pub fn vaminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -846,7 +846,7 @@ pub fn vbcaxq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -862,7 +862,7 @@ pub fn vbcaxq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -878,7 +878,7 @@ pub fn vbcaxq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -894,7 +894,7 @@ pub fn vbcaxq_s64(a: int64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -910,7 +910,7 @@ pub fn vbcaxq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -926,7 +926,7 @@ pub fn vbcaxq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -942,7 +942,7 @@ pub fn vbcaxq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Bit clear and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbcaxq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(bcax))]
@@ -958,7 +958,7 @@ pub fn vbcaxq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcadd_rot270_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fcma"))]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -976,7 +976,7 @@ pub fn vcadd_rot270_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot270_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fcma"))]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -994,7 +994,7 @@ pub fn vcaddq_rot270_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcadd_rot270_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcadd))]
@@ -1010,7 +1010,7 @@ pub fn vcadd_rot270_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot270_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcadd))]
@@ -1026,7 +1026,7 @@ pub fn vcaddq_rot270_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot270_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcadd))]
@@ -1042,7 +1042,7 @@ pub fn vcaddq_rot270_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcadd_rot90_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fcma"))]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -1060,7 +1060,7 @@ pub fn vcadd_rot90_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot90_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fcma"))]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -1078,7 +1078,7 @@ pub fn vcaddq_rot90_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcadd_rot90_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcadd))]
@@ -1094,7 +1094,7 @@ pub fn vcadd_rot90_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot90_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcadd))]
@@ -1110,7 +1110,7 @@ pub fn vcaddq_rot90_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point complex add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaddq_rot90_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcadd))]
@@ -1126,7 +1126,7 @@ pub fn vcaddq_rot90_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcage_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1142,7 +1142,7 @@ pub fn vcage_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcageq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1158,7 +1158,7 @@ pub fn vcageq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaged_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1174,7 +1174,7 @@ pub fn vcaged_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcages_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1190,7 +1190,7 @@ pub fn vcages_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcageh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(facge))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1207,7 +1207,7 @@ pub fn vcageh_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagt_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1223,7 +1223,7 @@ pub fn vcagt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1239,7 +1239,7 @@ pub fn vcagtq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1255,7 +1255,7 @@ pub fn vcagtd_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagts_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1271,7 +1271,7 @@ pub fn vcagts_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagth_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(facgt))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1288,7 +1288,7 @@ pub fn vcagth_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcale_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1297,7 +1297,7 @@ pub fn vcale_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaleq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1306,7 +1306,7 @@ pub fn vcaleq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaled_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1315,7 +1315,7 @@ pub fn vcaled_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcales_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1324,7 +1324,7 @@ pub fn vcales_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaleh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(facge))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1334,7 +1334,7 @@ pub fn vcaleh_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalt_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1343,7 +1343,7 @@ pub fn vcalt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1352,7 +1352,7 @@ pub fn vcaltq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1361,7 +1361,7 @@ pub fn vcaltd_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalts_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(facgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1370,7 +1370,7 @@ pub fn vcalts_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalth_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(facgt))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1380,7 +1380,7 @@ pub fn vcalth_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1389,7 +1389,7 @@ pub fn vceq_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1398,7 +1398,7 @@ pub fn vceqq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1407,7 +1407,7 @@ pub fn vceq_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1416,7 +1416,7 @@ pub fn vceqq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1425,7 +1425,7 @@ pub fn vceq_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1434,7 +1434,7 @@ pub fn vceqq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1443,7 +1443,7 @@ pub fn vceq_p64(a: poly64x1_t, b: poly64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1452,7 +1452,7 @@ pub fn vceqq_p64(a: poly64x2_t, b: poly64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1461,7 +1461,7 @@ pub fn vceqd_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1470,7 +1470,7 @@ pub fn vceqs_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Compare bitwise equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1479,7 +1479,7 @@ pub fn vceqd_s64(a: i64, b: i64) -> u64 {
 }
 #[doc = "Compare bitwise equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1488,7 +1488,7 @@ pub fn vceqd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1498,7 +1498,7 @@ pub fn vceqh_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -1509,7 +1509,7 @@ pub fn vceqz_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -1520,7 +1520,7 @@ pub fn vceqzq_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1530,7 +1530,7 @@ pub fn vceqz_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1540,7 +1540,7 @@ pub fn vceqzq_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1550,7 +1550,7 @@ pub fn vceqz_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1560,7 +1560,7 @@ pub fn vceqzq_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1570,7 +1570,7 @@ pub fn vceqz_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1580,7 +1580,7 @@ pub fn vceqzq_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1590,7 +1590,7 @@ pub fn vceqz_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1600,7 +1600,7 @@ pub fn vceqzq_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1610,7 +1610,7 @@ pub fn vceqz_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1620,7 +1620,7 @@ pub fn vceqzq_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1630,7 +1630,7 @@ pub fn vceqz_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1640,7 +1640,7 @@ pub fn vceqzq_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1650,7 +1650,7 @@ pub fn vceqz_p8(a: poly8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1660,7 +1660,7 @@ pub fn vceqzq_p8(a: poly8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1670,7 +1670,7 @@ pub fn vceqz_p64(a: poly64x1_t) -> uint64x1_t {
 }
 #[doc = "Signed compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1680,7 +1680,7 @@ pub fn vceqzq_p64(a: poly64x2_t) -> uint64x2_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1690,7 +1690,7 @@ pub fn vceqz_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1700,7 +1700,7 @@ pub fn vceqzq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1710,7 +1710,7 @@ pub fn vceqz_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1720,7 +1720,7 @@ pub fn vceqzq_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1730,7 +1730,7 @@ pub fn vceqz_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1740,7 +1740,7 @@ pub fn vceqzq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqz_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1750,7 +1750,7 @@ pub fn vceqz_u64(a: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmeq))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1760,7 +1760,7 @@ pub fn vceqzq_u64(a: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1769,7 +1769,7 @@ pub fn vceqzd_s64(a: i64) -> u64 {
 }
 #[doc = "Compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1778,7 +1778,7 @@ pub fn vceqzd_u64(a: u64) -> u64 {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1788,7 +1788,7 @@ pub fn vceqzh_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1797,7 +1797,7 @@ pub fn vceqzs_f32(a: f32) -> u32 {
 }
 #[doc = "Floating-point compare bitwise equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqzd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1806,7 +1806,7 @@ pub fn vceqzd_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1815,7 +1815,7 @@ pub fn vcge_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1824,7 +1824,7 @@ pub fn vcgeq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1833,7 +1833,7 @@ pub fn vcge_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1842,7 +1842,7 @@ pub fn vcgeq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1851,7 +1851,7 @@ pub fn vcge_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1860,7 +1860,7 @@ pub fn vcgeq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcged_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1869,7 +1869,7 @@ pub fn vcged_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcges_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1878,7 +1878,7 @@ pub fn vcges_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcged_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1887,7 +1887,7 @@ pub fn vcged_s64(a: i64, b: i64) -> u64 {
 }
 #[doc = "Compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcged_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1896,7 +1896,7 @@ pub fn vcged_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -1906,7 +1906,7 @@ pub fn vcgeh_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1916,7 +1916,7 @@ pub fn vcgez_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1926,7 +1926,7 @@ pub fn vcgezq_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1936,7 +1936,7 @@ pub fn vcgez_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1946,7 +1946,7 @@ pub fn vcgezq_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1956,7 +1956,7 @@ pub fn vcgez_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1966,7 +1966,7 @@ pub fn vcgezq_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1976,7 +1976,7 @@ pub fn vcgez_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1986,7 +1986,7 @@ pub fn vcgezq_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -1996,7 +1996,7 @@ pub fn vcgez_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2006,7 +2006,7 @@ pub fn vcgezq_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2016,7 +2016,7 @@ pub fn vcgez_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2026,7 +2026,7 @@ pub fn vcgezq_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2035,7 +2035,7 @@ pub fn vcgezd_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2044,7 +2044,7 @@ pub fn vcgezs_f32(a: f32) -> u32 {
 }
 #[doc = "Compare signed greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2053,7 +2053,7 @@ pub fn vcgezd_s64(a: i64) -> u64 {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2063,7 +2063,7 @@ pub fn vcgezh_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2072,7 +2072,7 @@ pub fn vcgt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2081,7 +2081,7 @@ pub fn vcgtq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2090,7 +2090,7 @@ pub fn vcgt_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2099,7 +2099,7 @@ pub fn vcgtq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhi))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2108,7 +2108,7 @@ pub fn vcgt_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhi))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2117,7 +2117,7 @@ pub fn vcgtq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2126,7 +2126,7 @@ pub fn vcgtd_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgts_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2135,7 +2135,7 @@ pub fn vcgts_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2144,7 +2144,7 @@ pub fn vcgtd_s64(a: i64, b: i64) -> u64 {
 }
 #[doc = "Compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2153,7 +2153,7 @@ pub fn vcgtd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgth_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2163,7 +2163,7 @@ pub fn vcgth_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2173,7 +2173,7 @@ pub fn vcgtz_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2183,7 +2183,7 @@ pub fn vcgtzq_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2193,7 +2193,7 @@ pub fn vcgtz_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2203,7 +2203,7 @@ pub fn vcgtzq_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2213,7 +2213,7 @@ pub fn vcgtz_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2223,7 +2223,7 @@ pub fn vcgtzq_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2233,7 +2233,7 @@ pub fn vcgtz_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2243,7 +2243,7 @@ pub fn vcgtzq_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2253,7 +2253,7 @@ pub fn vcgtz_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2263,7 +2263,7 @@ pub fn vcgtzq_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2273,7 +2273,7 @@ pub fn vcgtz_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2283,7 +2283,7 @@ pub fn vcgtzq_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2292,7 +2292,7 @@ pub fn vcgtzd_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2301,7 +2301,7 @@ pub fn vcgtzs_f32(a: f32) -> u32 {
 }
 #[doc = "Compare signed greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2310,7 +2310,7 @@ pub fn vcgtzd_s64(a: i64) -> u64 {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2320,7 +2320,7 @@ pub fn vcgtzh_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2329,7 +2329,7 @@ pub fn vcle_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2338,7 +2338,7 @@ pub fn vcleq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2347,7 +2347,7 @@ pub fn vcle_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmge))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2356,7 +2356,7 @@ pub fn vcleq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2365,7 +2365,7 @@ pub fn vcle_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2374,7 +2374,7 @@ pub fn vcleq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcled_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2383,7 +2383,7 @@ pub fn vcled_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcles_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2392,7 +2392,7 @@ pub fn vcles_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcled_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2401,7 +2401,7 @@ pub fn vcled_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcled_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2410,7 +2410,7 @@ pub fn vcled_s64(a: i64, b: i64) -> u64 {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2420,7 +2420,7 @@ pub fn vcleh_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2430,7 +2430,7 @@ pub fn vclez_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2440,7 +2440,7 @@ pub fn vclezq_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2450,7 +2450,7 @@ pub fn vclez_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2460,7 +2460,7 @@ pub fn vclezq_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2470,7 +2470,7 @@ pub fn vclez_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2480,7 +2480,7 @@ pub fn vclezq_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2490,7 +2490,7 @@ pub fn vclez_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2500,7 +2500,7 @@ pub fn vclezq_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2510,7 +2510,7 @@ pub fn vclez_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2520,7 +2520,7 @@ pub fn vclezq_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2530,7 +2530,7 @@ pub fn vclez_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmle))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2540,7 +2540,7 @@ pub fn vclezq_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2549,7 +2549,7 @@ pub fn vclezd_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2558,7 +2558,7 @@ pub fn vclezs_f32(a: f32) -> u32 {
 }
 #[doc = "Compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2567,7 +2567,7 @@ pub fn vclezd_s64(a: i64) -> u64 {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2577,7 +2577,7 @@ pub fn vclezh_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2586,7 +2586,7 @@ pub fn vclt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2595,7 +2595,7 @@ pub fn vcltq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2604,7 +2604,7 @@ pub fn vclt_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmgt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2613,7 +2613,7 @@ pub fn vcltq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhi))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2622,7 +2622,7 @@ pub fn vclt_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmhi))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2631,7 +2631,7 @@ pub fn vcltq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2640,7 +2640,7 @@ pub fn vcltd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2649,7 +2649,7 @@ pub fn vcltd_s64(a: i64, b: i64) -> u64 {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclth_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2659,7 +2659,7 @@ pub fn vclth_f16(a: f16, b: f16) -> u16 {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclts_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2668,7 +2668,7 @@ pub fn vclts_f32(a: f32, b: f32) -> u32 {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2677,7 +2677,7 @@ pub fn vcltd_f64(a: f64, b: f64) -> u64 {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2687,7 +2687,7 @@ pub fn vcltz_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2697,7 +2697,7 @@ pub fn vcltzq_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2707,7 +2707,7 @@ pub fn vcltz_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2717,7 +2717,7 @@ pub fn vcltzq_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2727,7 +2727,7 @@ pub fn vcltz_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2737,7 +2737,7 @@ pub fn vcltzq_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2747,7 +2747,7 @@ pub fn vcltz_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2757,7 +2757,7 @@ pub fn vcltzq_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2767,7 +2767,7 @@ pub fn vcltz_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2777,7 +2777,7 @@ pub fn vcltzq_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2787,7 +2787,7 @@ pub fn vcltz_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Compare signed less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmlt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2797,7 +2797,7 @@ pub fn vcltzq_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2806,7 +2806,7 @@ pub fn vcltzd_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2815,7 +2815,7 @@ pub fn vcltzs_f32(a: f32) -> u32 {
 }
 #[doc = "Compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(asr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -2824,7 +2824,7 @@ pub fn vcltzd_s64(a: i64) -> u64 {
 }
 #[doc = "Floating-point compare less than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcmp))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -2834,7 +2834,7 @@ pub fn vcltzh_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -2852,7 +2852,7 @@ pub fn vcmla_f16(a: float16x4_t, b: float16x4_t, c: float16x4_t) -> float16x4_t
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -2870,7 +2870,7 @@ pub fn vcmlaq_f16(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> float16x8_t
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -2886,7 +2886,7 @@ pub fn vcmla_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -2902,7 +2902,7 @@ pub fn vcmlaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -2918,7 +2918,7 @@ pub fn vcmlaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -2947,7 +2947,7 @@ pub fn vcmla_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -2980,7 +2980,7 @@ pub fn vcmlaq_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -2998,7 +2998,7 @@ pub fn vcmla_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3025,7 +3025,7 @@ pub fn vcmlaq_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3054,7 +3054,7 @@ pub fn vcmla_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3087,7 +3087,7 @@ pub fn vcmlaq_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3105,7 +3105,7 @@ pub fn vcmla_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3132,7 +3132,7 @@ pub fn vcmlaq_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -3150,7 +3150,7 @@ pub fn vcmla_rot180_f16(a: float16x4_t, b: float16x4_t, c: float16x4_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -3168,7 +3168,7 @@ pub fn vcmlaq_rot180_f16(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> floa
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3184,7 +3184,7 @@ pub fn vcmla_rot180_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3200,7 +3200,7 @@ pub fn vcmlaq_rot180_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> floa
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3216,7 +3216,7 @@ pub fn vcmlaq_rot180_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> floa
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3245,7 +3245,7 @@ pub fn vcmla_rot180_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3278,7 +3278,7 @@ pub fn vcmlaq_rot180_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3296,7 +3296,7 @@ pub fn vcmla_rot180_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3323,7 +3323,7 @@ pub fn vcmlaq_rot180_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3352,7 +3352,7 @@ pub fn vcmla_rot180_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3385,7 +3385,7 @@ pub fn vcmlaq_rot180_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot180_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3403,7 +3403,7 @@ pub fn vcmla_rot180_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot180_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3430,7 +3430,7 @@ pub fn vcmlaq_rot180_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -3448,7 +3448,7 @@ pub fn vcmla_rot270_f16(a: float16x4_t, b: float16x4_t, c: float16x4_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -3466,7 +3466,7 @@ pub fn vcmlaq_rot270_f16(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> floa
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3482,7 +3482,7 @@ pub fn vcmla_rot270_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3498,7 +3498,7 @@ pub fn vcmlaq_rot270_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> floa
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3514,7 +3514,7 @@ pub fn vcmlaq_rot270_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> floa
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3543,7 +3543,7 @@ pub fn vcmla_rot270_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3576,7 +3576,7 @@ pub fn vcmlaq_rot270_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3594,7 +3594,7 @@ pub fn vcmla_rot270_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3621,7 +3621,7 @@ pub fn vcmlaq_rot270_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3650,7 +3650,7 @@ pub fn vcmla_rot270_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3683,7 +3683,7 @@ pub fn vcmlaq_rot270_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot270_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3701,7 +3701,7 @@ pub fn vcmla_rot270_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot270_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3728,7 +3728,7 @@ pub fn vcmlaq_rot270_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -3746,7 +3746,7 @@ pub fn vcmla_rot90_f16(a: float16x4_t, b: float16x4_t, c: float16x4_t) -> float1
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
@@ -3764,7 +3764,7 @@ pub fn vcmlaq_rot90_f16(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3780,7 +3780,7 @@ pub fn vcmla_rot90_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float3
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3796,7 +3796,7 @@ pub fn vcmlaq_rot90_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[unstable(feature = "stdarch_neon_fcma", issue = "117222")]
 #[cfg_attr(test, assert_instr(fcmla))]
@@ -3812,7 +3812,7 @@ pub fn vcmlaq_rot90_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3841,7 +3841,7 @@ pub fn vcmla_rot90_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3874,7 +3874,7 @@ pub fn vcmlaq_rot90_lane_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3892,7 +3892,7 @@ pub fn vcmla_rot90_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3919,7 +3919,7 @@ pub fn vcmlaq_rot90_lane_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3948,7 +3948,7 @@ pub fn vcmla_rot90_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3981,7 +3981,7 @@ pub fn vcmlaq_rot90_laneq_f16(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmla_rot90_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -3999,7 +3999,7 @@ pub fn vcmla_rot90_laneq_f32(
 }
 #[doc = "Floating-point complex multiply accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcmlaq_rot90_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fcma")]
 #[cfg_attr(test, assert_instr(fcmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -4026,7 +4026,7 @@ pub fn vcmlaq_rot90_laneq_f32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4047,7 +4047,7 @@ pub fn vcopy_lane_f32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4071,7 +4071,7 @@ pub fn vcopy_lane_s8(a: int8x8_t, b: int8x8_
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4091,7 +4091,7 @@ pub fn vcopy_lane_s16(a: int16x4_t, b: int16
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4109,7 +4109,7 @@ pub fn vcopy_lane_s32(a: int32x2_t, b: int32
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4133,7 +4133,7 @@ pub fn vcopy_lane_u8(a: uint8x8_t, b: uint8x
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4156,7 +4156,7 @@ pub fn vcopy_lane_u16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4177,7 +4177,7 @@ pub fn vcopy_lane_u32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4201,7 +4201,7 @@ pub fn vcopy_lane_p8(a: poly8x8_t, b: poly8x
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4224,7 +4224,7 @@ pub fn vcopy_lane_p16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4246,7 +4246,7 @@ pub fn vcopy_laneq_f32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4272,7 +4272,7 @@ pub fn vcopy_laneq_s8(a: int8x8_t, b: int8x1
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4296,7 +4296,7 @@ pub fn vcopy_laneq_s16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4318,7 +4318,7 @@ pub fn vcopy_laneq_s32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4347,7 +4347,7 @@ pub fn vcopy_laneq_u8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4371,7 +4371,7 @@ pub fn vcopy_laneq_u16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4393,7 +4393,7 @@ pub fn vcopy_laneq_u32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4422,7 +4422,7 @@ pub fn vcopy_laneq_p8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopy_laneq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4446,7 +4446,7 @@ pub fn vcopy_laneq_p16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4470,7 +4470,7 @@ pub fn vcopyq_lane_f32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4492,7 +4492,7 @@ pub fn vcopyq_lane_f64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4514,7 +4514,7 @@ pub fn vcopyq_lane_s64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4536,7 +4536,7 @@ pub fn vcopyq_lane_u64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4558,7 +4558,7 @@ pub fn vcopyq_lane_p64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4928,7 +4928,7 @@ pub fn vcopyq_lane_s8(a: int8x16_t, b: int8x
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4956,7 +4956,7 @@ pub fn vcopyq_lane_s16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -4980,7 +4980,7 @@ pub fn vcopyq_lane_s32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5353,7 +5353,7 @@ pub fn vcopyq_lane_u8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5381,7 +5381,7 @@ pub fn vcopyq_lane_u16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5405,7 +5405,7 @@ pub fn vcopyq_lane_u32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5778,7 +5778,7 @@ pub fn vcopyq_lane_p8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5806,7 +5806,7 @@ pub fn vcopyq_lane_p16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5829,7 +5829,7 @@ pub fn vcopyq_laneq_f32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -5850,7 +5850,7 @@ pub fn vcopyq_laneq_f64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6221,7 +6221,7 @@ pub fn vcopyq_laneq_s8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6248,7 +6248,7 @@ pub fn vcopyq_laneq_s16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6271,7 +6271,7 @@ pub fn vcopyq_laneq_s32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6292,7 +6292,7 @@ pub fn vcopyq_laneq_s64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6663,7 +6663,7 @@ pub fn vcopyq_laneq_u8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6690,7 +6690,7 @@ pub fn vcopyq_laneq_u16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6713,7 +6713,7 @@ pub fn vcopyq_laneq_u32(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -6734,7 +6734,7 @@ pub fn vcopyq_laneq_u64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -7105,7 +7105,7 @@ pub fn vcopyq_laneq_p8(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -7132,7 +7132,7 @@ pub fn vcopyq_laneq_p16(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcopyq_laneq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))]
 #[rustc_legacy_const_generics(1, 3)]
@@ -7153,7 +7153,7 @@ pub fn vcopyq_laneq_p64(
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7162,7 +7162,7 @@ pub fn vcreate_f64(a: u64) -> float64x1_t {
 }
 #[doc = "Floating-point convert"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7171,7 +7171,7 @@ pub fn vcvt_f32_f64(a: float64x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point convert to higher precision long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f64_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7180,7 +7180,7 @@ pub fn vcvt_f64_f32(a: float32x2_t) -> float64x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7189,7 +7189,7 @@ pub fn vcvt_f64_s64(a: int64x1_t) -> float64x1_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7198,7 +7198,7 @@ pub fn vcvtq_f64_s64(a: int64x2_t) -> float64x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7207,7 +7207,7 @@ pub fn vcvt_f64_u64(a: uint64x1_t) -> float64x1_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7216,7 +7216,7 @@ pub fn vcvtq_f64_u64(a: uint64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point convert to lower precision"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_high_f16_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtn2))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -7226,7 +7226,7 @@ pub fn vcvt_high_f16_f32(a: float16x4_t, b: float32x4_t) -> float16x8_t {
 }
 #[doc = "Floating-point convert to higher precision"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_high_f32_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtl2))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -7236,7 +7236,7 @@ pub fn vcvt_high_f32_f16(a: float16x8_t) -> float32x4_t {
 }
 #[doc = "Floating-point convert to lower precision narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_high_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7245,7 +7245,7 @@ pub fn vcvt_high_f32_f64(a: float32x2_t, b: float64x2_t) -> float32x4_t {
 }
 #[doc = "Floating-point convert to higher precision long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_high_f64_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7257,7 +7257,7 @@ pub fn vcvt_high_f64_f32(a: float32x4_t) -> float64x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7275,7 +7275,7 @@ pub fn vcvt_n_f64_s64(a: int64x1_t) -> float64x1_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7293,7 +7293,7 @@ pub fn vcvtq_n_f64_s64(a: int64x2_t) -> float64x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7311,7 +7311,7 @@ pub fn vcvt_n_f64_u64(a: uint64x1_t) -> float64x1_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7329,7 +7329,7 @@ pub fn vcvtq_n_f64_u64(a: uint64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7347,7 +7347,7 @@ pub fn vcvt_n_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7365,7 +7365,7 @@ pub fn vcvtq_n_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7383,7 +7383,7 @@ pub fn vcvt_n_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -7401,7 +7401,7 @@ pub fn vcvtq_n_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7417,7 +7417,7 @@ pub fn vcvt_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Floating-point convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7433,7 +7433,7 @@ pub fn vcvtq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Floating-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7449,7 +7449,7 @@ pub fn vcvt_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7465,7 +7465,7 @@ pub fn vcvtq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -7482,7 +7482,7 @@ pub fn vcvta_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -7499,7 +7499,7 @@ pub fn vcvtaq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7515,7 +7515,7 @@ pub fn vcvta_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7531,7 +7531,7 @@ pub fn vcvtaq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7547,7 +7547,7 @@ pub fn vcvta_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7563,7 +7563,7 @@ pub fn vcvtaq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -7580,7 +7580,7 @@ pub fn vcvta_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -7597,7 +7597,7 @@ pub fn vcvtaq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7613,7 +7613,7 @@ pub fn vcvta_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7629,7 +7629,7 @@ pub fn vcvtaq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvta_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7645,7 +7645,7 @@ pub fn vcvta_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtaq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7661,7 +7661,7 @@ pub fn vcvtaq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7671,7 +7671,7 @@ pub fn vcvtah_s16_f16(a: f16) -> i16 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7688,7 +7688,7 @@ pub fn vcvtah_s32_f16(a: f16) -> i32 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7705,7 +7705,7 @@ pub fn vcvtah_s64_f16(a: f16) -> i64 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7715,7 +7715,7 @@ pub fn vcvtah_u16_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7732,7 +7732,7 @@ pub fn vcvtah_u32_f16(a: f16) -> u32 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtah_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7749,7 +7749,7 @@ pub fn vcvtah_u64_f16(a: f16) -> u64 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtas_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7765,7 +7765,7 @@ pub fn vcvtas_s32_f32(a: f32) -> i32 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtad_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtas))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7781,7 +7781,7 @@ pub fn vcvtad_s64_f64(a: f64) -> i64 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtas_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7797,7 +7797,7 @@ pub fn vcvtas_u32_f32(a: f32) -> u32 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtad_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtau))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7813,7 +7813,7 @@ pub fn vcvtad_u64_f64(a: f64) -> u64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7822,7 +7822,7 @@ pub fn vcvtd_f64_s64(a: i64) -> f64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -7831,7 +7831,7 @@ pub fn vcvts_f32_s32(a: i32) -> f32 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7841,7 +7841,7 @@ pub fn vcvth_f16_s16(a: i16) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_f16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7851,7 +7851,7 @@ pub fn vcvth_f16_s32(a: i32) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_f16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(scvtf))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7861,7 +7861,7 @@ pub fn vcvth_f16_s64(a: i64) -> f16 {
 }
 #[doc = "Unsigned fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7871,7 +7871,7 @@ pub fn vcvth_f16_u16(a: u16) -> f16 {
 }
 #[doc = "Unsigned fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_f16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7881,7 +7881,7 @@ pub fn vcvth_f16_u32(a: u32) -> f16 {
 }
 #[doc = "Unsigned fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_f16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -7891,7 +7891,7 @@ pub fn vcvth_f16_u64(a: u64) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -7903,7 +7903,7 @@ pub fn vcvth_n_f16_s16(a: i16) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_f16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -7922,7 +7922,7 @@ pub fn vcvth_n_f16_s32(a: i32) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_f16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -7941,7 +7941,7 @@ pub fn vcvth_n_f16_s64(a: i64) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -7953,7 +7953,7 @@ pub fn vcvth_n_f16_u16(a: u16) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_f16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -7972,7 +7972,7 @@ pub fn vcvth_n_f16_u32(a: u32) -> f16 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_f16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -7991,7 +7991,7 @@ pub fn vcvth_n_f16_u64(a: u64) -> f16 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -8003,7 +8003,7 @@ pub fn vcvth_n_s16_f16(a: f16) -> i16 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -8022,7 +8022,7 @@ pub fn vcvth_n_s32_f16(a: f16) -> i32 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -8041,7 +8041,7 @@ pub fn vcvth_n_s64_f16(a: f16) -> i64 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -8053,7 +8053,7 @@ pub fn vcvth_n_u16_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -8072,7 +8072,7 @@ pub fn vcvth_n_u32_f16(a: f16) -> u32 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_n_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -8091,7 +8091,7 @@ pub fn vcvth_n_u64_f16(a: f16) -> u64 {
 }
 #[doc = "Floating-point convert to signed fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8101,7 +8101,7 @@ pub fn vcvth_s16_f16(a: f16) -> i16 {
 }
 #[doc = "Floating-point convert to signed fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8111,7 +8111,7 @@ pub fn vcvth_s32_f16(a: f16) -> i32 {
 }
 #[doc = "Floating-point convert to signed fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8121,7 +8121,7 @@ pub fn vcvth_s64_f16(a: f16) -> i64 {
 }
 #[doc = "Floating-point convert to unsigned fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8131,7 +8131,7 @@ pub fn vcvth_u16_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point convert to unsigned fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8141,7 +8141,7 @@ pub fn vcvth_u32_f16(a: f16) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvth_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8151,7 +8151,7 @@ pub fn vcvth_u64_f16(a: f16) -> u64 {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8168,7 +8168,7 @@ pub fn vcvtm_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8185,7 +8185,7 @@ pub fn vcvtmq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8201,7 +8201,7 @@ pub fn vcvtm_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8217,7 +8217,7 @@ pub fn vcvtmq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8233,7 +8233,7 @@ pub fn vcvtm_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8249,7 +8249,7 @@ pub fn vcvtmq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8266,7 +8266,7 @@ pub fn vcvtm_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8283,7 +8283,7 @@ pub fn vcvtmq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8299,7 +8299,7 @@ pub fn vcvtm_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8315,7 +8315,7 @@ pub fn vcvtmq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtm_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8331,7 +8331,7 @@ pub fn vcvtm_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8347,7 +8347,7 @@ pub fn vcvtmq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point convert to integer, rounding towards minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmh_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8357,7 +8357,7 @@ pub fn vcvtmh_s16_f16(a: f16) -> i16 {
 }
 #[doc = "Floating-point convert to integer, rounding towards minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmh_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8374,7 +8374,7 @@ pub fn vcvtmh_s32_f16(a: f16) -> i32 {
 }
 #[doc = "Floating-point convert to integer, rounding towards minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmh_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8391,7 +8391,7 @@ pub fn vcvtmh_s64_f16(a: f16) -> i64 {
 }
 #[doc = "Floating-point convert to integer, rounding towards minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmh_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8401,7 +8401,7 @@ pub fn vcvtmh_u16_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding towards minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmh_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8418,7 +8418,7 @@ pub fn vcvtmh_u32_f16(a: f16) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding towards minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmh_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8435,7 +8435,7 @@ pub fn vcvtmh_u64_f16(a: f16) -> u64 {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtms_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8451,7 +8451,7 @@ pub fn vcvtms_s32_f32(a: f32) -> i32 {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmd_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtms))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8467,7 +8467,7 @@ pub fn vcvtmd_s64_f64(a: f64) -> i64 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtms_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8483,7 +8483,7 @@ pub fn vcvtms_u32_f32(a: f32) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtmd_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtmu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8499,7 +8499,7 @@ pub fn vcvtmd_u64_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8516,7 +8516,7 @@ pub fn vcvtn_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8533,7 +8533,7 @@ pub fn vcvtnq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8549,7 +8549,7 @@ pub fn vcvtn_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8565,7 +8565,7 @@ pub fn vcvtnq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8581,7 +8581,7 @@ pub fn vcvtn_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8597,7 +8597,7 @@ pub fn vcvtnq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8614,7 +8614,7 @@ pub fn vcvtn_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8631,7 +8631,7 @@ pub fn vcvtnq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8647,7 +8647,7 @@ pub fn vcvtn_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8663,7 +8663,7 @@ pub fn vcvtnq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtn_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8679,7 +8679,7 @@ pub fn vcvtn_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8695,7 +8695,7 @@ pub fn vcvtnq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnh_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8705,7 +8705,7 @@ pub fn vcvtnh_s16_f16(a: f16) -> i16 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnh_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8722,7 +8722,7 @@ pub fn vcvtnh_s32_f16(a: f16) -> i32 {
 }
 #[doc = "Floating-point convert to integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnh_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8739,7 +8739,7 @@ pub fn vcvtnh_s64_f16(a: f16) -> i64 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnh_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8749,7 +8749,7 @@ pub fn vcvtnh_u16_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnh_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8766,7 +8766,7 @@ pub fn vcvtnh_u32_f16(a: f16) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnh_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -8783,7 +8783,7 @@ pub fn vcvtnh_u64_f16(a: f16) -> u64 {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtns_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8799,7 +8799,7 @@ pub fn vcvtns_s32_f32(a: f32) -> i32 {
 }
 #[doc = "Floating-point convert to signed integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnd_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtns))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8815,7 +8815,7 @@ pub fn vcvtnd_s64_f64(a: f64) -> i64 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtns_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8831,7 +8831,7 @@ pub fn vcvtns_u32_f32(a: f32) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtnd_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtnu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8847,7 +8847,7 @@ pub fn vcvtnd_u64_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point convert to signed integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8864,7 +8864,7 @@ pub fn vcvtp_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8881,7 +8881,7 @@ pub fn vcvtpq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8897,7 +8897,7 @@ pub fn vcvtp_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8913,7 +8913,7 @@ pub fn vcvtpq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8929,7 +8929,7 @@ pub fn vcvtp_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8945,7 +8945,7 @@ pub fn vcvtpq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8962,7 +8962,7 @@ pub fn vcvtp_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -8979,7 +8979,7 @@ pub fn vcvtpq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -8995,7 +8995,7 @@ pub fn vcvtp_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9011,7 +9011,7 @@ pub fn vcvtpq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtp_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9027,7 +9027,7 @@ pub fn vcvtp_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9043,7 +9043,7 @@ pub fn vcvtpq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point convert to integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtph_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -9053,7 +9053,7 @@ pub fn vcvtph_s16_f16(a: f16) -> i16 {
 }
 #[doc = "Floating-point convert to integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtph_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -9070,7 +9070,7 @@ pub fn vcvtph_s32_f16(a: f16) -> i32 {
 }
 #[doc = "Floating-point convert to integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtph_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -9087,7 +9087,7 @@ pub fn vcvtph_s64_f16(a: f16) -> i64 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtph_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -9097,7 +9097,7 @@ pub fn vcvtph_u16_f16(a: f16) -> u16 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtph_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -9114,7 +9114,7 @@ pub fn vcvtph_u32_f16(a: f16) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding to plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtph_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -9131,7 +9131,7 @@ pub fn vcvtph_u64_f16(a: f16) -> u64 {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtps_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9147,7 +9147,7 @@ pub fn vcvtps_s32_f32(a: f32) -> i32 {
 }
 #[doc = "Floating-point convert to signed integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpd_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9163,7 +9163,7 @@ pub fn vcvtpd_s64_f64(a: f64) -> i64 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtps_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9179,7 +9179,7 @@ pub fn vcvtps_u32_f32(a: f32) -> u32 {
 }
 #[doc = "Floating-point convert to unsigned integer, rounding toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtpd_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtpu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9195,7 +9195,7 @@ pub fn vcvtpd_u64_f64(a: f64) -> u64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9204,7 +9204,7 @@ pub fn vcvts_f32_u32(a: u32) -> f32 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9213,7 +9213,7 @@ pub fn vcvtd_f64_u64(a: u64) -> f64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9231,7 +9231,7 @@ pub fn vcvts_n_f32_s32(a: i32) -> f32 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9249,7 +9249,7 @@ pub fn vcvtd_n_f64_s64(a: i64) -> f64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9267,7 +9267,7 @@ pub fn vcvts_n_f32_u32(a: u32) -> f32 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9285,7 +9285,7 @@ pub fn vcvtd_n_f64_u64(a: u64) -> f64 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9303,7 +9303,7 @@ pub fn vcvts_n_s32_f32(a: f32) -> i32 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9321,7 +9321,7 @@ pub fn vcvtd_n_s64_f64(a: f64) -> i64 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_n_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9339,7 +9339,7 @@ pub fn vcvts_n_u32_f32(a: f32) -> u32 {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_n_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9357,7 +9357,7 @@ pub fn vcvtd_n_u64_f64(a: f64) -> u64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9366,7 +9366,7 @@ pub fn vcvts_s32_f32(a: f32) -> i32 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzs))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9375,7 +9375,7 @@ pub fn vcvtd_s64_f64(a: f64) -> i64 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvts_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9384,7 +9384,7 @@ pub fn vcvts_u32_f32(a: f32) -> u32 {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtd_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtzu))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9393,7 +9393,7 @@ pub fn vcvtd_u64_f64(a: f64) -> u64 {
 }
 #[doc = "Floating-point convert to lower precision narrow, rounding to odd"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtx_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtxn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9409,7 +9409,7 @@ pub fn vcvtx_f32_f64(a: float64x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point convert to lower precision narrow, rounding to odd"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtx_high_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtxn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9418,7 +9418,7 @@ pub fn vcvtx_high_f32_f64(a: float32x2_t, b: float64x2_t) -> float32x4_t {
 }
 #[doc = "Floating-point convert to lower precision narrow, rounding to odd"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtxd_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fcvtxn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -9427,7 +9427,7 @@ pub fn vcvtxd_f32_f64(a: f64) -> f32 {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdiv_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -9437,7 +9437,7 @@ pub fn vdiv_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdivq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -9447,7 +9447,7 @@ pub fn vdivq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdiv_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fdiv))]
@@ -9456,7 +9456,7 @@ pub fn vdiv_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdivq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fdiv))]
@@ -9465,7 +9465,7 @@ pub fn vdivq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdiv_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fdiv))]
@@ -9474,7 +9474,7 @@ pub fn vdiv_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdivq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fdiv))]
@@ -9483,7 +9483,7 @@ pub fn vdivq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Divide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdivh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -9493,7 +9493,7 @@ pub fn vdivh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(test, assert_instr(sdot, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -9508,7 +9508,7 @@ pub fn vdot_laneq_s32(a: int32x2_t, b: int8x8_t, c: int8x16_t)
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(test, assert_instr(sdot, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -9524,7 +9524,7 @@ pub fn vdotq_laneq_s32(a: int32x4_t, b: int8x16_t, c: int8x16_t
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(test, assert_instr(udot, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -9539,7 +9539,7 @@ pub fn vdot_laneq_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x16_
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(test, assert_instr(udot, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -9555,7 +9555,7 @@ pub fn vdotq_laneq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x1
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9566,7 +9566,7 @@ pub fn vdup_lane_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9577,7 +9577,7 @@ pub fn vdup_lane_p64(a: poly64x1_t) -> poly64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9588,7 +9588,7 @@ pub fn vdup_laneq_f64(a: float64x2_t) -> float64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9599,7 +9599,7 @@ pub fn vdup_laneq_p64(a: poly64x2_t) -> poly64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
@@ -9610,7 +9610,7 @@ pub fn vdupb_lane_s8(a: int8x8_t) -> i8 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
@@ -9621,7 +9621,7 @@ pub fn vduph_laneq_s16(a: int16x8_t) -> i16 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
@@ -9632,7 +9632,7 @@ pub fn vdupb_lane_u8(a: uint8x8_t) -> u8 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
@@ -9643,7 +9643,7 @@ pub fn vduph_laneq_u16(a: uint16x8_t) -> u16 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
@@ -9654,7 +9654,7 @@ pub fn vdupb_lane_p8(a: poly8x8_t) -> p8 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
@@ -9665,7 +9665,7 @@ pub fn vduph_laneq_p16(a: poly16x8_t) -> p16 {
 }
 #[doc = "Extract an element from a vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_laneq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 8))]
 #[rustc_legacy_const_generics(1)]
@@ -9676,7 +9676,7 @@ pub fn vdupb_laneq_s8(a: int8x16_t) -> i8 {
 }
 #[doc = "Extract an element from a vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_laneq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 8))]
 #[rustc_legacy_const_generics(1)]
@@ -9687,7 +9687,7 @@ pub fn vdupb_laneq_u8(a: uint8x16_t) -> u8 {
 }
 #[doc = "Extract an element from a vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupb_laneq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 8))]
 #[rustc_legacy_const_generics(1)]
@@ -9698,7 +9698,7 @@ pub fn vdupb_laneq_p8(a: poly8x16_t) -> p8 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9709,7 +9709,7 @@ pub fn vdupd_lane_f64(a: float64x1_t) -> f64 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9720,7 +9720,7 @@ pub fn vdupd_lane_s64(a: int64x1_t) -> i64 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9731,7 +9731,7 @@ pub fn vdupd_lane_u64(a: uint64x1_t) -> u64 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -9743,7 +9743,7 @@ pub fn vduph_lane_f16(a: float16x4_t) -> f16 {
 }
 #[doc = "Extract an element from a vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(nop, N = 4))]
 #[rustc_legacy_const_generics(1)]
 #[target_feature(enable = "neon,fp16")]
@@ -9755,7 +9755,7 @@ pub fn vduph_laneq_f16(a: float16x8_t) -> f16 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(dup, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9766,7 +9766,7 @@ pub fn vdupq_lane_f64(a: float64x1_t) -> float64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(dup, N = 0))]
 #[rustc_legacy_const_generics(1)]
@@ -9777,7 +9777,7 @@ pub fn vdupq_lane_p64(a: poly64x1_t) -> poly64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(dup, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9788,7 +9788,7 @@ pub fn vdupq_laneq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(dup, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9799,7 +9799,7 @@ pub fn vdupq_laneq_p64(a: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9810,7 +9810,7 @@ pub fn vdups_lane_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9821,7 +9821,7 @@ pub fn vdupd_laneq_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9832,7 +9832,7 @@ pub fn vdups_lane_s32(a: int32x2_t) -> i32 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_laneq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9843,7 +9843,7 @@ pub fn vdupd_laneq_s64(a: int64x2_t) -> i64 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9854,7 +9854,7 @@ pub fn vdups_lane_u32(a: uint32x2_t) -> u32 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupd_laneq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 1))]
 #[rustc_legacy_const_generics(1)]
@@ -9865,7 +9865,7 @@ pub fn vdupd_laneq_u64(a: uint64x2_t) -> u64 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9876,7 +9876,7 @@ pub fn vdups_laneq_f32(a: float32x4_t) -> f32 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9887,7 +9887,7 @@ pub fn vduph_lane_s16(a: int16x4_t) -> i16 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9898,7 +9898,7 @@ pub fn vdups_laneq_s32(a: int32x4_t) -> i32 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9909,7 +9909,7 @@ pub fn vduph_lane_u16(a: uint16x4_t) -> u16 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdups_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9920,7 +9920,7 @@ pub fn vdups_laneq_u32(a: uint32x4_t) -> u32 {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vduph_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -9931,7 +9931,7 @@ pub fn vduph_lane_p16(a: poly16x4_t) -> p16 {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -9947,7 +9947,7 @@ pub fn veor3q_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -9963,7 +9963,7 @@ pub fn veor3q_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -9979,7 +9979,7 @@ pub fn veor3q_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -9995,7 +9995,7 @@ pub fn veor3q_s64(a: int64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -10011,7 +10011,7 @@ pub fn veor3q_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -10027,7 +10027,7 @@ pub fn veor3q_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -10043,7 +10043,7 @@ pub fn veor3q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Three-way exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor3q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
 #[cfg_attr(test, assert_instr(eor3))]
@@ -10059,7 +10059,7 @@ pub fn veor3q_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ext, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -10076,7 +10076,7 @@ pub fn vextq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ext, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -10093,7 +10093,7 @@ pub fn vextq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmadd))]
@@ -10102,7 +10102,7 @@ pub fn vfma_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10118,7 +10118,7 @@ pub fn vfma_lane_f16(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10134,7 +10134,7 @@ pub fn vfma_laneq_f16(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10150,7 +10150,7 @@ pub fn vfmaq_lane_f16(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10166,7 +10166,7 @@ pub fn vfmaq_laneq_f16(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10181,7 +10181,7 @@ pub fn vfma_lane_f32(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10196,7 +10196,7 @@ pub fn vfma_laneq_f32(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10211,7 +10211,7 @@ pub fn vfmaq_lane_f32(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10226,7 +10226,7 @@ pub fn vfmaq_laneq_f32(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10241,7 +10241,7 @@ pub fn vfmaq_laneq_f64(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10256,7 +10256,7 @@ pub fn vfma_lane_f64(
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10271,7 +10271,7 @@ pub fn vfma_laneq_f64(
 }
 #[doc = "Floating-point fused Multiply-Subtract from accumulator."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -10281,7 +10281,7 @@ pub fn vfma_n_f16(a: float16x4_t, b: float16x4_t, c: f16) -> float16x4_t {
 }
 #[doc = "Floating-point fused Multiply-Subtract from accumulator."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -10291,7 +10291,7 @@ pub fn vfmaq_n_f16(a: float16x8_t, b: float16x8_t, c: f16) -> float16x8_t {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_n_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmadd))]
@@ -10300,7 +10300,7 @@ pub fn vfma_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmad_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10314,7 +10314,7 @@ pub fn vfmad_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmah_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmadd))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -10324,7 +10324,7 @@ pub fn vfmah_f16(a: f16, b: f16, c: f16) -> f16 {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmah_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10339,7 +10339,7 @@ pub fn vfmah_lane_f16(a: f16, b: f16, v: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmah_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10354,7 +10354,7 @@ pub fn vfmah_laneq_f16(a: f16, b: f16, v: float16x8_t) -> f16 {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmla))]
@@ -10363,7 +10363,7 @@ pub fn vfmaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmla, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10378,7 +10378,7 @@ pub fn vfmaq_lane_f64(
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_n_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmla))]
@@ -10387,7 +10387,7 @@ pub fn vfmaq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmas_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10401,7 +10401,7 @@ pub fn vfmas_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmas_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10415,7 +10415,7 @@ pub fn vfmas_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point fused multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmad_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmadd, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10429,7 +10429,7 @@ pub fn vfmad_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_high_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10447,7 +10447,7 @@ pub fn vfmlal_high_f16(r: float32x2_t, a: float16x4_t, b: float16x4_t) -> float3
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlalq_high_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10465,7 +10465,7 @@ pub fn vfmlalq_high_f16(r: float32x4_t, a: float16x8_t, b: float16x8_t) -> float
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_lane_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10482,7 +10482,7 @@ pub fn vfmlal_lane_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_laneq_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10499,7 +10499,7 @@ pub fn vfmlal_laneq_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlalq_lane_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10516,7 +10516,7 @@ pub fn vfmlalq_lane_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlalq_laneq_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10533,7 +10533,7 @@ pub fn vfmlalq_laneq_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_lane_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10550,7 +10550,7 @@ pub fn vfmlal_lane_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_laneq_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10567,7 +10567,7 @@ pub fn vfmlal_laneq_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlalq_lane_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10584,7 +10584,7 @@ pub fn vfmlalq_lane_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlalq_laneq_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlal, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10601,7 +10601,7 @@ pub fn vfmlalq_laneq_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_low_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10619,7 +10619,7 @@ pub fn vfmlal_low_f16(r: float32x2_t, a: float16x4_t, b: float16x4_t) -> float32
 }
 #[doc = "Floating-point fused Multiply-Add Long to accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlalq_low_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10637,7 +10637,7 @@ pub fn vfmlalq_low_f16(r: float32x4_t, a: float16x8_t, b: float16x8_t) -> float3
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlsl_high_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10655,7 +10655,7 @@ pub fn vfmlsl_high_f16(r: float32x2_t, a: float16x4_t, b: float16x4_t) -> float3
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlslq_high_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10673,7 +10673,7 @@ pub fn vfmlslq_high_f16(r: float32x4_t, a: float16x8_t, b: float16x8_t) -> float
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlsl_lane_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10690,7 +10690,7 @@ pub fn vfmlsl_lane_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlsl_laneq_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10707,7 +10707,7 @@ pub fn vfmlsl_laneq_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlslq_lane_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10724,7 +10724,7 @@ pub fn vfmlslq_lane_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlslq_laneq_high_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl2, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10741,7 +10741,7 @@ pub fn vfmlslq_laneq_high_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlsl_lane_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10758,7 +10758,7 @@ pub fn vfmlsl_lane_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlsl_laneq_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10775,7 +10775,7 @@ pub fn vfmlsl_laneq_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlslq_lane_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10792,7 +10792,7 @@ pub fn vfmlslq_lane_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (by element)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlslq_laneq_low_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmlsl, LANE = 0))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
@@ -10809,7 +10809,7 @@ pub fn vfmlslq_laneq_low_f16(
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlsl_low_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10827,7 +10827,7 @@ pub fn vfmlsl_low_f16(r: float32x2_t, a: float16x4_t, b: float16x4_t) -> float32
 }
 #[doc = "Floating-point fused Multiply-Subtract Long from accumulator (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlslq_low_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(not(target_arch = "arm"), target_feature(enable = "fhm"))]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -10845,7 +10845,7 @@ pub fn vfmlslq_low_f16(r: float32x4_t, a: float16x8_t, b: float16x8_t) -> float3
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -10857,7 +10857,7 @@ pub fn vfms_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10873,7 +10873,7 @@ pub fn vfms_lane_f16(
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10889,7 +10889,7 @@ pub fn vfms_laneq_f16(
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10905,7 +10905,7 @@ pub fn vfmsq_lane_f16(
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -10921,7 +10921,7 @@ pub fn vfmsq_laneq_f16(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10936,7 +10936,7 @@ pub fn vfms_lane_f32(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10951,7 +10951,7 @@ pub fn vfms_laneq_f32(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10966,7 +10966,7 @@ pub fn vfmsq_lane_f32(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10981,7 +10981,7 @@ pub fn vfmsq_laneq_f32(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -10996,7 +10996,7 @@ pub fn vfmsq_laneq_f64(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11011,7 +11011,7 @@ pub fn vfms_lane_f64(
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11026,7 +11026,7 @@ pub fn vfms_laneq_f64(
 }
 #[doc = "Floating-point fused Multiply-Subtract from accumulator."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -11036,7 +11036,7 @@ pub fn vfms_n_f16(a: float16x4_t, b: float16x4_t, c: f16) -> float16x4_t {
 }
 #[doc = "Floating-point fused Multiply-Subtract from accumulator."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -11046,7 +11046,7 @@ pub fn vfmsq_n_f16(a: float16x8_t, b: float16x8_t, c: f16) -> float16x8_t {
 }
 #[doc = "Floating-point fused Multiply-subtract to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_n_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11055,7 +11055,7 @@ pub fn vfms_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmsub))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -11065,7 +11065,7 @@ pub fn vfmsh_f16(a: f16, b: f16, c: f16) -> f16 {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsh_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -11080,7 +11080,7 @@ pub fn vfmsh_lane_f16(a: f16, b: f16, v: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsh_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
 #[target_feature(enable = "neon,fp16")]
@@ -11095,7 +11095,7 @@ pub fn vfmsh_laneq_f16(a: f16, b: f16, v: float16x8_t) -> f16 {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11107,7 +11107,7 @@ pub fn vfmsq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11122,7 +11122,7 @@ pub fn vfmsq_lane_f64(
 }
 #[doc = "Floating-point fused Multiply-subtract to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_n_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmls))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11131,7 +11131,7 @@ pub fn vfmsq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t {
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmss_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11141,7 +11141,7 @@ pub fn vfmss_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmss_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11151,7 +11151,7 @@ pub fn vfmss_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsd_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11161,7 +11161,7 @@ pub fn vfmsd_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 {
 }
 #[doc = "Floating-point fused multiply-subtract to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsd_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmsub, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -11173,7 +11173,7 @@ pub fn vfmsd_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -11185,7 +11185,7 @@ pub unsafe fn vld1_f16(ptr: *const f16) -> float16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -11197,7 +11197,7 @@ pub unsafe fn vld1q_f16(ptr: *const f16) -> float16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11208,7 +11208,7 @@ pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11219,7 +11219,7 @@ pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11230,7 +11230,7 @@ pub unsafe fn vld1_f64(ptr: *const f64) -> float64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11241,7 +11241,7 @@ pub unsafe fn vld1q_f64(ptr: *const f64) -> float64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11252,7 +11252,7 @@ pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11263,7 +11263,7 @@ pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11274,7 +11274,7 @@ pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11285,7 +11285,7 @@ pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11296,7 +11296,7 @@ pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11307,7 +11307,7 @@ pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11318,7 +11318,7 @@ pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11329,7 +11329,7 @@ pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11340,7 +11340,7 @@ pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11351,7 +11351,7 @@ pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11362,7 +11362,7 @@ pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11373,7 +11373,7 @@ pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11384,7 +11384,7 @@ pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11395,7 +11395,7 @@ pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11406,7 +11406,7 @@ pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11417,7 +11417,7 @@ pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11428,7 +11428,7 @@ pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11439,7 +11439,7 @@ pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11450,7 +11450,7 @@ pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11461,7 +11461,7 @@ pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11472,7 +11472,7 @@ pub unsafe fn vld1_p64(ptr: *const p64) -> poly64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ldr))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11483,7 +11483,7 @@ pub unsafe fn vld1q_p64(ptr: *const p64) -> poly64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld1))]
@@ -11501,7 +11501,7 @@ pub unsafe fn vld1_f64_x2(a: *const f64) -> float64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld1))]
@@ -11519,7 +11519,7 @@ pub unsafe fn vld1_f64_x3(a: *const f64) -> float64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld1))]
@@ -11537,7 +11537,7 @@ pub unsafe fn vld1_f64_x4(a: *const f64) -> float64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld1))]
@@ -11555,7 +11555,7 @@ pub unsafe fn vld1q_f64_x2(a: *const f64) -> float64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld1))]
@@ -11573,7 +11573,7 @@ pub unsafe fn vld1q_f64_x3(a: *const f64) -> float64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld1))]
@@ -11591,7 +11591,7 @@ pub unsafe fn vld1q_f64_x4(a: *const f64) -> float64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld2r))]
@@ -11609,7 +11609,7 @@ pub unsafe fn vld2_dup_f64(a: *const f64) -> float64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld2r))]
@@ -11627,7 +11627,7 @@ pub unsafe fn vld2q_dup_f64(a: *const f64) -> float64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld2r))]
@@ -11645,7 +11645,7 @@ pub unsafe fn vld2q_dup_s64(a: *const i64) -> int64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -11663,7 +11663,7 @@ pub unsafe fn vld2_f64(a: *const f64) -> float64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11683,7 +11683,7 @@ pub unsafe fn vld2_lane_f64(a: *const f64, b: float64x1x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11703,7 +11703,7 @@ pub unsafe fn vld2_lane_s64(a: *const i64, b: int64x1x2_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11716,7 +11716,7 @@ pub unsafe fn vld2_lane_p64(a: *const p64, b: poly64x1x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11729,7 +11729,7 @@ pub unsafe fn vld2_lane_u64(a: *const u64, b: uint64x1x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11741,7 +11741,7 @@ pub unsafe fn vld2q_dup_p64(a: *const p64) -> poly64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11756,7 +11756,7 @@ pub unsafe fn vld2q_dup_p64(a: *const p64) -> poly64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11768,7 +11768,7 @@ pub unsafe fn vld2q_dup_u64(a: *const u64) -> uint64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11783,7 +11783,7 @@ pub unsafe fn vld2q_dup_u64(a: *const u64) -> uint64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld2))]
@@ -11801,7 +11801,7 @@ pub unsafe fn vld2q_f64(a: *const f64) -> float64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld2))]
@@ -11819,7 +11819,7 @@ pub unsafe fn vld2q_s64(a: *const i64) -> int64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11840,7 +11840,7 @@ pub unsafe fn vld2q_lane_f64(a: *const f64, b: float64x2x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11860,7 +11860,7 @@ pub unsafe fn vld2q_lane_s8(a: *const i8, b: int8x16x2_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11880,7 +11880,7 @@ pub unsafe fn vld2q_lane_s64(a: *const i64, b: int64x2x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11893,7 +11893,7 @@ pub unsafe fn vld2q_lane_p64(a: *const p64, b: poly64x2x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11906,7 +11906,7 @@ pub unsafe fn vld2q_lane_u8(a: *const u8, b: uint8x16x2_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11919,7 +11919,7 @@ pub unsafe fn vld2q_lane_u64(a: *const u64, b: uint64x2x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -11932,7 +11932,7 @@ pub unsafe fn vld2q_lane_p8(a: *const p8, b: poly8x16x2_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11944,7 +11944,7 @@ pub unsafe fn vld2q_p64(a: *const p64) -> poly64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11959,7 +11959,7 @@ pub unsafe fn vld2q_p64(a: *const p64) -> poly64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11971,7 +11971,7 @@ pub unsafe fn vld2q_u64(a: *const u64) -> uint64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -11986,7 +11986,7 @@ pub unsafe fn vld2q_u64(a: *const u64) -> uint64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld3r))]
@@ -12004,7 +12004,7 @@ pub unsafe fn vld3_dup_f64(a: *const f64) -> float64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld3r))]
@@ -12022,7 +12022,7 @@ pub unsafe fn vld3q_dup_f64(a: *const f64) -> float64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld3r))]
@@ -12040,7 +12040,7 @@ pub unsafe fn vld3q_dup_s64(a: *const i64) -> int64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -12058,7 +12058,7 @@ pub unsafe fn vld3_f64(a: *const f64) -> float64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12084,7 +12084,7 @@ pub unsafe fn vld3_lane_f64(a: *const f64, b: float64x1x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12097,7 +12097,7 @@ pub unsafe fn vld3_lane_p64(a: *const p64, b: poly64x1x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12123,7 +12123,7 @@ pub unsafe fn vld3_lane_s64(a: *const i64, b: int64x1x3_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12136,7 +12136,7 @@ pub unsafe fn vld3_lane_u64(a: *const u64, b: uint64x1x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12148,7 +12148,7 @@ pub unsafe fn vld3q_dup_p64(a: *const p64) -> poly64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12164,7 +12164,7 @@ pub unsafe fn vld3q_dup_p64(a: *const p64) -> poly64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12176,7 +12176,7 @@ pub unsafe fn vld3q_dup_u64(a: *const u64) -> uint64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12192,7 +12192,7 @@ pub unsafe fn vld3q_dup_u64(a: *const u64) -> uint64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld3))]
@@ -12210,7 +12210,7 @@ pub unsafe fn vld3q_f64(a: *const f64) -> float64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld3))]
@@ -12228,7 +12228,7 @@ pub unsafe fn vld3q_s64(a: *const i64) -> int64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12254,7 +12254,7 @@ pub unsafe fn vld3q_lane_f64(a: *const f64, b: float64x2x3_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12267,7 +12267,7 @@ pub unsafe fn vld3q_lane_p64(a: *const p64, b: poly64x2x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12293,7 +12293,7 @@ pub unsafe fn vld3q_lane_s8(a: *const i8, b: int8x16x3_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12319,7 +12319,7 @@ pub unsafe fn vld3q_lane_s64(a: *const i64, b: int64x2x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12332,7 +12332,7 @@ pub unsafe fn vld3q_lane_u8(a: *const u8, b: uint8x16x3_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12345,7 +12345,7 @@ pub unsafe fn vld3q_lane_u64(a: *const u64, b: uint64x2x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12358,7 +12358,7 @@ pub unsafe fn vld3q_lane_p8(a: *const p8, b: poly8x16x3_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12370,7 +12370,7 @@ pub unsafe fn vld3q_p64(a: *const p64) -> poly64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12386,7 +12386,7 @@ pub unsafe fn vld3q_p64(a: *const p64) -> poly64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12398,7 +12398,7 @@ pub unsafe fn vld3q_u64(a: *const u64) -> uint64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12414,7 +12414,7 @@ pub unsafe fn vld3q_u64(a: *const u64) -> uint64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4r))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12432,7 +12432,7 @@ pub unsafe fn vld4_dup_f64(a: *const f64) -> float64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4r))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12450,7 +12450,7 @@ pub unsafe fn vld4q_dup_f64(a: *const f64) -> float64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4r))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12468,7 +12468,7 @@ pub unsafe fn vld4q_dup_s64(a: *const i64) -> int64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -12486,7 +12486,7 @@ pub unsafe fn vld4_f64(a: *const f64) -> float64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12513,7 +12513,7 @@ pub unsafe fn vld4_lane_f64(a: *const f64, b: float64x1x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12540,7 +12540,7 @@ pub unsafe fn vld4_lane_s64(a: *const i64, b: int64x1x4_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12553,7 +12553,7 @@ pub unsafe fn vld4_lane_p64(a: *const p64, b: poly64x1x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12566,7 +12566,7 @@ pub unsafe fn vld4_lane_u64(a: *const u64, b: uint64x1x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -12578,7 +12578,7 @@ pub unsafe fn vld4q_dup_p64(a: *const p64) -> poly64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -12595,7 +12595,7 @@ pub unsafe fn vld4q_dup_p64(a: *const p64) -> poly64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -12607,7 +12607,7 @@ pub unsafe fn vld4q_dup_u64(a: *const u64) -> uint64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -12624,7 +12624,7 @@ pub unsafe fn vld4q_dup_u64(a: *const u64) -> uint64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld4))]
@@ -12642,7 +12642,7 @@ pub unsafe fn vld4q_f64(a: *const f64) -> float64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ld4))]
@@ -12660,7 +12660,7 @@ pub unsafe fn vld4q_s64(a: *const i64) -> int64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12687,7 +12687,7 @@ pub unsafe fn vld4q_lane_f64(a: *const f64, b: float64x2x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12714,7 +12714,7 @@ pub unsafe fn vld4q_lane_s8(a: *const i8, b: int8x16x4_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12741,7 +12741,7 @@ pub unsafe fn vld4q_lane_s64(a: *const i64, b: int64x2x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12754,7 +12754,7 @@ pub unsafe fn vld4q_lane_p64(a: *const p64, b: poly64x2x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12767,7 +12767,7 @@ pub unsafe fn vld4q_lane_u8(a: *const u8, b: uint8x16x4_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12780,7 +12780,7 @@ pub unsafe fn vld4q_lane_u64(a: *const u64, b: uint64x2x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -12793,7 +12793,7 @@ pub unsafe fn vld4q_lane_p8(a: *const p8, b: poly8x16x4_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
@@ -12805,7 +12805,7 @@ pub unsafe fn vld4q_p64(a: *const p64) -> poly64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
@@ -12822,7 +12822,7 @@ pub unsafe fn vld4q_p64(a: *const p64) -> poly64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12834,7 +12834,7 @@ pub unsafe fn vld4q_u64(a: *const u64) -> uint64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -12851,7 +12851,7 @@ pub unsafe fn vld4q_u64(a: *const u64) -> uint64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12871,7 +12871,7 @@ pub unsafe fn vluti2_lane_s8(a: int8x8_t, b: uint8x8_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12891,7 +12891,7 @@ pub unsafe fn vluti2q_lane_s8(a: int8x16_t, b: uint8x8_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12911,7 +12911,7 @@ pub unsafe fn vluti2_lane_s16(a: int16x4_t, b: uint8x8_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12931,7 +12931,7 @@ pub unsafe fn vluti2q_lane_s16(a: int16x8_t, b: uint8x8_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12944,7 +12944,7 @@ pub unsafe fn vluti2_lane_u8(a: uint8x8_t, b: uint8x8_t) -> uin
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12957,7 +12957,7 @@ pub unsafe fn vluti2q_lane_u8(a: uint8x16_t, b: uint8x8_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12970,7 +12970,7 @@ pub unsafe fn vluti2_lane_u16(a: uint16x4_t, b: uint8x8_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12983,7 +12983,7 @@ pub unsafe fn vluti2q_lane_u16(a: uint16x8_t, b: uint8x8_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -12996,7 +12996,7 @@ pub unsafe fn vluti2_lane_p8(a: poly8x8_t, b: uint8x8_t) -> pol
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13009,7 +13009,7 @@ pub unsafe fn vluti2q_lane_p8(a: poly8x16_t, b: uint8x8_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13022,7 +13022,7 @@ pub unsafe fn vluti2_lane_p16(a: poly16x4_t, b: uint8x8_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti2q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 1))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13035,7 +13035,7 @@ pub unsafe fn vluti2q_lane_p16(a: poly16x8_t, b: uint8x8_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut,fp16")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13048,7 +13048,7 @@ pub unsafe fn vluti4q_lane_f16_x2(a: float16x8x2_t, b: uint8x8_
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13061,7 +13061,7 @@ pub unsafe fn vluti4q_lane_u16_x2(a: uint16x8x2_t, b: uint8x8_t
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13074,7 +13074,7 @@ pub unsafe fn vluti4q_lane_p16_x2(a: poly16x8x2_t, b: uint8x8_t
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13094,7 +13094,7 @@ pub unsafe fn vluti4q_lane_s16_x2(a: int16x8x2_t, b: uint8x8_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13114,7 +13114,7 @@ pub unsafe fn vluti4q_lane_s8(a: int8x16_t, b: uint8x8_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13127,7 +13127,7 @@ pub unsafe fn vluti4q_lane_u8(a: uint8x16_t, b: uint8x8_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13140,7 +13140,7 @@ pub unsafe fn vluti4q_lane_p8(a: poly8x16_t, b: uint8x8_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut,fp16")]
 #[cfg_attr(test, assert_instr(nop, LANE = 3))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13156,7 +13156,7 @@ pub unsafe fn vluti4q_laneq_f16_x2(
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 3))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13169,7 +13169,7 @@ pub unsafe fn vluti4q_laneq_u16_x2(a: uint16x8x2_t, b: uint8x16
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 3))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13182,7 +13182,7 @@ pub unsafe fn vluti4q_laneq_p16_x2(a: poly16x8x2_t, b: uint8x16
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 3))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13202,7 +13202,7 @@ pub unsafe fn vluti4q_laneq_s16_x2(a: int16x8x2_t, b: uint8x16_
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13222,7 +13222,7 @@ pub unsafe fn vluti4q_laneq_s8(a: int8x16_t, b: uint8x16_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13235,7 +13235,7 @@ pub unsafe fn vluti4q_laneq_u8(a: uint8x16_t, b: uint8x16_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vluti4q_laneq_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,lut")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[unstable(feature = "stdarch_neon_feat_lut", issue = "138050")]
@@ -13246,7 +13246,7 @@ pub unsafe fn vluti4q_laneq_p8(a: poly8x16_t, b: uint8x16_t) ->
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmax))]
@@ -13262,7 +13262,7 @@ pub fn vmax_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmax))]
@@ -13278,7 +13278,7 @@ pub fn vmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13295,7 +13295,7 @@ pub fn vmaxh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point Maximum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnm_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxnm))]
@@ -13304,7 +13304,7 @@ pub fn vmaxnm_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point Maximum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxnm))]
@@ -13313,7 +13313,7 @@ pub fn vmaxnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point Maximum Number"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13323,7 +13323,7 @@ pub fn vmaxnmh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmv_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13333,7 +13333,7 @@ pub fn vmaxnmv_f16(a: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmvq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13343,7 +13343,7 @@ pub fn vmaxnmvq_f16(a: float16x8_t) -> f16 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmv_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
@@ -13352,7 +13352,7 @@ pub fn vmaxnmv_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmvq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
@@ -13361,7 +13361,7 @@ pub fn vmaxnmvq_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmvq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxnmv))]
@@ -13370,7 +13370,7 @@ pub fn vmaxnmvq_f32(a: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13387,7 +13387,7 @@ pub fn vmaxv_f16(a: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point maximum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13404,7 +13404,7 @@ pub fn vmaxvq_f16(a: float16x8_t) -> f16 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxp))]
@@ -13420,7 +13420,7 @@ pub fn vmaxv_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxv))]
@@ -13436,7 +13436,7 @@ pub fn vmaxvq_f32(a: float32x4_t) -> f32 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxp))]
@@ -13452,7 +13452,7 @@ pub fn vmaxvq_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxv))]
@@ -13461,7 +13461,7 @@ pub fn vmaxv_s8(a: int8x8_t) -> i8 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxv))]
@@ -13470,7 +13470,7 @@ pub fn vmaxvq_s8(a: int8x16_t) -> i8 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxv))]
@@ -13479,7 +13479,7 @@ pub fn vmaxv_s16(a: int16x4_t) -> i16 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxv))]
@@ -13488,7 +13488,7 @@ pub fn vmaxvq_s16(a: int16x8_t) -> i16 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxp))]
@@ -13497,7 +13497,7 @@ pub fn vmaxv_s32(a: int32x2_t) -> i32 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxv))]
@@ -13506,7 +13506,7 @@ pub fn vmaxvq_s32(a: int32x4_t) -> i32 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxv))]
@@ -13515,7 +13515,7 @@ pub fn vmaxv_u8(a: uint8x8_t) -> u8 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxv))]
@@ -13524,7 +13524,7 @@ pub fn vmaxvq_u8(a: uint8x16_t) -> u8 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxv))]
@@ -13533,7 +13533,7 @@ pub fn vmaxv_u16(a: uint16x4_t) -> u16 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxv))]
@@ -13542,7 +13542,7 @@ pub fn vmaxvq_u16(a: uint16x8_t) -> u16 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxv_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxp))]
@@ -13551,7 +13551,7 @@ pub fn vmaxv_u32(a: uint32x2_t) -> u32 {
 }
 #[doc = "Horizontal vector max."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxvq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxv))]
@@ -13560,7 +13560,7 @@ pub fn vmaxvq_u32(a: uint32x4_t) -> u32 {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmin))]
@@ -13576,7 +13576,7 @@ pub fn vmin_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmin))]
@@ -13592,7 +13592,7 @@ pub fn vminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13609,7 +13609,7 @@ pub fn vminh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point Minimum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnm_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminnm))]
@@ -13618,7 +13618,7 @@ pub fn vminnm_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point Minimum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminnm))]
@@ -13627,7 +13627,7 @@ pub fn vminnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point Minimum Number"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13637,7 +13637,7 @@ pub fn vminnmh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmv_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13647,7 +13647,7 @@ pub fn vminnmv_f16(a: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmvq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13657,7 +13657,7 @@ pub fn vminnmvq_f16(a: float16x8_t) -> f16 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmv_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -13666,7 +13666,7 @@ pub fn vminnmv_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmvq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -13675,7 +13675,7 @@ pub fn vminnmvq_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmvq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmv))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -13684,7 +13684,7 @@ pub fn vminnmvq_f32(a: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13701,7 +13701,7 @@ pub fn vminv_f16(a: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point minimum number across vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -13718,7 +13718,7 @@ pub fn vminvq_f16(a: float16x8_t) -> f16 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminp))]
@@ -13734,7 +13734,7 @@ pub fn vminv_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminv))]
@@ -13750,7 +13750,7 @@ pub fn vminvq_f32(a: float32x4_t) -> f32 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminp))]
@@ -13766,7 +13766,7 @@ pub fn vminvq_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminv))]
@@ -13775,7 +13775,7 @@ pub fn vminv_s8(a: int8x8_t) -> i8 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminv))]
@@ -13784,7 +13784,7 @@ pub fn vminvq_s8(a: int8x16_t) -> i8 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminv))]
@@ -13793,7 +13793,7 @@ pub fn vminv_s16(a: int16x4_t) -> i16 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminv))]
@@ -13802,7 +13802,7 @@ pub fn vminvq_s16(a: int16x8_t) -> i16 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminp))]
@@ -13811,7 +13811,7 @@ pub fn vminv_s32(a: int32x2_t) -> i32 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminv))]
@@ -13820,7 +13820,7 @@ pub fn vminvq_s32(a: int32x4_t) -> i32 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminv))]
@@ -13829,7 +13829,7 @@ pub fn vminv_u8(a: uint8x8_t) -> u8 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminv))]
@@ -13838,7 +13838,7 @@ pub fn vminvq_u8(a: uint8x16_t) -> u8 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminv))]
@@ -13847,7 +13847,7 @@ pub fn vminv_u16(a: uint16x4_t) -> u16 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminv))]
@@ -13856,7 +13856,7 @@ pub fn vminvq_u16(a: uint16x8_t) -> u16 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminv_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminp))]
@@ -13865,7 +13865,7 @@ pub fn vminv_u32(a: uint32x2_t) -> u32 {
 }
 #[doc = "Horizontal vector min."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminvq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminv))]
@@ -13874,7 +13874,7 @@ pub fn vminvq_u32(a: uint32x4_t) -> u32 {
 }
 #[doc = "Floating-point multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -13883,7 +13883,7 @@ pub fn vmla_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -13892,7 +13892,7 @@ pub fn vmlaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -13922,7 +13922,7 @@ pub fn vmlal_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -13956,7 +13956,7 @@ pub fn vmlal_high_laneq_s16(
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -13973,7 +13973,7 @@ pub fn vmlal_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -13994,7 +13994,7 @@ pub fn vmlal_high_laneq_s32(
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14028,7 +14028,7 @@ pub fn vmlal_high_lane_u16(
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14062,7 +14062,7 @@ pub fn vmlal_high_laneq_u16(
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14083,7 +14083,7 @@ pub fn vmlal_high_lane_u32(
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14104,7 +14104,7 @@ pub fn vmlal_high_laneq_u32(
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14113,7 +14113,7 @@ pub fn vmlal_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t {
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14122,7 +14122,7 @@ pub fn vmlal_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t {
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14131,7 +14131,7 @@ pub fn vmlal_high_n_u16(a: uint32x4_t, b: uint16x8_t, c: u16) -> uint32x4_t {
 }
 #[doc = "Multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14140,7 +14140,7 @@ pub fn vmlal_high_n_u32(a: uint64x2_t, b: uint32x4_t, c: u32) -> uint64x2_t {
 }
 #[doc = "Signed multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14153,7 +14153,7 @@ pub fn vmlal_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14166,7 +14166,7 @@ pub fn vmlal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14179,7 +14179,7 @@ pub fn vmlal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14192,7 +14192,7 @@ pub fn vmlal_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t
 }
 #[doc = "Unsigned multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14205,7 +14205,7 @@ pub fn vmlal_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t
 }
 #[doc = "Unsigned multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14218,7 +14218,7 @@ pub fn vmlal_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t
 }
 #[doc = "Floating-point multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14227,7 +14227,7 @@ pub fn vmls_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14236,7 +14236,7 @@ pub fn vmlsq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14266,7 +14266,7 @@ pub fn vmlsl_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14300,7 +14300,7 @@ pub fn vmlsl_high_laneq_s16(
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14317,7 +14317,7 @@ pub fn vmlsl_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14338,7 +14338,7 @@ pub fn vmlsl_high_laneq_s32(
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14372,7 +14372,7 @@ pub fn vmlsl_high_lane_u16(
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14406,7 +14406,7 @@ pub fn vmlsl_high_laneq_u16(
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14427,7 +14427,7 @@ pub fn vmlsl_high_lane_u32(
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -14448,7 +14448,7 @@ pub fn vmlsl_high_laneq_u32(
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14457,7 +14457,7 @@ pub fn vmlsl_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t {
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14466,7 +14466,7 @@ pub fn vmlsl_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t {
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14475,7 +14475,7 @@ pub fn vmlsl_high_n_u16(a: uint32x4_t, b: uint16x8_t, c: u16) -> uint32x4_t {
 }
 #[doc = "Multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14484,7 +14484,7 @@ pub fn vmlsl_high_n_u32(a: uint64x2_t, b: uint32x4_t, c: u32) -> uint64x2_t {
 }
 #[doc = "Signed multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14497,7 +14497,7 @@ pub fn vmlsl_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14510,7 +14510,7 @@ pub fn vmlsl_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14523,7 +14523,7 @@ pub fn vmlsl_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14536,7 +14536,7 @@ pub fn vmlsl_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t
 }
 #[doc = "Unsigned multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14549,7 +14549,7 @@ pub fn vmlsl_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t
 }
 #[doc = "Unsigned multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14562,7 +14562,7 @@ pub fn vmlsl_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t
 }
 #[doc = "Vector move"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sxtl2))]
@@ -14574,7 +14574,7 @@ pub fn vmovl_high_s8(a: int8x16_t) -> int16x8_t {
 }
 #[doc = "Vector move"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sxtl2))]
@@ -14586,7 +14586,7 @@ pub fn vmovl_high_s16(a: int16x8_t) -> int32x4_t {
 }
 #[doc = "Vector move"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sxtl2))]
@@ -14598,7 +14598,7 @@ pub fn vmovl_high_s32(a: int32x4_t) -> int64x2_t {
 }
 #[doc = "Vector move"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uxtl2))]
@@ -14610,7 +14610,7 @@ pub fn vmovl_high_u8(a: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector move"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uxtl2))]
@@ -14622,7 +14622,7 @@ pub fn vmovl_high_u16(a: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector move"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uxtl2))]
@@ -14634,7 +14634,7 @@ pub fn vmovl_high_u32(a: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(xtn2))]
@@ -14646,7 +14646,7 @@ pub fn vmovn_high_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t {
 }
 #[doc = "Extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(xtn2))]
@@ -14658,7 +14658,7 @@ pub fn vmovn_high_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t {
 }
 #[doc = "Extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(xtn2))]
@@ -14670,7 +14670,7 @@ pub fn vmovn_high_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t {
 }
 #[doc = "Extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(xtn2))]
@@ -14682,7 +14682,7 @@ pub fn vmovn_high_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t {
 }
 #[doc = "Extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(xtn2))]
@@ -14694,7 +14694,7 @@ pub fn vmovn_high_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t {
 }
 #[doc = "Extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(xtn2))]
@@ -14706,7 +14706,7 @@ pub fn vmovn_high_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmul))]
@@ -14715,7 +14715,7 @@ pub fn vmul_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmul))]
@@ -14724,7 +14724,7 @@ pub fn vmulq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -14735,7 +14735,7 @@ pub fn vmul_lane_f64(a: float64x1_t, b: float64x1_t) -> float64
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -14752,7 +14752,7 @@ pub fn vmul_laneq_f16(a: float16x4_t, b: float16x8_t) -> float1
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -14782,7 +14782,7 @@ pub fn vmulq_laneq_f16(a: float16x8_t, b: float16x8_t) -> float
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -14793,7 +14793,7 @@ pub fn vmul_laneq_f64(a: float64x1_t, b: float64x2_t) -> float6
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14802,7 +14802,7 @@ pub fn vmul_n_f64(a: float64x1_t, b: f64) -> float64x1_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -14811,7 +14811,7 @@ pub fn vmulq_n_f64(a: float64x2_t, b: f64) -> float64x2_t {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuld_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -14825,7 +14825,7 @@ pub fn vmuld_lane_f64(a: f64, b: float64x1_t) -> f64 {
 }
 #[doc = "Add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -14835,7 +14835,7 @@ pub fn vmulh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulh_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -14850,7 +14850,7 @@ pub fn vmulh_lane_f16(a: f16, b: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulh_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -14865,7 +14865,7 @@ pub fn vmulh_laneq_f16(a: f16, b: float16x8_t) -> f16 {
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -14894,7 +14894,7 @@ pub fn vmull_high_lane_s16(a: int16x8_t, b: int16x4_t) -> int32
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -14923,7 +14923,7 @@ pub fn vmull_high_laneq_s16(a: int16x8_t, b: int16x8_t) -> int3
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -14939,7 +14939,7 @@ pub fn vmull_high_lane_s32(a: int32x4_t, b: int32x2_t) -> int64
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -14955,7 +14955,7 @@ pub fn vmull_high_laneq_s32(a: int32x4_t, b: int32x4_t) -> int6
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -14984,7 +14984,7 @@ pub fn vmull_high_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uin
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -15013,7 +15013,7 @@ pub fn vmull_high_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> ui
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -15029,7 +15029,7 @@ pub fn vmull_high_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uin
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umull2, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -15045,7 +15045,7 @@ pub fn vmull_high_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> ui
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15054,7 +15054,7 @@ pub fn vmull_high_n_s16(a: int16x8_t, b: i16) -> int32x4_t {
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(smull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15063,7 +15063,7 @@ pub fn vmull_high_n_s32(a: int32x4_t, b: i32) -> int64x2_t {
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15072,7 +15072,7 @@ pub fn vmull_high_n_u16(a: uint16x8_t, b: u16) -> uint32x4_t {
 }
 #[doc = "Multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(umull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15081,7 +15081,7 @@ pub fn vmull_high_n_u32(a: uint32x4_t, b: u32) -> uint64x2_t {
 }
 #[doc = "Polynomial multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(pmull2))]
@@ -15090,7 +15090,7 @@ pub fn vmull_high_p64(a: poly64x2_t, b: poly64x2_t) -> p128 {
 }
 #[doc = "Polynomial multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(pmull2))]
@@ -15103,7 +15103,7 @@ pub fn vmull_high_p8(a: poly8x16_t, b: poly8x16_t) -> poly16x8_t {
 }
 #[doc = "Signed multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smull2))]
@@ -15116,7 +15116,7 @@ pub fn vmull_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smull2))]
@@ -15129,7 +15129,7 @@ pub fn vmull_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smull2))]
@@ -15142,7 +15142,7 @@ pub fn vmull_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umull2))]
@@ -15155,7 +15155,7 @@ pub fn vmull_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Unsigned multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umull2))]
@@ -15168,7 +15168,7 @@ pub fn vmull_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Unsigned multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umull2))]
@@ -15181,7 +15181,7 @@ pub fn vmull_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Polynomial multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(pmull))]
@@ -15197,7 +15197,7 @@ pub fn vmull_p64(a: p64, b: p64) -> p128 {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15208,7 +15208,7 @@ pub fn vmulq_lane_f64(a: float64x2_t, b: float64x1_t) -> float6
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15219,7 +15219,7 @@ pub fn vmulq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuls_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15233,7 +15233,7 @@ pub fn vmuls_lane_f32(a: f32, b: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuls_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15247,7 +15247,7 @@ pub fn vmuls_laneq_f32(a: f32, b: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmuld_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmul, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15261,7 +15261,7 @@ pub fn vmuld_laneq_f64(a: f64, b: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -15278,7 +15278,7 @@ pub fn vmulx_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -15295,7 +15295,7 @@ pub fn vmulxq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmulx))]
@@ -15311,7 +15311,7 @@ pub fn vmulx_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmulx))]
@@ -15327,7 +15327,7 @@ pub fn vmulxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmulx))]
@@ -15343,7 +15343,7 @@ pub fn vmulx_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmulx))]
@@ -15359,7 +15359,7 @@ pub fn vmulxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -15376,7 +15376,7 @@ pub fn vmulx_lane_f16(a: float16x4_t, b: float16x4_t) -> float1
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -15393,7 +15393,7 @@ pub fn vmulx_laneq_f16(a: float16x4_t, b: float16x8_t) -> float
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -15423,7 +15423,7 @@ pub fn vmulxq_lane_f16(a: float16x8_t, b: float16x4_t) -> float
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -15453,7 +15453,7 @@ pub fn vmulxq_laneq_f16(a: float16x8_t, b: float16x8_t) -> floa
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15464,7 +15464,7 @@ pub fn vmulx_lane_f32(a: float32x2_t, b: float32x2_t) -> float3
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15475,7 +15475,7 @@ pub fn vmulx_laneq_f32(a: float32x2_t, b: float32x4_t) -> float
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15491,7 +15491,7 @@ pub fn vmulxq_lane_f32(a: float32x4_t, b: float32x2_t) -> float
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15507,7 +15507,7 @@ pub fn vmulxq_laneq_f32(a: float32x4_t, b: float32x4_t) -> floa
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15518,7 +15518,7 @@ pub fn vmulxq_laneq_f64(a: float64x2_t, b: float64x2_t) -> floa
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15529,7 +15529,7 @@ pub fn vmulx_lane_f64(a: float64x1_t, b: float64x1_t) -> float6
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15540,7 +15540,7 @@ pub fn vmulx_laneq_f64(a: float64x1_t, b: float64x2_t) -> float
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulx_n_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -15550,7 +15550,7 @@ pub fn vmulx_n_f16(a: float16x4_t, b: f16) -> float16x4_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_n_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -15560,7 +15560,7 @@ pub fn vmulxq_n_f16(a: float16x8_t, b: f16) -> float16x8_t {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmulx))]
@@ -15576,7 +15576,7 @@ pub fn vmulxd_f64(a: f64, b: f64) -> f64 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmulx))]
@@ -15592,7 +15592,7 @@ pub fn vmulxs_f32(a: f32, b: f32) -> f32 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxd_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15603,7 +15603,7 @@ pub fn vmulxd_lane_f64(a: f64, b: float64x1_t) -> f64 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxd_laneq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15614,7 +15614,7 @@ pub fn vmulxd_laneq_f64(a: f64, b: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxs_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15625,7 +15625,7 @@ pub fn vmulxs_lane_f32(a: f32, b: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxs_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15636,7 +15636,7 @@ pub fn vmulxs_laneq_f32(a: f32, b: float32x4_t) -> f32 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -15653,7 +15653,7 @@ pub fn vmulxh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxh_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -15665,7 +15665,7 @@ pub fn vmulxh_lane_f16(a: f16, b: float16x4_t) -> f16 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxh_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
 #[target_feature(enable = "neon,fp16")]
@@ -15677,7 +15677,7 @@ pub fn vmulxh_laneq_f16(a: f16, b: float16x8_t) -> f16 {
 }
 #[doc = "Floating-point multiply extended"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulxq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmulx, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -15688,7 +15688,7 @@ pub fn vmulxq_lane_f64(a: float64x2_t, b: float64x1_t) -> float
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fneg))]
@@ -15697,7 +15697,7 @@ pub fn vneg_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fneg))]
@@ -15706,7 +15706,7 @@ pub fn vnegq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(neg))]
@@ -15715,7 +15715,7 @@ pub fn vneg_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(neg))]
@@ -15724,7 +15724,7 @@ pub fn vnegq_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(neg))]
@@ -15733,7 +15733,7 @@ pub fn vnegd_s64(a: i64) -> i64 {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -15743,7 +15743,7 @@ pub fn vnegh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -15756,7 +15756,7 @@ pub fn vpaddd_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadds_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -15769,7 +15769,7 @@ pub fn vpadds_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -15778,7 +15778,7 @@ pub fn vpaddd_s64(a: int64x2_t) -> i64 {
 }
 #[doc = "Add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -15787,7 +15787,7 @@ pub fn vpaddd_u64(a: uint64x2_t) -> u64 {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -15804,7 +15804,7 @@ pub fn vpaddq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(faddp))]
@@ -15820,7 +15820,7 @@ pub fn vpaddq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(faddp))]
@@ -15836,7 +15836,7 @@ pub fn vpaddq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -15852,7 +15852,7 @@ pub fn vpaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -15868,7 +15868,7 @@ pub fn vpaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -15884,7 +15884,7 @@ pub fn vpaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(addp))]
@@ -15900,7 +15900,7 @@ pub fn vpaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15910,7 +15910,7 @@ pub fn vpaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15931,7 +15931,7 @@ pub fn vpaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15941,7 +15941,7 @@ pub fn vpaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15956,7 +15956,7 @@ pub fn vpaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15966,7 +15966,7 @@ pub fn vpaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15981,7 +15981,7 @@ pub fn vpaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -15991,7 +15991,7 @@ pub fn vpaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Add Pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddq_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16006,7 +16006,7 @@ pub fn vpaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16023,7 +16023,7 @@ pub fn vpmax_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16040,7 +16040,7 @@ pub fn vpmaxq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnm_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16057,7 +16057,7 @@ pub fn vpmaxnm_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16074,7 +16074,7 @@ pub fn vpmaxnmq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point Maximum Number Pairwise (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnm_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16090,7 +16090,7 @@ pub fn vpmaxnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point Maximum Number Pairwise (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16106,7 +16106,7 @@ pub fn vpmaxnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point Maximum Number Pairwise (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16122,7 +16122,7 @@ pub fn vpmaxnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point maximum number pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnmqd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16138,7 +16138,7 @@ pub fn vpmaxnmqd_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point maximum number pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxnms_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fmaxnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16154,7 +16154,7 @@ pub fn vpmaxnms_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxp))]
@@ -16170,7 +16170,7 @@ pub fn vpmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxp))]
@@ -16186,7 +16186,7 @@ pub fn vpmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxp))]
@@ -16202,7 +16202,7 @@ pub fn vpmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxp))]
@@ -16218,7 +16218,7 @@ pub fn vpmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(smaxp))]
@@ -16234,7 +16234,7 @@ pub fn vpmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxp))]
@@ -16250,7 +16250,7 @@ pub fn vpmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxp))]
@@ -16266,7 +16266,7 @@ pub fn vpmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(umaxp))]
@@ -16282,7 +16282,7 @@ pub fn vpmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point maximum pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxqd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxp))]
@@ -16298,7 +16298,7 @@ pub fn vpmaxqd_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point maximum pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmaxs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fmaxp))]
@@ -16314,7 +16314,7 @@ pub fn vpmaxs_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16331,7 +16331,7 @@ pub fn vpmin_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16348,7 +16348,7 @@ pub fn vpminq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnm_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16365,7 +16365,7 @@ pub fn vpminnm_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -16382,7 +16382,7 @@ pub fn vpminnmq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point Minimum Number Pairwise (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnm_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16398,7 +16398,7 @@ pub fn vpminnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point Minimum Number Pairwise (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16414,7 +16414,7 @@ pub fn vpminnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point Minimum Number Pairwise (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16430,7 +16430,7 @@ pub fn vpminnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point minimum number pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnmqd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16446,7 +16446,7 @@ pub fn vpminnmqd_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point minimum number pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminnms_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fminnmp))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16462,7 +16462,7 @@ pub fn vpminnms_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminp))]
@@ -16478,7 +16478,7 @@ pub fn vpminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminp))]
@@ -16494,7 +16494,7 @@ pub fn vpminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminp))]
@@ -16510,7 +16510,7 @@ pub fn vpminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminp))]
@@ -16526,7 +16526,7 @@ pub fn vpminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sminp))]
@@ -16542,7 +16542,7 @@ pub fn vpminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminp))]
@@ -16558,7 +16558,7 @@ pub fn vpminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminp))]
@@ -16574,7 +16574,7 @@ pub fn vpminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uminp))]
@@ -16590,7 +16590,7 @@ pub fn vpminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point minimum pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpminqd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminp))]
@@ -16606,7 +16606,7 @@ pub fn vpminqd_f64(a: float64x2_t) -> f64 {
 }
 #[doc = "Floating-point minimum pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmins_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fminp))]
@@ -16622,7 +16622,7 @@ pub fn vpmins_f32(a: float32x2_t) -> f32 {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sqabs))]
@@ -16638,7 +16638,7 @@ pub fn vqabs_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sqabs))]
@@ -16654,7 +16654,7 @@ pub fn vqabsq_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Signed saturating absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sqabs))]
@@ -16663,7 +16663,7 @@ pub fn vqabsb_s8(a: i8) -> i8 {
 }
 #[doc = "Signed saturating absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sqabs))]
@@ -16672,7 +16672,7 @@ pub fn vqabsh_s16(a: i16) -> i16 {
 }
 #[doc = "Signed saturating absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabss_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sqabs))]
@@ -16688,7 +16688,7 @@ pub fn vqabss_s32(a: i32) -> i32 {
 }
 #[doc = "Signed saturating absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(sqabs))]
@@ -16704,7 +16704,7 @@ pub fn vqabsd_s64(a: i64) -> i64 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqadd))]
@@ -16715,7 +16715,7 @@ pub fn vqaddb_s8(a: i8, b: i8) -> i8 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqadd))]
@@ -16726,7 +16726,7 @@ pub fn vqaddh_s16(a: i16, b: i16) -> i16 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddb_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqadd))]
@@ -16737,7 +16737,7 @@ pub fn vqaddb_u8(a: u8, b: u8) -> u8 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddh_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqadd))]
@@ -16748,7 +16748,7 @@ pub fn vqaddh_u16(a: u16, b: u16) -> u16 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadds_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqadd))]
@@ -16764,7 +16764,7 @@ pub fn vqadds_s32(a: i32, b: i32) -> i32 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqadd))]
@@ -16780,7 +16780,7 @@ pub fn vqaddd_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadds_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqadd))]
@@ -16796,7 +16796,7 @@ pub fn vqadds_u32(a: u32, b: u32) -> u32 {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqadd))]
@@ -16812,7 +16812,7 @@ pub fn vqaddd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -16823,7 +16823,7 @@ pub fn vqdmlal_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -16834,7 +16834,7 @@ pub fn vqdmlal_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -16845,7 +16845,7 @@ pub fn vqdmlal_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -16856,7 +16856,7 @@ pub fn vqdmlal_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16865,7 +16865,7 @@ pub fn vqdmlal_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16874,7 +16874,7 @@ pub fn vqdmlal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16883,7 +16883,7 @@ pub fn vqdmlal_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16892,7 +16892,7 @@ pub fn vqdmlal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t {
 }
 #[doc = "Vector widening saturating doubling multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal, N = 2))]
 #[rustc_legacy_const_generics(3)]
@@ -16903,7 +16903,7 @@ pub fn vqdmlal_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t)
 }
 #[doc = "Vector widening saturating doubling multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -16914,7 +16914,7 @@ pub fn vqdmlal_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t)
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlalh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -16925,7 +16925,7 @@ pub fn vqdmlalh_lane_s16(a: i32, b: i16, c: int16x4_t) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlalh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -16936,7 +16936,7 @@ pub fn vqdmlalh_laneq_s16(a: i32, b: i16, c: int16x8_t) -> i32
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlals_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -16947,7 +16947,7 @@ pub fn vqdmlals_lane_s32(a: i64, b: i32, c: int32x2_t) -> i64 {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlals_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -16958,7 +16958,7 @@ pub fn vqdmlals_laneq_s32(a: i64, b: i32, c: int32x4_t) -> i64
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlalh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16968,7 +16968,7 @@ pub fn vqdmlalh_s16(a: i32, b: i16, c: i16) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlals_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlal))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -16978,7 +16978,7 @@ pub fn vqdmlals_s32(a: i64, b: i32, c: i32) -> i64 {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -16989,7 +16989,7 @@ pub fn vqdmlsl_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17000,7 +17000,7 @@ pub fn vqdmlsl_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17011,7 +17011,7 @@ pub fn vqdmlsl_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17022,7 +17022,7 @@ pub fn vqdmlsl_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17031,7 +17031,7 @@ pub fn vqdmlsl_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17040,7 +17040,7 @@ pub fn vqdmlsl_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17049,7 +17049,7 @@ pub fn vqdmlsl_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17058,7 +17058,7 @@ pub fn vqdmlsl_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t {
 }
 #[doc = "Vector widening saturating doubling multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl, N = 2))]
 #[rustc_legacy_const_generics(3)]
@@ -17069,7 +17069,7 @@ pub fn vqdmlsl_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t)
 }
 #[doc = "Vector widening saturating doubling multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl, N = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17080,7 +17080,7 @@ pub fn vqdmlsl_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t)
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlslh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -17091,7 +17091,7 @@ pub fn vqdmlslh_lane_s16(a: i32, b: i16, c: int16x4_t) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlslh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -17102,7 +17102,7 @@ pub fn vqdmlslh_laneq_s16(a: i32, b: i16, c: int16x8_t) -> i32
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsls_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -17113,7 +17113,7 @@ pub fn vqdmlsls_lane_s32(a: i64, b: i32, c: int32x2_t) -> i64 {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsls_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl, LANE = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -17124,7 +17124,7 @@ pub fn vqdmlsls_laneq_s32(a: i64, b: i32, c: int32x4_t) -> i64
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlslh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17134,7 +17134,7 @@ pub fn vqdmlslh_s16(a: i32, b: i16, c: i16) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsls_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmlsl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17144,7 +17144,7 @@ pub fn vqdmlsls_s32(a: i64, b: i32, c: i32) -> i64 {
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -17155,7 +17155,7 @@ pub fn vqdmulh_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -17166,7 +17166,7 @@ pub fn vqdmulhq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -17177,7 +17177,7 @@ pub fn vqdmulh_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -17188,7 +17188,7 @@ pub fn vqdmulhq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17202,7 +17202,7 @@ pub fn vqdmulhh_lane_s16(a: i16, b: int16x4_t) -> i16 {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17216,7 +17216,7 @@ pub fn vqdmulhh_laneq_s16(a: i16, b: int16x8_t) -> i16 {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17227,7 +17227,7 @@ pub fn vqdmulhh_s16(a: i16, b: i16) -> i16 {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17238,7 +17238,7 @@ pub fn vqdmulhs_s32(a: i32, b: i32) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhs_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -17252,7 +17252,7 @@ pub fn vqdmulhs_lane_s32(a: i32, b: int32x2_t) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhs_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmulh, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -17266,7 +17266,7 @@ pub fn vqdmulhs_laneq_s32(a: i32, b: int32x4_t) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17281,7 +17281,7 @@ pub fn vqdmull_high_lane_s16(a: int16x8_t, b: int16x4_t) -> int32x
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17296,7 +17296,7 @@ pub fn vqdmull_high_laneq_s32(a: int32x4_t, b: int32x4_t) -> int64
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -17311,7 +17311,7 @@ pub fn vqdmull_high_lane_s32(a: int32x4_t, b: int32x2_t) -> int64x
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2, N = 4))]
 #[rustc_legacy_const_generics(2)]
@@ -17326,7 +17326,7 @@ pub fn vqdmull_high_laneq_s16(a: int16x8_t, b: int16x8_t) -> int32
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17339,7 +17339,7 @@ pub fn vqdmull_high_n_s16(a: int16x8_t, b: i16) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17352,7 +17352,7 @@ pub fn vqdmull_high_n_s32(a: int32x4_t, b: i32) -> int64x2_t {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17365,7 +17365,7 @@ pub fn vqdmull_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17378,7 +17378,7 @@ pub fn vqdmull_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Vector saturating doubling long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull, N = 4))]
 #[rustc_legacy_const_generics(2)]
@@ -17392,7 +17392,7 @@ pub fn vqdmull_laneq_s16(a: int16x4_t, b: int16x8_t) -> int32x4_t
 }
 #[doc = "Vector saturating doubling long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17406,7 +17406,7 @@ pub fn vqdmull_laneq_s32(a: int32x2_t, b: int32x4_t) -> int64x2_t
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmullh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17420,7 +17420,7 @@ pub fn vqdmullh_lane_s16(a: i16, b: int16x4_t) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulls_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -17434,7 +17434,7 @@ pub fn vqdmulls_laneq_s32(a: i32, b: int32x4_t) -> i64 {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmullh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull, N = 4))]
 #[rustc_legacy_const_generics(2)]
@@ -17448,7 +17448,7 @@ pub fn vqdmullh_laneq_s16(a: i16, b: int16x8_t) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmullh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17459,7 +17459,7 @@ pub fn vqdmullh_s16(a: i16, b: i16) -> i32 {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulls_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -17473,7 +17473,7 @@ pub fn vqdmulls_lane_s32(a: i32, b: int32x2_t) -> i64 {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulls_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqdmull))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17489,7 +17489,7 @@ pub fn vqdmulls_s32(a: i32, b: i32) -> i64 {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17504,7 +17504,7 @@ pub fn vqmovn_high_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17513,7 +17513,7 @@ pub fn vqmovn_high_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17522,7 +17522,7 @@ pub fn vqmovn_high_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqxtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17537,7 +17537,7 @@ pub fn vqmovn_high_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqxtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17546,7 +17546,7 @@ pub fn vqmovn_high_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqxtn2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17555,7 +17555,7 @@ pub fn vqmovn_high_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t {
 }
 #[doc = "Saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17571,7 +17571,7 @@ pub fn vqmovnd_s64(a: i64) -> i32 {
 }
 #[doc = "Saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqxtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17587,7 +17587,7 @@ pub fn vqmovnd_u64(a: u64) -> u32 {
 }
 #[doc = "Saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17596,7 +17596,7 @@ pub fn vqmovnh_s16(a: i16) -> i8 {
 }
 #[doc = "Saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovns_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17605,7 +17605,7 @@ pub fn vqmovns_s32(a: i32) -> i16 {
 }
 #[doc = "Saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovnh_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqxtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17614,7 +17614,7 @@ pub fn vqmovnh_u16(a: u16) -> u8 {
 }
 #[doc = "Saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovns_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqxtn))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17623,7 +17623,7 @@ pub fn vqmovns_u32(a: u32) -> u16 {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtun2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17638,7 +17638,7 @@ pub fn vqmovun_high_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtun2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17647,7 +17647,7 @@ pub fn vqmovun_high_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtun2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17656,7 +17656,7 @@ pub fn vqmovun_high_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovunh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtun))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17665,7 +17665,7 @@ pub fn vqmovunh_s16(a: i16) -> u8 {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovuns_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtun))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17674,7 +17674,7 @@ pub fn vqmovuns_s32(a: i32) -> u16 {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovund_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqxtun))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -17683,7 +17683,7 @@ pub fn vqmovund_s64(a: i64) -> u32 {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqneg))]
@@ -17699,7 +17699,7 @@ pub fn vqneg_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqneg))]
@@ -17715,7 +17715,7 @@ pub fn vqnegq_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqneg))]
@@ -17724,7 +17724,7 @@ pub fn vqnegb_s8(a: i8) -> i8 {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqneg))]
@@ -17733,7 +17733,7 @@ pub fn vqnegh_s16(a: i16) -> i16 {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqneg))]
@@ -17742,7 +17742,7 @@ pub fn vqnegs_s32(a: i32) -> i32 {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqneg))]
@@ -17751,7 +17751,7 @@ pub fn vqnegd_s64(a: i64) -> i64 {
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17766,7 +17766,7 @@ pub fn vqrdmlah_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17780,7 +17780,7 @@ pub fn vqrdmlah_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17795,7 +17795,7 @@ pub fn vqrdmlah_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17809,7 +17809,7 @@ pub fn vqrdmlah_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17836,7 +17836,7 @@ pub fn vqrdmlahq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17851,7 +17851,7 @@ pub fn vqrdmlahq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17878,7 +17878,7 @@ pub fn vqrdmlahq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17893,7 +17893,7 @@ pub fn vqrdmlahq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -17909,7 +17909,7 @@ pub fn vqrdmlah_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -17925,7 +17925,7 @@ pub fn vqrdmlahq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlah_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -17941,7 +17941,7 @@ pub fn vqrdmlah_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -17957,7 +17957,7 @@ pub fn vqrdmlahq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17968,7 +17968,7 @@ pub fn vqrdmlahh_lane_s16(a: i16, b: i16, c: int16x4_t) -> i16
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17979,7 +17979,7 @@ pub fn vqrdmlahh_laneq_s16(a: i16, b: i16, c: int16x8_t) -> i16
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahs_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -17990,7 +17990,7 @@ pub fn vqrdmlahs_lane_s32(a: i32, b: i32, c: int32x2_t) -> i32
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahs_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18001,7 +18001,7 @@ pub fn vqrdmlahs_laneq_s32(a: i32, b: i32, c: int32x4_t) -> i32
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18013,7 +18013,7 @@ pub fn vqrdmlahh_s16(a: i16, b: i16, c: i16) -> i16 {
 }
 #[doc = "Signed saturating rounding doubling multiply accumulate returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlahs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlah))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18025,7 +18025,7 @@ pub fn vqrdmlahs_s32(a: i32, b: i32, c: i32) -> i32 {
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18040,7 +18040,7 @@ pub fn vqrdmlsh_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18054,7 +18054,7 @@ pub fn vqrdmlsh_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18069,7 +18069,7 @@ pub fn vqrdmlsh_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18083,7 +18083,7 @@ pub fn vqrdmlsh_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18110,7 +18110,7 @@ pub fn vqrdmlshq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18125,7 +18125,7 @@ pub fn vqrdmlshq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18152,7 +18152,7 @@ pub fn vqrdmlshq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18167,7 +18167,7 @@ pub fn vqrdmlshq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18183,7 +18183,7 @@ pub fn vqrdmlsh_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18199,7 +18199,7 @@ pub fn vqrdmlshq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlsh_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18215,7 +18215,7 @@ pub fn vqrdmlsh_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18231,7 +18231,7 @@ pub fn vqrdmlshq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18242,7 +18242,7 @@ pub fn vqrdmlshh_lane_s16(a: i16, b: i16, c: int16x4_t) -> i16
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18253,7 +18253,7 @@ pub fn vqrdmlshh_laneq_s16(a: i16, b: i16, c: int16x8_t) -> i16
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshs_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18264,7 +18264,7 @@ pub fn vqrdmlshs_lane_s32(a: i32, b: i32, c: int32x2_t) -> i32
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshs_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh, LANE = 1))]
 #[rustc_legacy_const_generics(3)]
@@ -18275,7 +18275,7 @@ pub fn vqrdmlshs_laneq_s32(a: i32, b: i32, c: int32x4_t) -> i32
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18287,7 +18287,7 @@ pub fn vqrdmlshh_s16(a: i16, b: i16, c: i16) -> i16 {
 }
 #[doc = "Signed saturating rounding doubling multiply subtract returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmlshs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "rdm")]
 #[cfg_attr(test, assert_instr(sqrdmlsh))]
 #[stable(feature = "rdm_intrinsics", since = "1.62.0")]
@@ -18299,7 +18299,7 @@ pub fn vqrdmlshs_s32(a: i32, b: i32, c: i32) -> i32 {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -18310,7 +18310,7 @@ pub fn vqrdmulhh_lane_s16(a: i16, b: int16x4_t) -> i16 {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -18321,7 +18321,7 @@ pub fn vqrdmulhh_laneq_s16(a: i16, b: int16x8_t) -> i16 {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhs_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -18332,7 +18332,7 @@ pub fn vqrdmulhs_lane_s32(a: i32, b: int32x2_t) -> i32 {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhs_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -18343,7 +18343,7 @@ pub fn vqrdmulhs_laneq_s32(a: i32, b: int32x4_t) -> i32 {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrdmulh))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18352,7 +18352,7 @@ pub fn vqrdmulhh_s16(a: i16, b: i16) -> i16 {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrdmulh))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18361,7 +18361,7 @@ pub fn vqrdmulhs_s32(a: i32, b: i32) -> i32 {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18372,7 +18372,7 @@ pub fn vqrshlb_s8(a: i8, b: i8) -> i8 {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18383,7 +18383,7 @@ pub fn vqrshlh_s16(a: i16, b: i16) -> i16 {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlb_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18394,7 +18394,7 @@ pub fn vqrshlb_u8(a: u8, b: i8) -> u8 {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlh_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18405,7 +18405,7 @@ pub fn vqrshlh_u16(a: u16, b: i16) -> u16 {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshld_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18421,7 +18421,7 @@ pub fn vqrshld_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshls_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18437,7 +18437,7 @@ pub fn vqrshls_s32(a: i32, b: i32) -> i32 {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshls_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18453,7 +18453,7 @@ pub fn vqrshls_u32(a: u32, b: i32) -> u32 {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshld_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18469,7 +18469,7 @@ pub fn vqrshld_u64(a: u64, b: i64) -> u64 {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18486,7 +18486,7 @@ pub fn vqrshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18497,7 +18497,7 @@ pub fn vqrshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18508,7 +18508,7 @@ pub fn vqrshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t
 }
 #[doc = "Unsigned saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18525,7 +18525,7 @@ pub fn vqrshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16
 }
 #[doc = "Unsigned saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18536,7 +18536,7 @@ pub fn vqrshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x
 }
 #[doc = "Unsigned saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_high_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18547,7 +18547,7 @@ pub fn vqrshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x
 }
 #[doc = "Unsigned saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnd_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18559,7 +18559,7 @@ pub fn vqrshrnd_n_u64(a: u64) -> u32 {
 }
 #[doc = "Unsigned saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnh_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18571,7 +18571,7 @@ pub fn vqrshrnh_n_u16(a: u16) -> u8 {
 }
 #[doc = "Unsigned saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrns_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqrshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18583,7 +18583,7 @@ pub fn vqrshrns_n_u32(a: u32) -> u16 {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18595,7 +18595,7 @@ pub fn vqrshrnh_n_s16(a: i16) -> i8 {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrns_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18607,7 +18607,7 @@ pub fn vqrshrns_n_s32(a: i32) -> i16 {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrnd_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18619,7 +18619,7 @@ pub fn vqrshrnd_n_s64(a: i64) -> i32 {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrun2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18636,7 +18636,7 @@ pub fn vqrshrun_high_n_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrun2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18647,7 +18647,7 @@ pub fn vqrshrun_high_n_s32(a: uint16x4_t, b: int32x4_t) -> uint16x
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_high_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrun2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18658,7 +18658,7 @@ pub fn vqrshrun_high_n_s64(a: uint32x2_t, b: int64x2_t) -> uint32x
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrund_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrun, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18670,7 +18670,7 @@ pub fn vqrshrund_n_s64(a: i64) -> u32 {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrunh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrun, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18682,7 +18682,7 @@ pub fn vqrshrunh_n_s16(a: i16) -> u8 {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshruns_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqrshrun, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18694,7 +18694,7 @@ pub fn vqrshruns_n_s32(a: i32) -> u16 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18705,7 +18705,7 @@ pub fn vqshlb_n_s8(a: i8) -> i8 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18716,7 +18716,7 @@ pub fn vqshld_n_s64(a: i64) -> i64 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18727,7 +18727,7 @@ pub fn vqshlh_n_s16(a: i16) -> i16 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18738,7 +18738,7 @@ pub fn vqshls_n_s32(a: i32) -> i32 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18749,7 +18749,7 @@ pub fn vqshlb_n_u8(a: u8) -> u8 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18760,7 +18760,7 @@ pub fn vqshld_n_u64(a: u64) -> u64 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18771,7 +18771,7 @@ pub fn vqshlh_n_u16(a: u16) -> u16 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18782,7 +18782,7 @@ pub fn vqshls_n_u32(a: u32) -> u32 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18792,7 +18792,7 @@ pub fn vqshlb_s8(a: i8, b: i8) -> i8 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18802,7 +18802,7 @@ pub fn vqshlh_s16(a: i16, b: i16) -> i16 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18812,7 +18812,7 @@ pub fn vqshls_s32(a: i32, b: i32) -> i32 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlb_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18822,7 +18822,7 @@ pub fn vqshlb_u8(a: u8, b: i8) -> u8 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlh_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18832,7 +18832,7 @@ pub fn vqshlh_u16(a: u16, b: i16) -> u16 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshls_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18842,7 +18842,7 @@ pub fn vqshls_u32(a: u32, b: i32) -> u32 {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18858,7 +18858,7 @@ pub fn vqshld_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshld_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -18874,7 +18874,7 @@ pub fn vqshld_u64(a: u64, b: i64) -> u64 {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlub_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18885,7 +18885,7 @@ pub fn vqshlub_n_s8(a: i8) -> u8 {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlud_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18896,7 +18896,7 @@ pub fn vqshlud_n_s64(a: i64) -> u64 {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18907,7 +18907,7 @@ pub fn vqshluh_n_s16(a: i16) -> u16 {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlus_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -18918,7 +18918,7 @@ pub fn vqshlus_n_s32(a: i32) -> u32 {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18935,7 +18935,7 @@ pub fn vqshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18946,7 +18946,7 @@ pub fn vqshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18957,7 +18957,7 @@ pub fn vqshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18974,7 +18974,7 @@ pub fn vqshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18985,7 +18985,7 @@ pub fn vqshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_high_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -18996,7 +18996,7 @@ pub fn vqshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnd_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19014,7 +19014,7 @@ pub fn vqshrnd_n_s64(a: i64) -> i32 {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnd_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19032,7 +19032,7 @@ pub fn vqshrnd_n_u64(a: u64) -> u32 {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19043,7 +19043,7 @@ pub fn vqshrnh_n_s16(a: i16) -> i8 {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrns_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19054,7 +19054,7 @@ pub fn vqshrns_n_s32(a: i32) -> i16 {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrnh_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19065,7 +19065,7 @@ pub fn vqshrnh_n_u16(a: u16) -> u8 {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrns_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(uqshrn, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19076,7 +19076,7 @@ pub fn vqshrns_n_u32(a: u32) -> u16 {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrun2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -19093,7 +19093,7 @@ pub fn vqshrun_high_n_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrun2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -19104,7 +19104,7 @@ pub fn vqshrun_high_n_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_high_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrun2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -19115,7 +19115,7 @@ pub fn vqshrun_high_n_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrund_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrun, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19126,7 +19126,7 @@ pub fn vqshrund_n_s64(a: i64) -> u32 {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrunh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrun, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19137,7 +19137,7 @@ pub fn vqshrunh_n_s16(a: i16) -> u8 {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshruns_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sqshrun, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -19148,7 +19148,7 @@ pub fn vqshruns_n_s32(a: i32) -> u16 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqsub))]
@@ -19159,7 +19159,7 @@ pub fn vqsubb_s8(a: i8, b: i8) -> i8 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqsub))]
@@ -19170,7 +19170,7 @@ pub fn vqsubh_s16(a: i16, b: i16) -> i16 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubb_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqsub))]
@@ -19181,7 +19181,7 @@ pub fn vqsubb_u8(a: u8, b: u8) -> u8 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubh_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqsub))]
@@ -19192,7 +19192,7 @@ pub fn vqsubh_u16(a: u16, b: u16) -> u16 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqsub))]
@@ -19208,7 +19208,7 @@ pub fn vqsubs_s32(a: i32, b: i32) -> i32 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sqsub))]
@@ -19224,7 +19224,7 @@ pub fn vqsubd_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubs_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqsub))]
@@ -19240,7 +19240,7 @@ pub fn vqsubs_u32(a: u32, b: u32) -> u32 {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(uqsub))]
@@ -19256,7 +19256,7 @@ pub fn vqsubd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19272,7 +19272,7 @@ fn vqtbl1(a: int8x16_t, b: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19288,7 +19288,7 @@ fn vqtbl1q(a: int8x16_t, b: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19297,7 +19297,7 @@ pub fn vqtbl1_s8(a: int8x16_t, b: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19306,7 +19306,7 @@ pub fn vqtbl1q_s8(a: int8x16_t, b: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19315,7 +19315,7 @@ pub fn vqtbl1_u8(a: uint8x16_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19324,7 +19324,7 @@ pub fn vqtbl1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19333,7 +19333,7 @@ pub fn vqtbl1_p8(a: poly8x16_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl1q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19342,7 +19342,7 @@ pub fn vqtbl1q_p8(a: poly8x16_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19358,7 +19358,7 @@ fn vqtbl2(a: int8x16_t, b: int8x16_t, c: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19374,7 +19374,7 @@ fn vqtbl2q(a: int8x16_t, b: int8x16_t, c: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19383,7 +19383,7 @@ pub fn vqtbl2_s8(a: int8x16x2_t, b: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19392,7 +19392,7 @@ pub fn vqtbl2q_s8(a: int8x16x2_t, b: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19402,7 +19402,7 @@ pub fn vqtbl2_u8(a: uint8x16x2_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19431,7 +19431,7 @@ pub fn vqtbl2_u8(a: uint8x16x2_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19441,7 +19441,7 @@ pub fn vqtbl2q_u8(a: uint8x16x2_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19475,7 +19475,7 @@ pub fn vqtbl2q_u8(a: uint8x16x2_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19485,7 +19485,7 @@ pub fn vqtbl2_p8(a: poly8x16x2_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19514,7 +19514,7 @@ pub fn vqtbl2_p8(a: poly8x16x2_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19524,7 +19524,7 @@ pub fn vqtbl2q_p8(a: poly8x16x2_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl2q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19558,7 +19558,7 @@ pub fn vqtbl2q_p8(a: poly8x16x2_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19574,7 +19574,7 @@ fn vqtbl3(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19590,7 +19590,7 @@ fn vqtbl3q(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: uint8x16_t) -> int8x16_t
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19599,7 +19599,7 @@ pub fn vqtbl3_s8(a: int8x16x3_t, b: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19608,7 +19608,7 @@ pub fn vqtbl3q_s8(a: int8x16x3_t, b: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19618,7 +19618,7 @@ pub fn vqtbl3_u8(a: uint8x16x3_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19655,7 +19655,7 @@ pub fn vqtbl3_u8(a: uint8x16x3_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19665,7 +19665,7 @@ pub fn vqtbl3q_u8(a: uint8x16x3_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19707,7 +19707,7 @@ pub fn vqtbl3q_u8(a: uint8x16x3_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19717,7 +19717,7 @@ pub fn vqtbl3_p8(a: poly8x16x3_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19754,7 +19754,7 @@ pub fn vqtbl3_p8(a: poly8x16x3_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19764,7 +19764,7 @@ pub fn vqtbl3q_p8(a: poly8x16x3_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl3q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19806,7 +19806,7 @@ pub fn vqtbl3q_p8(a: poly8x16x3_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19828,7 +19828,7 @@ fn vqtbl4(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, e: uint8x8_t)
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19850,7 +19850,7 @@ fn vqtbl4q(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, e: uint8x16_t
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19859,7 +19859,7 @@ pub fn vqtbl4_s8(a: int8x16x4_t, b: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -19868,7 +19868,7 @@ pub fn vqtbl4q_s8(a: int8x16x4_t, b: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19886,7 +19886,7 @@ pub fn vqtbl4_u8(a: uint8x16x4_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19935,7 +19935,7 @@ pub fn vqtbl4_u8(a: uint8x16x4_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -19953,7 +19953,7 @@ pub fn vqtbl4q_u8(a: uint8x16x4_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -20007,7 +20007,7 @@ pub fn vqtbl4q_u8(a: uint8x16x4_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -20025,7 +20025,7 @@ pub fn vqtbl4_p8(a: poly8x16x4_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -20074,7 +20074,7 @@ pub fn vqtbl4_p8(a: poly8x16x4_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -20092,7 +20092,7 @@ pub fn vqtbl4q_p8(a: poly8x16x4_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbl4q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -20146,7 +20146,7 @@ pub fn vqtbl4q_p8(a: poly8x16x4_t, b: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20162,7 +20162,7 @@ fn vqtbx1(a: int8x8_t, b: int8x16_t, c: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20178,7 +20178,7 @@ fn vqtbx1q(a: int8x16_t, b: int8x16_t, c: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20187,7 +20187,7 @@ pub fn vqtbx1_s8(a: int8x8_t, b: int8x16_t, c: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20196,7 +20196,7 @@ pub fn vqtbx1q_s8(a: int8x16_t, b: int8x16_t, c: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20205,7 +20205,7 @@ pub fn vqtbx1_u8(a: uint8x8_t, b: uint8x16_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20214,7 +20214,7 @@ pub fn vqtbx1q_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20223,7 +20223,7 @@ pub fn vqtbx1_p8(a: poly8x8_t, b: poly8x16_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx1q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20232,7 +20232,7 @@ pub fn vqtbx1q_p8(a: poly8x16_t, b: poly8x16_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20248,7 +20248,7 @@ fn vqtbx2(a: int8x8_t, b: int8x16_t, c: int8x16_t, d: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20264,7 +20264,7 @@ fn vqtbx2q(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: uint8x16_t) -> int8x16_t
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20273,7 +20273,7 @@ pub fn vqtbx2_s8(a: int8x8_t, b: int8x16x2_t, c: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20282,7 +20282,7 @@ pub fn vqtbx2q_s8(a: int8x16_t, b: int8x16x2_t, c: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20292,7 +20292,7 @@ pub fn vqtbx2_u8(a: uint8x8_t, b: uint8x16x2_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20322,7 +20322,7 @@ pub fn vqtbx2_u8(a: uint8x8_t, b: uint8x16x2_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20332,7 +20332,7 @@ pub fn vqtbx2q_u8(a: uint8x16_t, b: uint8x16x2_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20369,7 +20369,7 @@ pub fn vqtbx2q_u8(a: uint8x16_t, b: uint8x16x2_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20379,7 +20379,7 @@ pub fn vqtbx2_p8(a: poly8x8_t, b: poly8x16x2_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20409,7 +20409,7 @@ pub fn vqtbx2_p8(a: poly8x8_t, b: poly8x16x2_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20419,7 +20419,7 @@ pub fn vqtbx2q_p8(a: poly8x16_t, b: poly8x16x2_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx2q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20456,7 +20456,7 @@ pub fn vqtbx2q_p8(a: poly8x16_t, b: poly8x16x2_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20473,7 +20473,7 @@ fn vqtbx3(a: int8x8_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, e: uint8x8_t) -
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20495,7 +20495,7 @@ fn vqtbx3q(a: int8x16_t, b: int8x16_t, c: int8x16_t, d: int8x16_t, e: uint8x16_t
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20504,7 +20504,7 @@ pub fn vqtbx3_s8(a: int8x8_t, b: int8x16x3_t, c: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20513,7 +20513,7 @@ pub fn vqtbx3q_s8(a: int8x16_t, b: int8x16x3_t, c: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20531,7 +20531,7 @@ pub fn vqtbx3_u8(a: uint8x8_t, b: uint8x16x3_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20574,7 +20574,7 @@ pub fn vqtbx3_u8(a: uint8x8_t, b: uint8x16x3_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20592,7 +20592,7 @@ pub fn vqtbx3q_u8(a: uint8x16_t, b: uint8x16x3_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20641,7 +20641,7 @@ pub fn vqtbx3q_u8(a: uint8x16_t, b: uint8x16x3_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20659,7 +20659,7 @@ pub fn vqtbx3_p8(a: poly8x8_t, b: poly8x16x3_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20702,7 +20702,7 @@ pub fn vqtbx3_p8(a: poly8x8_t, b: poly8x16x3_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20720,7 +20720,7 @@ pub fn vqtbx3q_p8(a: poly8x16_t, b: poly8x16x3_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx3q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20769,7 +20769,7 @@ pub fn vqtbx3q_p8(a: poly8x16_t, b: poly8x16x3_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20799,7 +20799,7 @@ fn vqtbx4(
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4q)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20829,7 +20829,7 @@ fn vqtbx4q(
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20838,7 +20838,7 @@ pub fn vqtbx4_s8(a: int8x8_t, b: int8x16x4_t, c: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20847,7 +20847,7 @@ pub fn vqtbx4q_s8(a: int8x16_t, b: int8x16x4_t, c: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20866,7 +20866,7 @@ pub fn vqtbx4_u8(a: uint8x8_t, b: uint8x16x4_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20917,7 +20917,7 @@ pub fn vqtbx4_u8(a: uint8x8_t, b: uint8x16x4_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20936,7 +20936,7 @@ pub fn vqtbx4q_u8(a: uint8x16_t, b: uint8x16x4_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4q_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -20993,7 +20993,7 @@ pub fn vqtbx4q_u8(a: uint8x16_t, b: uint8x16x4_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -21012,7 +21012,7 @@ pub fn vqtbx4_p8(a: poly8x8_t, b: poly8x16x4_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -21063,7 +21063,7 @@ pub fn vqtbx4_p8(a: poly8x8_t, b: poly8x16x4_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -21082,7 +21082,7 @@ pub fn vqtbx4q_p8(a: poly8x16_t, b: poly8x16x4_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqtbx4q_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -21139,7 +21139,7 @@ pub fn vqtbx4q_p8(a: poly8x16_t, b: poly8x16x4_t, c: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Rotate and exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrax1q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[cfg_attr(test, assert_instr(rax1))]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
@@ -21155,7 +21155,7 @@ pub fn vrax1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(rbit))]
@@ -21164,7 +21164,7 @@ pub fn vrbit_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(rbit))]
@@ -21173,7 +21173,7 @@ pub fn vrbitq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21183,7 +21183,7 @@ pub fn vrbit_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21197,7 +21197,7 @@ pub fn vrbit_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21207,7 +21207,7 @@ pub fn vrbitq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21226,7 +21226,7 @@ pub fn vrbitq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21236,7 +21236,7 @@ pub fn vrbit_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbit_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21250,7 +21250,7 @@ pub fn vrbit_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21260,7 +21260,7 @@ pub fn vrbitq_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Reverse bit order"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrbitq_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21279,7 +21279,7 @@ pub fn vrbitq_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecpe))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21295,7 +21295,7 @@ pub fn vrecpe_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecpe))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21311,7 +21311,7 @@ pub fn vrecpeq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecped_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecpe))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21327,7 +21327,7 @@ pub fn vrecped_f64(a: f64) -> f64 {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpes_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecpe))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21343,7 +21343,7 @@ pub fn vrecpes_f32(a: f32) -> f32 {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(frecpe))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -21360,7 +21360,7 @@ pub fn vrecpeh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecps_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21376,7 +21376,7 @@ pub fn vrecps_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21392,7 +21392,7 @@ pub fn vrecpsq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21408,7 +21408,7 @@ pub fn vrecpsd_f64(a: f64, b: f64) -> f64 {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpss_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecps))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21424,7 +21424,7 @@ pub fn vrecpss_f32(a: f32, b: f32) -> f32 {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(frecps))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -21441,7 +21441,7 @@ pub fn vrecpsh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Floating-point reciprocal exponent"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpxd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecpx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21457,7 +21457,7 @@ pub fn vrecpxd_f64(a: f64) -> f64 {
 }
 #[doc = "Floating-point reciprocal exponent"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpxs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frecpx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21473,7 +21473,7 @@ pub fn vrecpxs_f32(a: f32) -> f32 {
 }
 #[doc = "Floating-point reciprocal exponent"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpxh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(frecpx))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -21490,7 +21490,7 @@ pub fn vrecpxh_f16(a: f16) -> f16 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21501,7 +21501,7 @@ pub fn vreinterpret_f64_f16(a: float16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21513,7 +21513,7 @@ pub fn vreinterpret_f64_f16(a: float16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21524,7 +21524,7 @@ pub fn vreinterpretq_f64_f16(a: float16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21539,7 +21539,7 @@ pub fn vreinterpretq_f64_f16(a: float16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21550,7 +21550,7 @@ pub fn vreinterpret_f16_f64(a: float64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21564,7 +21564,7 @@ pub fn vreinterpret_f16_f64(a: float64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21575,7 +21575,7 @@ pub fn vreinterpretq_f16_f64(a: float64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -21590,7 +21590,7 @@ pub fn vreinterpretq_f16_f64(a: float64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21600,7 +21600,7 @@ pub fn vreinterpretq_f64_p128(a: p128) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21613,7 +21613,7 @@ pub fn vreinterpretq_f64_p128(a: p128) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21623,7 +21623,7 @@ pub fn vreinterpret_f64_f32(a: float32x2_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21634,7 +21634,7 @@ pub fn vreinterpret_f64_f32(a: float32x2_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21644,7 +21644,7 @@ pub fn vreinterpret_p64_f32(a: float32x2_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21655,7 +21655,7 @@ pub fn vreinterpret_p64_f32(a: float32x2_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21665,7 +21665,7 @@ pub fn vreinterpretq_f64_f32(a: float32x4_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21679,7 +21679,7 @@ pub fn vreinterpretq_f64_f32(a: float32x4_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21689,7 +21689,7 @@ pub fn vreinterpretq_p64_f32(a: float32x4_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21703,7 +21703,7 @@ pub fn vreinterpretq_p64_f32(a: float32x4_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21713,7 +21713,7 @@ pub fn vreinterpret_f32_f64(a: float64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21726,7 +21726,7 @@ pub fn vreinterpret_f32_f64(a: float64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21736,7 +21736,7 @@ pub fn vreinterpret_s8_f64(a: float64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21749,7 +21749,7 @@ pub fn vreinterpret_s8_f64(a: float64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21759,7 +21759,7 @@ pub fn vreinterpret_s16_f64(a: float64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21772,7 +21772,7 @@ pub fn vreinterpret_s16_f64(a: float64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21782,7 +21782,7 @@ pub fn vreinterpret_s32_f64(a: float64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21795,7 +21795,7 @@ pub fn vreinterpret_s32_f64(a: float64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -21804,7 +21804,7 @@ pub fn vreinterpret_s64_f64(a: float64x1_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21814,7 +21814,7 @@ pub fn vreinterpret_u8_f64(a: float64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21827,7 +21827,7 @@ pub fn vreinterpret_u8_f64(a: float64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21837,7 +21837,7 @@ pub fn vreinterpret_u16_f64(a: float64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21850,7 +21850,7 @@ pub fn vreinterpret_u16_f64(a: float64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21860,7 +21860,7 @@ pub fn vreinterpret_u32_f64(a: float64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21873,7 +21873,7 @@ pub fn vreinterpret_u32_f64(a: float64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -21882,7 +21882,7 @@ pub fn vreinterpret_u64_f64(a: float64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21892,7 +21892,7 @@ pub fn vreinterpret_p8_f64(a: float64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21905,7 +21905,7 @@ pub fn vreinterpret_p8_f64(a: float64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21915,7 +21915,7 @@ pub fn vreinterpret_p16_f64(a: float64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21928,7 +21928,7 @@ pub fn vreinterpret_p16_f64(a: float64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -21937,7 +21937,7 @@ pub fn vreinterpret_p64_f64(a: float64x1_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21947,7 +21947,7 @@ pub fn vreinterpretq_p128_f64(a: float64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21958,7 +21958,7 @@ pub fn vreinterpretq_p128_f64(a: float64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21968,7 +21968,7 @@ pub fn vreinterpretq_f32_f64(a: float64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21982,7 +21982,7 @@ pub fn vreinterpretq_f32_f64(a: float64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21992,7 +21992,7 @@ pub fn vreinterpretq_s8_f64(a: float64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22010,7 +22010,7 @@ pub fn vreinterpretq_s8_f64(a: float64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22020,7 +22020,7 @@ pub fn vreinterpretq_s16_f64(a: float64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22034,7 +22034,7 @@ pub fn vreinterpretq_s16_f64(a: float64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22044,7 +22044,7 @@ pub fn vreinterpretq_s32_f64(a: float64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22058,7 +22058,7 @@ pub fn vreinterpretq_s32_f64(a: float64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22068,7 +22068,7 @@ pub fn vreinterpretq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22082,7 +22082,7 @@ pub fn vreinterpretq_s64_f64(a: float64x2_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22092,7 +22092,7 @@ pub fn vreinterpretq_u8_f64(a: float64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22110,7 +22110,7 @@ pub fn vreinterpretq_u8_f64(a: float64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22120,7 +22120,7 @@ pub fn vreinterpretq_u16_f64(a: float64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22134,7 +22134,7 @@ pub fn vreinterpretq_u16_f64(a: float64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22144,7 +22144,7 @@ pub fn vreinterpretq_u32_f64(a: float64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22158,7 +22158,7 @@ pub fn vreinterpretq_u32_f64(a: float64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22168,7 +22168,7 @@ pub fn vreinterpretq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22182,7 +22182,7 @@ pub fn vreinterpretq_u64_f64(a: float64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22192,7 +22192,7 @@ pub fn vreinterpretq_p8_f64(a: float64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22210,7 +22210,7 @@ pub fn vreinterpretq_p8_f64(a: float64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22220,7 +22220,7 @@ pub fn vreinterpretq_p16_f64(a: float64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22234,7 +22234,7 @@ pub fn vreinterpretq_p16_f64(a: float64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22244,7 +22244,7 @@ pub fn vreinterpretq_p64_f64(a: float64x2_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22258,7 +22258,7 @@ pub fn vreinterpretq_p64_f64(a: float64x2_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22268,7 +22268,7 @@ pub fn vreinterpret_f64_s8(a: int8x8_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22279,7 +22279,7 @@ pub fn vreinterpret_f64_s8(a: int8x8_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22289,7 +22289,7 @@ pub fn vreinterpretq_f64_s8(a: int8x16_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22304,7 +22304,7 @@ pub fn vreinterpretq_f64_s8(a: int8x16_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22314,7 +22314,7 @@ pub fn vreinterpret_f64_s16(a: int16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22325,7 +22325,7 @@ pub fn vreinterpret_f64_s16(a: int16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22335,7 +22335,7 @@ pub fn vreinterpretq_f64_s16(a: int16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22349,7 +22349,7 @@ pub fn vreinterpretq_f64_s16(a: int16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22359,7 +22359,7 @@ pub fn vreinterpret_f64_s32(a: int32x2_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22370,7 +22370,7 @@ pub fn vreinterpret_f64_s32(a: int32x2_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22380,7 +22380,7 @@ pub fn vreinterpretq_f64_s32(a: int32x4_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22394,7 +22394,7 @@ pub fn vreinterpretq_f64_s32(a: int32x4_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22403,7 +22403,7 @@ pub fn vreinterpret_f64_s64(a: int64x1_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22412,7 +22412,7 @@ pub fn vreinterpret_p64_s64(a: int64x1_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22422,7 +22422,7 @@ pub fn vreinterpretq_f64_s64(a: int64x2_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22436,7 +22436,7 @@ pub fn vreinterpretq_f64_s64(a: int64x2_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22446,7 +22446,7 @@ pub fn vreinterpretq_p64_s64(a: int64x2_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22460,7 +22460,7 @@ pub fn vreinterpretq_p64_s64(a: int64x2_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22470,7 +22470,7 @@ pub fn vreinterpret_f64_u8(a: uint8x8_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22481,7 +22481,7 @@ pub fn vreinterpret_f64_u8(a: uint8x8_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22491,7 +22491,7 @@ pub fn vreinterpretq_f64_u8(a: uint8x16_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22506,7 +22506,7 @@ pub fn vreinterpretq_f64_u8(a: uint8x16_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22516,7 +22516,7 @@ pub fn vreinterpret_f64_u16(a: uint16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22527,7 +22527,7 @@ pub fn vreinterpret_f64_u16(a: uint16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22537,7 +22537,7 @@ pub fn vreinterpretq_f64_u16(a: uint16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22551,7 +22551,7 @@ pub fn vreinterpretq_f64_u16(a: uint16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22561,7 +22561,7 @@ pub fn vreinterpret_f64_u32(a: uint32x2_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22572,7 +22572,7 @@ pub fn vreinterpret_f64_u32(a: uint32x2_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22582,7 +22582,7 @@ pub fn vreinterpretq_f64_u32(a: uint32x4_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22596,7 +22596,7 @@ pub fn vreinterpretq_f64_u32(a: uint32x4_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22605,7 +22605,7 @@ pub fn vreinterpret_f64_u64(a: uint64x1_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22614,7 +22614,7 @@ pub fn vreinterpret_p64_u64(a: uint64x1_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22624,7 +22624,7 @@ pub fn vreinterpretq_f64_u64(a: uint64x2_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22638,7 +22638,7 @@ pub fn vreinterpretq_f64_u64(a: uint64x2_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22648,7 +22648,7 @@ pub fn vreinterpretq_p64_u64(a: uint64x2_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22662,7 +22662,7 @@ pub fn vreinterpretq_p64_u64(a: uint64x2_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22672,7 +22672,7 @@ pub fn vreinterpret_f64_p8(a: poly8x8_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22683,7 +22683,7 @@ pub fn vreinterpret_f64_p8(a: poly8x8_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22693,7 +22693,7 @@ pub fn vreinterpretq_f64_p8(a: poly8x16_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22708,7 +22708,7 @@ pub fn vreinterpretq_f64_p8(a: poly8x16_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22718,7 +22718,7 @@ pub fn vreinterpret_f64_p16(a: poly16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22729,7 +22729,7 @@ pub fn vreinterpret_f64_p16(a: poly16x4_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22739,7 +22739,7 @@ pub fn vreinterpretq_f64_p16(a: poly16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22753,7 +22753,7 @@ pub fn vreinterpretq_f64_p16(a: poly16x8_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22763,7 +22763,7 @@ pub fn vreinterpret_f32_p64(a: poly64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22776,7 +22776,7 @@ pub fn vreinterpret_f32_p64(a: poly64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f64_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22785,7 +22785,7 @@ pub fn vreinterpret_f64_p64(a: poly64x1_t) -> float64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22794,7 +22794,7 @@ pub fn vreinterpret_s64_p64(a: poly64x1_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -22803,7 +22803,7 @@ pub fn vreinterpret_u64_p64(a: poly64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22813,7 +22813,7 @@ pub fn vreinterpretq_f32_p64(a: poly64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22827,7 +22827,7 @@ pub fn vreinterpretq_f32_p64(a: poly64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22837,7 +22837,7 @@ pub fn vreinterpretq_f64_p64(a: poly64x2_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f64_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22851,7 +22851,7 @@ pub fn vreinterpretq_f64_p64(a: poly64x2_t) -> float64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22861,7 +22861,7 @@ pub fn vreinterpretq_s64_p64(a: poly64x2_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22875,7 +22875,7 @@ pub fn vreinterpretq_s64_p64(a: poly64x2_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22885,7 +22885,7 @@ pub fn vreinterpretq_u64_p64(a: poly64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22899,7 +22899,7 @@ pub fn vreinterpretq_u64_p64(a: poly64x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point round to 32-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32x_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32x))]
@@ -22915,7 +22915,7 @@ pub fn vrnd32x_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to 32-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32xq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32x))]
@@ -22931,7 +22931,7 @@ pub fn vrnd32xq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to 32-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32xq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32x))]
@@ -22947,7 +22947,7 @@ pub fn vrnd32xq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to 32-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32x_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32x))]
@@ -22963,7 +22963,7 @@ pub fn vrnd32x_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to 32-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32z_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32z))]
@@ -22979,7 +22979,7 @@ pub fn vrnd32z_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to 32-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32zq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32z))]
@@ -22995,7 +22995,7 @@ pub fn vrnd32zq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to 32-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32zq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32z))]
@@ -23011,7 +23011,7 @@ pub fn vrnd32zq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to 32-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd32z_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint32z))]
@@ -23027,7 +23027,7 @@ pub fn vrnd32z_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to 64-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64x_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64x))]
@@ -23043,7 +23043,7 @@ pub fn vrnd64x_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to 64-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64xq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64x))]
@@ -23059,7 +23059,7 @@ pub fn vrnd64xq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to 64-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64xq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64x))]
@@ -23075,7 +23075,7 @@ pub fn vrnd64xq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to 64-bit integer, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64x_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64x))]
@@ -23091,7 +23091,7 @@ pub fn vrnd64x_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to 64-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64z_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64z))]
@@ -23107,7 +23107,7 @@ pub fn vrnd64z_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to 64-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64zq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64z))]
@@ -23123,7 +23123,7 @@ pub fn vrnd64zq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to 64-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64zq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64z))]
@@ -23139,7 +23139,7 @@ pub fn vrnd64zq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to 64-bit integer toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd64z_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,frintts")]
 #[unstable(feature = "stdarch_neon_ftts", issue = "117227")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(frint64z))]
@@ -23155,7 +23155,7 @@ pub fn vrnd64z_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23165,7 +23165,7 @@ pub fn vrnd_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral, toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23175,7 +23175,7 @@ pub fn vrndq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral, toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintz))]
@@ -23184,7 +23184,7 @@ pub fn vrnd_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral, toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintz))]
@@ -23193,7 +23193,7 @@ pub fn vrndq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to integral, toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintz))]
@@ -23202,7 +23202,7 @@ pub fn vrnd_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintz))]
@@ -23211,7 +23211,7 @@ pub fn vrndq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnda_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23221,7 +23221,7 @@ pub fn vrnda_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndaq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23231,7 +23231,7 @@ pub fn vrndaq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnda_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinta))]
@@ -23240,7 +23240,7 @@ pub fn vrnda_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndaq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinta))]
@@ -23249,7 +23249,7 @@ pub fn vrndaq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrnda_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinta))]
@@ -23258,7 +23258,7 @@ pub fn vrnda_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndaq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinta))]
@@ -23267,7 +23267,7 @@ pub fn vrndaq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndah_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23277,7 +23277,7 @@ pub fn vrndah_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to away"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23287,7 +23287,7 @@ pub fn vrndh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndi_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23304,7 +23304,7 @@ pub fn vrndi_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndiq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23321,7 +23321,7 @@ pub fn vrndiq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndi_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinti))]
@@ -23337,7 +23337,7 @@ pub fn vrndi_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndiq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinti))]
@@ -23353,7 +23353,7 @@ pub fn vrndiq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndi_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinti))]
@@ -23369,7 +23369,7 @@ pub fn vrndi_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndiq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frinti))]
@@ -23385,7 +23385,7 @@ pub fn vrndiq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndih_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23402,7 +23402,7 @@ pub fn vrndih_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndm_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23412,7 +23412,7 @@ pub fn vrndm_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndmq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23422,7 +23422,7 @@ pub fn vrndmq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndm_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintm))]
@@ -23431,7 +23431,7 @@ pub fn vrndm_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndmq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintm))]
@@ -23440,7 +23440,7 @@ pub fn vrndmq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndm_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintm))]
@@ -23449,7 +23449,7 @@ pub fn vrndm_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndmq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintm))]
@@ -23458,7 +23458,7 @@ pub fn vrndmq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndmh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23468,7 +23468,7 @@ pub fn vrndmh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndn_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintn))]
@@ -23484,7 +23484,7 @@ pub fn vrndn_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndnq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintn))]
@@ -23500,7 +23500,7 @@ pub fn vrndnq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, toward minus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndnh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23517,7 +23517,7 @@ pub fn vrndnh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndns_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintn))]
@@ -23533,7 +23533,7 @@ pub fn vrndns_f32(a: f32) -> f32 {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndp_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23543,7 +23543,7 @@ pub fn vrndp_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndpq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23553,7 +23553,7 @@ pub fn vrndpq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndp_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintp))]
@@ -23562,7 +23562,7 @@ pub fn vrndp_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndpq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintp))]
@@ -23571,7 +23571,7 @@ pub fn vrndpq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndp_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintp))]
@@ -23580,7 +23580,7 @@ pub fn vrndp_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndpq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintp))]
@@ -23589,7 +23589,7 @@ pub fn vrndpq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, toward plus infinity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndph_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23599,7 +23599,7 @@ pub fn vrndph_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point round to integral exact, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndx_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23609,7 +23609,7 @@ pub fn vrndx_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral exact, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndxq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23619,7 +23619,7 @@ pub fn vrndxq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral exact, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndx_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintx))]
@@ -23628,7 +23628,7 @@ pub fn vrndx_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral exact, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndxq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintx))]
@@ -23637,7 +23637,7 @@ pub fn vrndxq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point round to integral exact, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndx_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintx))]
@@ -23646,7 +23646,7 @@ pub fn vrndx_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point round to integral exact, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndxq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(frintx))]
@@ -23655,7 +23655,7 @@ pub fn vrndxq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndxh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -23665,7 +23665,7 @@ pub fn vrndxh_f16(a: f16) -> f16 {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshld_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(srshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23681,7 +23681,7 @@ pub fn vrshld_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshld_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(urshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23697,7 +23697,7 @@ pub fn vrshld_u64(a: u64, b: i64) -> u64 {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrd_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(srshr, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -23708,7 +23708,7 @@ pub fn vrshrd_n_s64(a: i64) -> i64 {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrd_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(urshr, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -23719,7 +23719,7 @@ pub fn vrshrd_n_u64(a: u64) -> u64 {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(rshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23736,7 +23736,7 @@ pub fn vrshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(rshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23747,7 +23747,7 @@ pub fn vrshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(rshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23758,7 +23758,7 @@ pub fn vrshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(rshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23775,7 +23775,7 @@ pub fn vrshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(rshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23786,7 +23786,7 @@ pub fn vrshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_high_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(rshrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23797,7 +23797,7 @@ pub fn vrshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrte))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23813,7 +23813,7 @@ pub fn vrsqrte_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrte))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23829,7 +23829,7 @@ pub fn vrsqrteq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrted_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrte))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23845,7 +23845,7 @@ pub fn vrsqrted_f64(a: f64) -> f64 {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtes_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrte))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23861,7 +23861,7 @@ pub fn vrsqrtes_f32(a: f32) -> f32 {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(frsqrte))]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -23878,7 +23878,7 @@ pub fn vrsqrteh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrts_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrts))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23894,7 +23894,7 @@ pub fn vrsqrts_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrts))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23910,7 +23910,7 @@ pub fn vrsqrtsq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsd_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrts))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23926,7 +23926,7 @@ pub fn vrsqrtsd_f64(a: f64, b: f64) -> f64 {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtss_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(frsqrts))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -23942,7 +23942,7 @@ pub fn vrsqrtss_f32(a: f32, b: f32) -> f32 {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(test, assert_instr(frsqrts))]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
@@ -23959,7 +23959,7 @@ pub fn vrsqrtsh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Signed rounding shift right and accumulate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsrad_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(srshr, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23971,7 +23971,7 @@ pub fn vrsrad_n_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Unsigned rounding shift right and accumulate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsrad_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(urshr, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -23983,7 +23983,7 @@ pub fn vrsrad_n_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "little")]
 #[cfg_attr(test, assert_instr(rsubhn2))]
@@ -23994,7 +23994,7 @@ pub fn vrsubhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "little")]
 #[cfg_attr(test, assert_instr(rsubhn2))]
@@ -24005,7 +24005,7 @@ pub fn vrsubhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "little")]
 #[cfg_attr(test, assert_instr(rsubhn2))]
@@ -24016,7 +24016,7 @@ pub fn vrsubhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "little")]
 #[cfg_attr(test, assert_instr(rsubhn2))]
@@ -24027,7 +24027,7 @@ pub fn vrsubhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "little")]
 #[cfg_attr(test, assert_instr(rsubhn2))]
@@ -24038,7 +24038,7 @@ pub fn vrsubhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "little")]
 #[cfg_attr(test, assert_instr(rsubhn2))]
@@ -24049,7 +24049,7 @@ pub fn vrsubhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "big")]
 #[cfg_attr(test, assert_instr(rsubhn))]
@@ -24060,7 +24060,7 @@ pub fn vrsubhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "big")]
 #[cfg_attr(test, assert_instr(rsubhn))]
@@ -24071,7 +24071,7 @@ pub fn vrsubhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "big")]
 #[cfg_attr(test, assert_instr(rsubhn))]
@@ -24082,7 +24082,7 @@ pub fn vrsubhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "big")]
 #[cfg_attr(test, assert_instr(rsubhn))]
@@ -24093,7 +24093,7 @@ pub fn vrsubhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "big")]
 #[cfg_attr(test, assert_instr(rsubhn))]
@@ -24104,7 +24104,7 @@ pub fn vrsubhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_endian = "big")]
 #[cfg_attr(test, assert_instr(rsubhn))]
@@ -24115,7 +24115,7 @@ pub fn vrsubhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -24126,7 +24126,7 @@ pub fn vset_lane_f64(a: f64, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -24137,7 +24137,7 @@ pub fn vsetq_lane_f64(a: f64, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "SHA512 hash update part 2"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512h2q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[cfg_attr(test, assert_instr(sha512h2))]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
@@ -24153,7 +24153,7 @@ pub fn vsha512h2q_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t
 }
 #[doc = "SHA512 hash update part 1"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512hq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[cfg_attr(test, assert_instr(sha512h))]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
@@ -24169,7 +24169,7 @@ pub fn vsha512hq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t
 }
 #[doc = "SHA512 schedule update 0"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512su0q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[cfg_attr(test, assert_instr(sha512su0))]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
@@ -24185,7 +24185,7 @@ pub fn vsha512su0q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "SHA512 schedule update 1"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha512su1q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[cfg_attr(test, assert_instr(sha512su1))]
 #[stable(feature = "stdarch_neon_sha3", since = "1.79.0")]
@@ -24201,7 +24201,7 @@ pub fn vsha512su1q_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshld_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sshl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -24210,7 +24210,7 @@ pub fn vshld_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshld_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ushl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -24219,7 +24219,7 @@ pub fn vshld_u64(a: u64, b: i64) -> u64 {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sshll2, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -24233,7 +24233,7 @@ pub fn vshll_high_n_s8(a: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sshll2, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -24247,7 +24247,7 @@ pub fn vshll_high_n_s16(a: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sshll2, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -24261,7 +24261,7 @@ pub fn vshll_high_n_s32(a: int32x4_t) -> int64x2_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ushll2, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -24275,7 +24275,7 @@ pub fn vshll_high_n_u8(a: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ushll2, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -24289,7 +24289,7 @@ pub fn vshll_high_n_u16(a: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ushll2, N = 2))]
 #[rustc_legacy_const_generics(1)]
@@ -24303,7 +24303,7 @@ pub fn vshll_high_n_u32(a: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(shrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -24320,7 +24320,7 @@ pub fn vshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(shrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -24331,7 +24331,7 @@ pub fn vshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(shrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -24342,7 +24342,7 @@ pub fn vshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(shrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -24359,7 +24359,7 @@ pub fn vshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(shrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -24370,7 +24370,7 @@ pub fn vshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_high_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(shrn2, N = 2))]
 #[rustc_legacy_const_generics(2)]
@@ -24381,7 +24381,7 @@ pub fn vshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24399,7 +24399,7 @@ pub fn vsli_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24417,7 +24417,7 @@ pub fn vsliq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24435,7 +24435,7 @@ pub fn vsli_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24453,7 +24453,7 @@ pub fn vsliq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24471,7 +24471,7 @@ pub fn vsli_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24489,7 +24489,7 @@ pub fn vsliq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24507,7 +24507,7 @@ pub fn vsli_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24525,7 +24525,7 @@ pub fn vsliq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24536,7 +24536,7 @@ pub fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24547,7 +24547,7 @@ pub fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24558,7 +24558,7 @@ pub fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24569,7 +24569,7 @@ pub fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24580,7 +24580,7 @@ pub fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24591,7 +24591,7 @@ pub fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24602,7 +24602,7 @@ pub fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24613,7 +24613,7 @@ pub fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24624,7 +24624,7 @@ pub fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24635,7 +24635,7 @@ pub fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24646,7 +24646,7 @@ pub fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24657,7 +24657,7 @@ pub fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24668,7 +24668,7 @@ pub fn vsli_n_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(sli, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -24679,7 +24679,7 @@ pub fn vsliq_n_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Shift left and insert"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vslid_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[rustc_legacy_const_generics(2)]
@@ -24690,7 +24690,7 @@ pub fn vslid_n_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Shift left and insert"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vslid_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[rustc_legacy_const_generics(2)]
@@ -24701,7 +24701,7 @@ pub fn vslid_n_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "SM3PARTW1"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3partw1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3partw1))]
 #[unstable(feature = "stdarch_neon_sm4", issue = "117226")]
@@ -24717,7 +24717,7 @@ pub fn vsm3partw1q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_
 }
 #[doc = "SM3PARTW2"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3partw2q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3partw2))]
 #[unstable(feature = "stdarch_neon_sm4", issue = "117226")]
@@ -24733,7 +24733,7 @@ pub fn vsm3partw2q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_
 }
 #[doc = "SM3SS1"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3ss1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3ss1))]
 #[unstable(feature = "stdarch_neon_sm4", issue = "117226")]
@@ -24749,7 +24749,7 @@ pub fn vsm3ss1q_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "SM3TT1A"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3tt1aq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3tt1a, IMM2 = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -24767,7 +24767,7 @@ pub fn vsm3tt1aq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_
 }
 #[doc = "SM3TT1B"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3tt1bq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3tt1b, IMM2 = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -24785,7 +24785,7 @@ pub fn vsm3tt1bq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_
 }
 #[doc = "SM3TT2A"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3tt2aq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3tt2a, IMM2 = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -24803,7 +24803,7 @@ pub fn vsm3tt2aq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_
 }
 #[doc = "SM3TT2B"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm3tt2bq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm3tt2b, IMM2 = 0))]
 #[rustc_legacy_const_generics(3)]
@@ -24821,7 +24821,7 @@ pub fn vsm3tt2bq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_
 }
 #[doc = "SM4 key"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm4ekeyq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm4ekey))]
 #[unstable(feature = "stdarch_neon_sm4", issue = "117226")]
@@ -24837,7 +24837,7 @@ pub fn vsm4ekeyq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "SM4 encode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsm4eq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sm4")]
 #[cfg_attr(test, assert_instr(sm4e))]
 #[unstable(feature = "stdarch_neon_sm4", issue = "117226")]
@@ -24853,7 +24853,7 @@ pub fn vsm4eq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqadd_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24869,7 +24869,7 @@ pub fn vsqadd_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24885,7 +24885,7 @@ pub fn vsqaddq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqadd_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24901,7 +24901,7 @@ pub fn vsqadd_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24917,7 +24917,7 @@ pub fn vsqaddq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqadd_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24933,7 +24933,7 @@ pub fn vsqadd_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24949,7 +24949,7 @@ pub fn vsqaddq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqadd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24965,7 +24965,7 @@ pub fn vsqadd_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned saturating Accumulate of Signed value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usqadd))]
@@ -24981,7 +24981,7 @@ pub fn vsqaddq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Unsigned saturating accumulate of signed value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddb_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(usqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -24990,7 +24990,7 @@ pub fn vsqaddb_u8(a: u8, b: i8) -> u8 {
 }
 #[doc = "Unsigned saturating accumulate of signed value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddh_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(usqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -24999,7 +24999,7 @@ pub fn vsqaddh_u16(a: u16, b: i16) -> u16 {
 }
 #[doc = "Unsigned saturating accumulate of signed value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqaddd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(usqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25015,7 +25015,7 @@ pub fn vsqaddd_u64(a: u64, b: i64) -> u64 {
 }
 #[doc = "Unsigned saturating accumulate of signed value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqadds_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(usqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25031,7 +25031,7 @@ pub fn vsqadds_u32(a: u32, b: i32) -> u32 {
 }
 #[doc = "Calculates the square root of each lane."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrt_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fsqrt))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -25041,7 +25041,7 @@ pub fn vsqrt_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Calculates the square root of each lane."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrtq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(test, assert_instr(fsqrt))]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
@@ -25051,7 +25051,7 @@ pub fn vsqrtq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Calculates the square root of each lane."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrt_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fsqrt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25060,7 +25060,7 @@ pub fn vsqrt_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Calculates the square root of each lane."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrtq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fsqrt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25069,7 +25069,7 @@ pub fn vsqrtq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Calculates the square root of each lane."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrt_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fsqrt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25078,7 +25078,7 @@ pub fn vsqrt_f64(a: float64x1_t) -> float64x1_t {
 }
 #[doc = "Calculates the square root of each lane."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrtq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(fsqrt))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25087,7 +25087,7 @@ pub fn vsqrtq_f64(a: float64x2_t) -> float64x2_t {
 }
 #[doc = "Floating-point round to integral, using current rounding mode"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsqrth_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -25097,7 +25097,7 @@ pub fn vsqrth_f16(a: f16) -> f16 {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25115,7 +25115,7 @@ pub fn vsri_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25133,7 +25133,7 @@ pub fn vsriq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25151,7 +25151,7 @@ pub fn vsri_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25169,7 +25169,7 @@ pub fn vsriq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25187,7 +25187,7 @@ pub fn vsri_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25205,7 +25205,7 @@ pub fn vsriq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25223,7 +25223,7 @@ pub fn vsri_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25241,7 +25241,7 @@ pub fn vsriq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25252,7 +25252,7 @@ pub fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25263,7 +25263,7 @@ pub fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25274,7 +25274,7 @@ pub fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25285,7 +25285,7 @@ pub fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25296,7 +25296,7 @@ pub fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25307,7 +25307,7 @@ pub fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25318,7 +25318,7 @@ pub fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25329,7 +25329,7 @@ pub fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25340,7 +25340,7 @@ pub fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25351,7 +25351,7 @@ pub fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25362,7 +25362,7 @@ pub fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25373,7 +25373,7 @@ pub fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25384,7 +25384,7 @@ pub fn vsri_n_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(sri, N = 1))]
 #[rustc_legacy_const_generics(2)]
@@ -25395,7 +25395,7 @@ pub fn vsriq_n_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Shift right and insert"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsrid_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[rustc_legacy_const_generics(2)]
@@ -25406,7 +25406,7 @@ pub fn vsrid_n_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Shift right and insert"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsrid_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[rustc_legacy_const_generics(2)]
@@ -25419,7 +25419,7 @@ pub fn vsrid_n_u64(a: u64, b: u64) -> u64 {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25432,7 +25432,7 @@ pub unsafe fn vst1_f16(ptr: *mut f16, a: float16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25445,7 +25445,7 @@ pub unsafe fn vst1q_f16(ptr: *mut f16, a: float16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25457,7 +25457,7 @@ pub unsafe fn vst1_f32(ptr: *mut f32, a: float32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25469,7 +25469,7 @@ pub unsafe fn vst1q_f32(ptr: *mut f32, a: float32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25481,7 +25481,7 @@ pub unsafe fn vst1_f64(ptr: *mut f64, a: float64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25493,7 +25493,7 @@ pub unsafe fn vst1q_f64(ptr: *mut f64, a: float64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25505,7 +25505,7 @@ pub unsafe fn vst1_s8(ptr: *mut i8, a: int8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25517,7 +25517,7 @@ pub unsafe fn vst1q_s8(ptr: *mut i8, a: int8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25529,7 +25529,7 @@ pub unsafe fn vst1_s16(ptr: *mut i16, a: int16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25541,7 +25541,7 @@ pub unsafe fn vst1q_s16(ptr: *mut i16, a: int16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25553,7 +25553,7 @@ pub unsafe fn vst1_s32(ptr: *mut i32, a: int32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25565,7 +25565,7 @@ pub unsafe fn vst1q_s32(ptr: *mut i32, a: int32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25577,7 +25577,7 @@ pub unsafe fn vst1_s64(ptr: *mut i64, a: int64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25589,7 +25589,7 @@ pub unsafe fn vst1q_s64(ptr: *mut i64, a: int64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25601,7 +25601,7 @@ pub unsafe fn vst1_u8(ptr: *mut u8, a: uint8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25613,7 +25613,7 @@ pub unsafe fn vst1q_u8(ptr: *mut u8, a: uint8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25625,7 +25625,7 @@ pub unsafe fn vst1_u16(ptr: *mut u16, a: uint16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25637,7 +25637,7 @@ pub unsafe fn vst1q_u16(ptr: *mut u16, a: uint16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25649,7 +25649,7 @@ pub unsafe fn vst1_u32(ptr: *mut u32, a: uint32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25661,7 +25661,7 @@ pub unsafe fn vst1q_u32(ptr: *mut u32, a: uint32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25673,7 +25673,7 @@ pub unsafe fn vst1_u64(ptr: *mut u64, a: uint64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25685,7 +25685,7 @@ pub unsafe fn vst1q_u64(ptr: *mut u64, a: uint64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25697,7 +25697,7 @@ pub unsafe fn vst1_p8(ptr: *mut p8, a: poly8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25709,7 +25709,7 @@ pub unsafe fn vst1q_p8(ptr: *mut p8, a: poly8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25721,7 +25721,7 @@ pub unsafe fn vst1_p16(ptr: *mut p16, a: poly16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25733,7 +25733,7 @@ pub unsafe fn vst1q_p16(ptr: *mut p16, a: poly16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25745,7 +25745,7 @@ pub unsafe fn vst1_p64(ptr: *mut p64, a: poly64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(str))]
 #[allow(clippy::cast_ptr_alignment)]
@@ -25757,7 +25757,7 @@ pub unsafe fn vst1q_p64(ptr: *mut p64, a: poly64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st1))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25775,7 +25775,7 @@ pub unsafe fn vst1_f64_x2(a: *mut f64, b: float64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st1))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25793,7 +25793,7 @@ pub unsafe fn vst1q_f64_x2(a: *mut f64, b: float64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st1))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25811,7 +25811,7 @@ pub unsafe fn vst1_f64_x3(a: *mut f64, b: float64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st1))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25829,7 +25829,7 @@ pub unsafe fn vst1q_f64_x3(a: *mut f64, b: float64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st1))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25853,7 +25853,7 @@ pub unsafe fn vst1_f64_x4(a: *mut f64, b: float64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st1))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -25877,7 +25877,7 @@ pub unsafe fn vst1q_f64_x4(a: *mut f64, b: float64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -25890,7 +25890,7 @@ pub unsafe fn vst1_lane_f64(a: *mut f64, b: float64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(nop, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -25903,7 +25903,7 @@ pub unsafe fn vst1q_lane_f64(a: *mut f64, b: float64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st1))]
@@ -25921,7 +25921,7 @@ pub unsafe fn vst2_f64(a: *mut f64, b: float64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -25941,7 +25941,7 @@ pub unsafe fn vst2_lane_f64(a: *mut f64, b: float64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -25961,7 +25961,7 @@ pub unsafe fn vst2_lane_s64(a: *mut i64, b: int64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -25974,7 +25974,7 @@ pub unsafe fn vst2_lane_p64(a: *mut p64, b: poly64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -25987,7 +25987,7 @@ pub unsafe fn vst2_lane_u64(a: *mut u64, b: uint64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st2))]
@@ -26005,7 +26005,7 @@ pub unsafe fn vst2q_f64(a: *mut f64, b: float64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st2))]
@@ -26023,7 +26023,7 @@ pub unsafe fn vst2q_s64(a: *mut i64, b: int64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26043,7 +26043,7 @@ pub unsafe fn vst2q_lane_f64(a: *mut f64, b: float64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26063,7 +26063,7 @@ pub unsafe fn vst2q_lane_s8(a: *mut i8, b: int8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26083,7 +26083,7 @@ pub unsafe fn vst2q_lane_s64(a: *mut i64, b: int64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26096,7 +26096,7 @@ pub unsafe fn vst2q_lane_p64(a: *mut p64, b: poly64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26109,7 +26109,7 @@ pub unsafe fn vst2q_lane_u8(a: *mut u8, b: uint8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26122,7 +26122,7 @@ pub unsafe fn vst2q_lane_u64(a: *mut u64, b: uint64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st2, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26135,7 +26135,7 @@ pub unsafe fn vst2q_lane_p8(a: *mut p8, b: poly8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st2))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26146,7 +26146,7 @@ pub unsafe fn vst2q_p64(a: *mut p64, b: poly64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st2))]
@@ -26157,7 +26157,7 @@ pub unsafe fn vst2q_u64(a: *mut u64, b: uint64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -26175,7 +26175,7 @@ pub unsafe fn vst3_f64(a: *mut f64, b: float64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26195,7 +26195,7 @@ pub unsafe fn vst3_lane_f64(a: *mut f64, b: float64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26215,7 +26215,7 @@ pub unsafe fn vst3_lane_s64(a: *mut i64, b: int64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
@@ -26228,7 +26228,7 @@ pub unsafe fn vst3_lane_p64(a: *mut p64, b: poly64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
@@ -26241,7 +26241,7 @@ pub unsafe fn vst3_lane_u64(a: *mut u64, b: uint64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3))]
@@ -26259,7 +26259,7 @@ pub unsafe fn vst3q_f64(a: *mut f64, b: float64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3))]
@@ -26277,7 +26277,7 @@ pub unsafe fn vst3q_s64(a: *mut i64, b: int64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26297,7 +26297,7 @@ pub unsafe fn vst3q_lane_f64(a: *mut f64, b: float64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26317,7 +26317,7 @@ pub unsafe fn vst3q_lane_s8(a: *mut i8, b: int8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26337,7 +26337,7 @@ pub unsafe fn vst3q_lane_s64(a: *mut i64, b: int64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
@@ -26350,7 +26350,7 @@ pub unsafe fn vst3q_lane_p64(a: *mut p64, b: poly64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
@@ -26363,7 +26363,7 @@ pub unsafe fn vst3q_lane_u8(a: *mut u8, b: uint8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
@@ -26376,7 +26376,7 @@ pub unsafe fn vst3q_lane_u64(a: *mut u64, b: uint64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3, LANE = 0))]
@@ -26389,7 +26389,7 @@ pub unsafe fn vst3q_lane_p8(a: *mut p8, b: poly8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st3))]
@@ -26400,7 +26400,7 @@ pub unsafe fn vst3q_p64(a: *mut p64, b: poly64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st3))]
@@ -26411,7 +26411,7 @@ pub unsafe fn vst3q_u64(a: *mut u64, b: uint64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -26429,7 +26429,7 @@ pub unsafe fn vst4_f64(a: *mut f64, b: float64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26456,7 +26456,7 @@ pub unsafe fn vst4_lane_f64(a: *mut f64, b: float64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26483,7 +26483,7 @@ pub unsafe fn vst4_lane_s64(a: *mut i64, b: int64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
@@ -26496,7 +26496,7 @@ pub unsafe fn vst4_lane_p64(a: *mut p64, b: poly64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
@@ -26509,7 +26509,7 @@ pub unsafe fn vst4_lane_u64(a: *mut u64, b: uint64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4))]
@@ -26527,7 +26527,7 @@ pub unsafe fn vst4q_f64(a: *mut f64, b: float64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4))]
@@ -26545,7 +26545,7 @@ pub unsafe fn vst4q_s64(a: *mut i64, b: int64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26572,7 +26572,7 @@ pub unsafe fn vst4q_lane_f64(a: *mut f64, b: float64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26599,7 +26599,7 @@ pub unsafe fn vst4q_lane_s8(a: *mut i8, b: int8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -26626,7 +26626,7 @@ pub unsafe fn vst4q_lane_s64(a: *mut i64, b: int64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
@@ -26639,7 +26639,7 @@ pub unsafe fn vst4q_lane_p64(a: *mut p64, b: poly64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
@@ -26652,7 +26652,7 @@ pub unsafe fn vst4q_lane_u8(a: *mut u8, b: uint8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
@@ -26665,7 +26665,7 @@ pub unsafe fn vst4q_lane_u64(a: *mut u64, b: uint64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4, LANE = 0))]
@@ -26678,7 +26678,7 @@ pub unsafe fn vst4q_lane_p8(a: *mut p8, b: poly8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(test, assert_instr(st4))]
@@ -26689,7 +26689,7 @@ pub unsafe fn vst4q_p64(a: *mut p64, b: poly64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(st4))]
@@ -26698,7 +26698,7 @@ pub unsafe fn vst4q_u64(a: *mut u64, b: uint64x2x4_t) {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fsub))]
@@ -26707,7 +26707,7 @@ pub fn vsub_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(fsub))]
@@ -26716,7 +26716,7 @@ pub fn vsubq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sub))]
@@ -26725,7 +26725,7 @@ pub fn vsubd_s64(a: i64, b: i64) -> i64 {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(sub))]
@@ -26734,7 +26734,7 @@ pub fn vsubd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubh_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[unstable(feature = "stdarch_neon_f16", issue = "136306")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -26744,7 +26744,7 @@ pub fn vsubh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Signed Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ssubl2))]
@@ -26759,7 +26759,7 @@ pub fn vsubl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ssubl2))]
@@ -26774,7 +26774,7 @@ pub fn vsubl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ssubl2))]
@@ -26789,7 +26789,7 @@ pub fn vsubl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usubl2))]
@@ -26804,7 +26804,7 @@ pub fn vsubl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usubl2))]
@@ -26819,7 +26819,7 @@ pub fn vsubl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usubl2))]
@@ -26834,7 +26834,7 @@ pub fn vsubl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Signed Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ssubw2))]
@@ -26846,7 +26846,7 @@ pub fn vsubw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ssubw2))]
@@ -26858,7 +26858,7 @@ pub fn vsubw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(ssubw2))]
@@ -26870,7 +26870,7 @@ pub fn vsubw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usubw2))]
@@ -26882,7 +26882,7 @@ pub fn vsubw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usubw2))]
@@ -26894,7 +26894,7 @@ pub fn vsubw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(usubw2))]
@@ -26906,7 +26906,7 @@ pub fn vsubw_high_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Dot product index form with signed and unsigned integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsudot_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(test, assert_instr(sudot, LANE = 3))]
 #[rustc_legacy_const_generics(3)]
@@ -26921,7 +26921,7 @@ pub fn vsudot_laneq_s32(a: int32x2_t, b: int8x8_t, c: uint8x16_
 }
 #[doc = "Dot product index form with signed and unsigned integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsudotq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(test, assert_instr(sudot, LANE = 3))]
 #[rustc_legacy_const_generics(3)]
@@ -26937,7 +26937,7 @@ pub fn vsudotq_laneq_s32(a: int32x4_t, b: int8x16_t, c: uint8x1
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26950,7 +26950,7 @@ pub fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26959,7 +26959,7 @@ pub fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26968,7 +26968,7 @@ pub fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26977,7 +26977,7 @@ pub fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -26987,7 +26987,7 @@ pub fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27004,7 +27004,7 @@ pub fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27014,7 +27014,7 @@ pub fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27031,7 +27031,7 @@ pub fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27044,7 +27044,7 @@ pub fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27058,7 +27058,7 @@ pub fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27080,7 +27080,7 @@ pub fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27094,7 +27094,7 @@ pub fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27116,7 +27116,7 @@ pub fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27126,7 +27126,7 @@ pub fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27137,7 +27137,7 @@ pub fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27157,7 +27157,7 @@ pub fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27168,7 +27168,7 @@ pub fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbl))]
@@ -27188,7 +27188,7 @@ pub fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27207,7 +27207,7 @@ pub fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27226,7 +27226,7 @@ pub fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27245,7 +27245,7 @@ pub fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27254,7 +27254,7 @@ pub fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27264,7 +27264,7 @@ pub fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27283,7 +27283,7 @@ pub fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27293,7 +27293,7 @@ pub fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27312,7 +27312,7 @@ pub fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27336,7 +27336,7 @@ pub fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27356,7 +27356,7 @@ pub fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27383,7 +27383,7 @@ pub fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27403,7 +27403,7 @@ pub fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27430,7 +27430,7 @@ pub fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27446,7 +27446,7 @@ pub fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27463,7 +27463,7 @@ pub fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27488,7 +27488,7 @@ pub fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27505,7 +27505,7 @@ pub fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tbx))]
@@ -27530,7 +27530,7 @@ pub fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -27540,7 +27540,7 @@ pub fn vtrn1_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -27550,7 +27550,7 @@ pub fn vtrn1q_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27559,7 +27559,7 @@ pub fn vtrn1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27568,7 +27568,7 @@ pub fn vtrn1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27577,7 +27577,7 @@ pub fn vtrn1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27586,7 +27586,7 @@ pub fn vtrn1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27595,7 +27595,7 @@ pub fn vtrn1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27604,7 +27604,7 @@ pub fn vtrn1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -27613,7 +27613,7 @@ pub fn vtrn1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27622,7 +27622,7 @@ pub fn vtrn1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27631,7 +27631,7 @@ pub fn vtrn1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27646,7 +27646,7 @@ pub fn vtrn1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27655,7 +27655,7 @@ pub fn vtrn1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27664,7 +27664,7 @@ pub fn vtrn1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27673,7 +27673,7 @@ pub fn vtrn1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27682,7 +27682,7 @@ pub fn vtrn1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27697,7 +27697,7 @@ pub fn vtrn1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27706,7 +27706,7 @@ pub fn vtrn1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27715,7 +27715,7 @@ pub fn vtrn1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27724,7 +27724,7 @@ pub fn vtrn1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27733,7 +27733,7 @@ pub fn vtrn1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27748,7 +27748,7 @@ pub fn vtrn1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27757,7 +27757,7 @@ pub fn vtrn1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn1q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn1))]
@@ -27766,7 +27766,7 @@ pub fn vtrn1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -27776,7 +27776,7 @@ pub fn vtrn2_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -27786,7 +27786,7 @@ pub fn vtrn2q_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27795,7 +27795,7 @@ pub fn vtrn2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27804,7 +27804,7 @@ pub fn vtrn2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27813,7 +27813,7 @@ pub fn vtrn2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27822,7 +27822,7 @@ pub fn vtrn2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27831,7 +27831,7 @@ pub fn vtrn2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27840,7 +27840,7 @@ pub fn vtrn2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -27849,7 +27849,7 @@ pub fn vtrn2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27858,7 +27858,7 @@ pub fn vtrn2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27867,7 +27867,7 @@ pub fn vtrn2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27882,7 +27882,7 @@ pub fn vtrn2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27891,7 +27891,7 @@ pub fn vtrn2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27900,7 +27900,7 @@ pub fn vtrn2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27909,7 +27909,7 @@ pub fn vtrn2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27918,7 +27918,7 @@ pub fn vtrn2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27933,7 +27933,7 @@ pub fn vtrn2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27942,7 +27942,7 @@ pub fn vtrn2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27951,7 +27951,7 @@ pub fn vtrn2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27960,7 +27960,7 @@ pub fn vtrn2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27969,7 +27969,7 @@ pub fn vtrn2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27984,7 +27984,7 @@ pub fn vtrn2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -27993,7 +27993,7 @@ pub fn vtrn2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Transpose vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn2q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(trn2))]
@@ -28002,7 +28002,7 @@ pub fn vtrn2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmtst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28015,7 +28015,7 @@ pub fn vtst_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmtst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28028,7 +28028,7 @@ pub fn vtstq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmtst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28041,7 +28041,7 @@ pub fn vtst_p64(a: poly64x1_t, b: poly64x1_t) -> uint64x1_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmtst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28054,7 +28054,7 @@ pub fn vtstq_p64(a: poly64x2_t, b: poly64x2_t) -> uint64x2_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmtst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28067,7 +28067,7 @@ pub fn vtst_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(cmtst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28080,7 +28080,7 @@ pub fn vtstq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Compare bitwise test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28089,7 +28089,7 @@ pub fn vtstd_s64(a: i64, b: i64) -> u64 {
 }
 #[doc = "Compare bitwise test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(tst))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28098,7 +28098,7 @@ pub fn vtstd_u64(a: u64, b: u64) -> u64 {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqadd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28114,7 +28114,7 @@ pub fn vuqadd_s8(a: int8x8_t, b: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28130,7 +28130,7 @@ pub fn vuqaddq_s8(a: int8x16_t, b: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqadd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28146,7 +28146,7 @@ pub fn vuqadd_s16(a: int16x4_t, b: uint16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28162,7 +28162,7 @@ pub fn vuqaddq_s16(a: int16x8_t, b: uint16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqadd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28178,7 +28178,7 @@ pub fn vuqadd_s32(a: int32x2_t, b: uint32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28194,7 +28194,7 @@ pub fn vuqaddq_s32(a: int32x4_t, b: uint32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqadd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28210,7 +28210,7 @@ pub fn vuqadd_s64(a: int64x1_t, b: uint64x1_t) -> int64x1_t {
 }
 #[doc = "Signed saturating Accumulate of Unsigned value."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(test, assert_instr(suqadd))]
@@ -28226,7 +28226,7 @@ pub fn vuqaddq_s64(a: int64x2_t, b: uint64x2_t) -> int64x2_t {
 }
 #[doc = "Signed saturating accumulate of unsigned value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddb_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(suqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28235,7 +28235,7 @@ pub fn vuqaddb_s8(a: i8, b: u8) -> i8 {
 }
 #[doc = "Signed saturating accumulate of unsigned value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(suqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28244,7 +28244,7 @@ pub fn vuqaddh_s16(a: i16, b: u16) -> i16 {
 }
 #[doc = "Signed saturating accumulate of unsigned value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqaddd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(suqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28260,7 +28260,7 @@ pub fn vuqaddd_s64(a: i64, b: u64) -> i64 {
 }
 #[doc = "Signed saturating accumulate of unsigned value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuqadds_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(suqadd))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -28276,7 +28276,7 @@ pub fn vuqadds_s32(a: i32, b: u32) -> i32 {
 }
 #[doc = "Dot product index form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdot_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(test, assert_instr(usdot, LANE = 3))]
 #[rustc_legacy_const_generics(3)]
@@ -28291,7 +28291,7 @@ pub fn vusdot_laneq_s32(a: int32x2_t, b: uint8x8_t, c: int8x16_
 }
 #[doc = "Dot product index form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdotq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(test, assert_instr(usdot, LANE = 3))]
 #[rustc_legacy_const_generics(3)]
@@ -28307,7 +28307,7 @@ pub fn vusdotq_laneq_s32(a: int32x4_t, b: uint8x16_t, c: int8x1
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -28317,7 +28317,7 @@ pub fn vuzp1_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -28327,7 +28327,7 @@ pub fn vuzp1q_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28336,7 +28336,7 @@ pub fn vuzp1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28345,7 +28345,7 @@ pub fn vuzp1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28354,7 +28354,7 @@ pub fn vuzp1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28363,7 +28363,7 @@ pub fn vuzp1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28372,7 +28372,7 @@ pub fn vuzp1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28381,7 +28381,7 @@ pub fn vuzp1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28390,7 +28390,7 @@ pub fn vuzp1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28399,7 +28399,7 @@ pub fn vuzp1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28408,7 +28408,7 @@ pub fn vuzp1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28423,7 +28423,7 @@ pub fn vuzp1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28432,7 +28432,7 @@ pub fn vuzp1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28441,7 +28441,7 @@ pub fn vuzp1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28450,7 +28450,7 @@ pub fn vuzp1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28459,7 +28459,7 @@ pub fn vuzp1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28474,7 +28474,7 @@ pub fn vuzp1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28483,7 +28483,7 @@ pub fn vuzp1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28492,7 +28492,7 @@ pub fn vuzp1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28501,7 +28501,7 @@ pub fn vuzp1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28510,7 +28510,7 @@ pub fn vuzp1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28525,7 +28525,7 @@ pub fn vuzp1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28534,7 +28534,7 @@ pub fn vuzp1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp1q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp1))]
@@ -28543,7 +28543,7 @@ pub fn vuzp1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -28553,7 +28553,7 @@ pub fn vuzp2_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -28563,7 +28563,7 @@ pub fn vuzp2q_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28572,7 +28572,7 @@ pub fn vuzp2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28581,7 +28581,7 @@ pub fn vuzp2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28590,7 +28590,7 @@ pub fn vuzp2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28599,7 +28599,7 @@ pub fn vuzp2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28608,7 +28608,7 @@ pub fn vuzp2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28617,7 +28617,7 @@ pub fn vuzp2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -28626,7 +28626,7 @@ pub fn vuzp2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28635,7 +28635,7 @@ pub fn vuzp2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28644,7 +28644,7 @@ pub fn vuzp2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28659,7 +28659,7 @@ pub fn vuzp2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28668,7 +28668,7 @@ pub fn vuzp2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28677,7 +28677,7 @@ pub fn vuzp2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28686,7 +28686,7 @@ pub fn vuzp2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28695,7 +28695,7 @@ pub fn vuzp2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28710,7 +28710,7 @@ pub fn vuzp2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28719,7 +28719,7 @@ pub fn vuzp2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28728,7 +28728,7 @@ pub fn vuzp2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28737,7 +28737,7 @@ pub fn vuzp2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28746,7 +28746,7 @@ pub fn vuzp2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28761,7 +28761,7 @@ pub fn vuzp2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28770,7 +28770,7 @@ pub fn vuzp2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp2q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(uzp2))]
@@ -28779,7 +28779,7 @@ pub fn vuzp2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Exclusive OR and rotate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vxarq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,sha3")]
 #[cfg_attr(test, assert_instr(xar, IMM6 = 0))]
 #[rustc_legacy_const_generics(2)]
@@ -28797,7 +28797,7 @@ pub fn vxarq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -28807,7 +28807,7 @@ pub fn vzip1_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -28817,7 +28817,7 @@ pub fn vzip1q_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28826,7 +28826,7 @@ pub fn vzip1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28835,7 +28835,7 @@ pub fn vzip1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28844,7 +28844,7 @@ pub fn vzip1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28853,7 +28853,7 @@ pub fn vzip1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28868,7 +28868,7 @@ pub fn vzip1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28877,7 +28877,7 @@ pub fn vzip1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28886,7 +28886,7 @@ pub fn vzip1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28895,7 +28895,7 @@ pub fn vzip1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28904,7 +28904,7 @@ pub fn vzip1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28913,7 +28913,7 @@ pub fn vzip1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28922,7 +28922,7 @@ pub fn vzip1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28937,7 +28937,7 @@ pub fn vzip1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28946,7 +28946,7 @@ pub fn vzip1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28955,7 +28955,7 @@ pub fn vzip1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28964,7 +28964,7 @@ pub fn vzip1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28973,7 +28973,7 @@ pub fn vzip1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28982,7 +28982,7 @@ pub fn vzip1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -28991,7 +28991,7 @@ pub fn vzip1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -29006,7 +29006,7 @@ pub fn vzip1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -29015,7 +29015,7 @@ pub fn vzip1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -29024,7 +29024,7 @@ pub fn vzip1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip1q_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip1))]
@@ -29033,7 +29033,7 @@ pub fn vzip1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -29043,7 +29043,7 @@ pub fn vzip2_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[stable(feature = "stdarch_neon_fp16", since = "CURRENT_RUSTC_VERSION")]
 #[cfg(not(target_arch = "arm64ec"))]
@@ -29053,7 +29053,7 @@ pub fn vzip2q_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29062,7 +29062,7 @@ pub fn vzip2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29071,7 +29071,7 @@ pub fn vzip2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_f64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29080,7 +29080,7 @@ pub fn vzip2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29089,7 +29089,7 @@ pub fn vzip2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29104,7 +29104,7 @@ pub fn vzip2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29113,7 +29113,7 @@ pub fn vzip2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29122,7 +29122,7 @@ pub fn vzip2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29131,7 +29131,7 @@ pub fn vzip2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29140,7 +29140,7 @@ pub fn vzip2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29149,7 +29149,7 @@ pub fn vzip2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29158,7 +29158,7 @@ pub fn vzip2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29173,7 +29173,7 @@ pub fn vzip2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29182,7 +29182,7 @@ pub fn vzip2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29191,7 +29191,7 @@ pub fn vzip2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29200,7 +29200,7 @@ pub fn vzip2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29209,7 +29209,7 @@ pub fn vzip2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29218,7 +29218,7 @@ pub fn vzip2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29227,7 +29227,7 @@ pub fn vzip2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29242,7 +29242,7 @@ pub fn vzip2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29251,7 +29251,7 @@ pub fn vzip2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
@@ -29260,7 +29260,7 @@ pub fn vzip2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip2q_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(zip2))]
diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs
index 0aeb740efcd0..324ff73f62a1 100644
--- a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs
+++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs
@@ -14,7 +14,7 @@ use super::*;
 
 #[doc = "CRC32 single round checksum for bytes (8 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32b)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(crc32b))]
@@ -39,7 +39,7 @@ pub fn __crc32b(crc: u32, data: u8) -> u32 {
 }
 #[doc = "CRC32-C single round checksum for bytes (8 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cb)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(crc32cb))]
@@ -64,7 +64,7 @@ pub fn __crc32cb(crc: u32, data: u8) -> u32 {
 }
 #[doc = "CRC32-C single round checksum for quad words (64 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cd)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(crc32cw))]
@@ -83,7 +83,7 @@ pub fn __crc32cd(crc: u32, data: u64) -> u32 {
 }
 #[doc = "CRC32-C single round checksum for bytes (16 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32ch)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(crc32ch))]
@@ -108,7 +108,7 @@ pub fn __crc32ch(crc: u32, data: u16) -> u32 {
 }
 #[doc = "CRC32-C single round checksum for bytes (32 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32cw)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(crc32cw))]
@@ -133,7 +133,7 @@ pub fn __crc32cw(crc: u32, data: u32) -> u32 {
 }
 #[doc = "CRC32 single round checksum for quad words (64 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32d)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(crc32w))]
@@ -152,7 +152,7 @@ pub fn __crc32d(crc: u32, data: u64) -> u32 {
 }
 #[doc = "CRC32 single round checksum for bytes (16 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32h)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(crc32h))]
@@ -177,7 +177,7 @@ pub fn __crc32h(crc: u32, data: u16) -> u32 {
 }
 #[doc = "CRC32 single round checksum for bytes (32 bits)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/__crc32w)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "crc")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(crc32w))]
@@ -202,7 +202,7 @@ pub fn __crc32w(crc: u32, data: u32) -> u32 {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadal_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -220,7 +220,7 @@ fn priv_vpadal_s8(a: int16x4_t, b: int8x8_t) -> int16x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadalq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -238,7 +238,7 @@ fn priv_vpadalq_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadal_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -256,7 +256,7 @@ fn priv_vpadal_s16(a: int32x2_t, b: int16x4_t) -> int32x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadalq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -274,7 +274,7 @@ fn priv_vpadalq_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadal_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -292,7 +292,7 @@ fn priv_vpadal_s32(a: int64x1_t, b: int32x2_t) -> int64x1_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadalq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -310,7 +310,7 @@ fn priv_vpadalq_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadal_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -328,7 +328,7 @@ fn priv_vpadal_u8(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadalq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -346,7 +346,7 @@ fn priv_vpadalq_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadal_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -364,7 +364,7 @@ fn priv_vpadal_u16(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadalq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -382,7 +382,7 @@ fn priv_vpadalq_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadal_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -400,7 +400,7 @@ fn priv_vpadal_u32(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/priv_vpadalq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -418,7 +418,7 @@ fn priv_vpadalq_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Absolute difference and accumulate (64-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaba_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s16"))]
@@ -439,7 +439,7 @@ pub fn vaba_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t {
 }
 #[doc = "Absolute difference and accumulate (64-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaba_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s32"))]
@@ -460,7 +460,7 @@ pub fn vaba_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t {
 }
 #[doc = "Absolute difference and accumulate (64-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaba_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s8"))]
@@ -481,7 +481,7 @@ pub fn vaba_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Absolute difference and accumulate (64-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaba_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u16"))]
@@ -502,7 +502,7 @@ pub fn vaba_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Absolute difference and accumulate (64-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaba_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u32"))]
@@ -523,7 +523,7 @@ pub fn vaba_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Absolute difference and accumulate (64-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaba_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u8"))]
@@ -544,7 +544,7 @@ pub fn vaba_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s8"))]
@@ -569,7 +569,7 @@ pub fn vabal_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s16"))]
@@ -594,7 +594,7 @@ pub fn vabal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s32"))]
@@ -619,7 +619,7 @@ pub fn vabal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u8"))]
@@ -641,7 +641,7 @@ pub fn vabal_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u16"))]
@@ -663,7 +663,7 @@ pub fn vabal_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Absolute difference and Accumulate Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabal_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u32"))]
@@ -685,7 +685,7 @@ pub fn vabal_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Absolute difference and accumulate (128-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabaq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s16"))]
@@ -706,7 +706,7 @@ pub fn vabaq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Absolute difference and accumulate (128-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabaq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s32"))]
@@ -727,7 +727,7 @@ pub fn vabaq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Absolute difference and accumulate (128-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabaq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s8"))]
@@ -748,7 +748,7 @@ pub fn vabaq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t {
 }
 #[doc = "Absolute difference and accumulate (128-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabaq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u16"))]
@@ -769,7 +769,7 @@ pub fn vabaq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Absolute difference and accumulate (128-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabaq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u32"))]
@@ -790,7 +790,7 @@ pub fn vabaq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Absolute difference and accumulate (128-bit)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabaq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u8"))]
@@ -811,7 +811,7 @@ pub fn vabaq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Absolute difference between the arguments of Floating"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f16"))]
 #[cfg_attr(
@@ -841,7 +841,7 @@ pub fn vabd_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Absolute difference between the arguments of Floating"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f16"))]
 #[cfg_attr(
@@ -871,7 +871,7 @@ pub fn vabdq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Absolute difference between the arguments of Floating"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f32"))]
@@ -900,7 +900,7 @@ pub fn vabd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Absolute difference between the arguments of Floating"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f32"))]
@@ -929,7 +929,7 @@ pub fn vabdq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s8"))]
@@ -958,7 +958,7 @@ pub fn vabd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s8"))]
@@ -987,7 +987,7 @@ pub fn vabdq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s16"))]
@@ -1016,7 +1016,7 @@ pub fn vabd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s16"))]
@@ -1045,7 +1045,7 @@ pub fn vabdq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s32"))]
@@ -1074,7 +1074,7 @@ pub fn vabd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s32"))]
@@ -1103,7 +1103,7 @@ pub fn vabdq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u8"))]
@@ -1132,7 +1132,7 @@ pub fn vabd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u8"))]
@@ -1161,7 +1161,7 @@ pub fn vabdq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u16"))]
@@ -1190,7 +1190,7 @@ pub fn vabd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u16"))]
@@ -1219,7 +1219,7 @@ pub fn vabdq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabd_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u32"))]
@@ -1248,7 +1248,7 @@ pub fn vabd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Absolute difference between the arguments"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u32"))]
@@ -1277,7 +1277,7 @@ pub fn vabdq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Signed Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s8"))]
@@ -1301,7 +1301,7 @@ pub fn vabdl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s16"))]
@@ -1325,7 +1325,7 @@ pub fn vabdl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s32"))]
@@ -1349,7 +1349,7 @@ pub fn vabdl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u8"))]
@@ -1370,7 +1370,7 @@ pub fn vabdl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u16"))]
@@ -1391,7 +1391,7 @@ pub fn vabdl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Absolute difference Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabdl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u32"))]
@@ -1412,7 +1412,7 @@ pub fn vabdl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
 #[cfg_attr(
@@ -1434,7 +1434,7 @@ pub fn vabs_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
 #[cfg_attr(
@@ -1456,7 +1456,7 @@ pub fn vabsq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1477,7 +1477,7 @@ pub fn vabs_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1498,7 +1498,7 @@ pub fn vabsq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Absolute value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1523,7 +1523,7 @@ pub fn vabs_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Absolute value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1548,7 +1548,7 @@ pub fn vabsq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Absolute value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1573,7 +1573,7 @@ pub fn vabs_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Absolute value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1598,7 +1598,7 @@ pub fn vabsq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Absolute value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1623,7 +1623,7 @@ pub fn vabs_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Absolute value (wrapping)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
@@ -1648,7 +1648,7 @@ pub fn vabsq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vabsh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))]
 #[cfg_attr(
@@ -1663,7 +1663,7 @@ pub fn vabsh_f16(a: f16) -> f16 {
 }
 #[doc = "Floating-point Add (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vadd.f16"))]
 #[cfg_attr(
@@ -1685,7 +1685,7 @@ pub fn vadd_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point Add (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vadd.f16"))]
 #[cfg_attr(
@@ -1707,7 +1707,7 @@ pub fn vaddq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1728,7 +1728,7 @@ pub fn vadd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1749,7 +1749,7 @@ pub fn vadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1770,7 +1770,7 @@ pub fn vadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1791,7 +1791,7 @@ pub fn vadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1812,7 +1812,7 @@ pub fn vadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1833,7 +1833,7 @@ pub fn vadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1854,7 +1854,7 @@ pub fn vadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1875,7 +1875,7 @@ pub fn vaddq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1896,7 +1896,7 @@ pub fn vaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1917,7 +1917,7 @@ pub fn vaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1938,7 +1938,7 @@ pub fn vaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1959,7 +1959,7 @@ pub fn vaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -1980,7 +1980,7 @@ pub fn vaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -2001,7 +2001,7 @@ pub fn vaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -2022,7 +2022,7 @@ pub fn vaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector add."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vadd))]
@@ -2043,7 +2043,7 @@ pub fn vaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2064,7 +2064,7 @@ pub fn vadd_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2085,7 +2085,7 @@ pub fn vaddq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2106,7 +2106,7 @@ pub fn vadd_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2127,7 +2127,7 @@ pub fn vaddq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vadd_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2148,7 +2148,7 @@ pub fn vadd_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x1_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2169,7 +2169,7 @@ pub fn vaddq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "Add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddh_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vadd.f16"))]
 #[cfg_attr(
@@ -2184,7 +2184,7 @@ pub fn vaddh_f16(a: f16, b: f16) -> f16 {
 }
 #[doc = "Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2208,7 +2208,7 @@ pub fn vaddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x16_t {
 }
 #[doc = "Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2232,7 +2232,7 @@ pub fn vaddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int16x8_t {
 }
 #[doc = "Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2256,7 +2256,7 @@ pub fn vaddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int32x4_t {
 }
 #[doc = "Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2280,7 +2280,7 @@ pub fn vaddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> uint8x16_t
 }
 #[doc = "Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2304,7 +2304,7 @@ pub fn vaddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> uint16x8_
 }
 #[doc = "Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2328,7 +2328,7 @@ pub fn vaddhn_high_u64(r: uint32x2_t, a: uint64x2_t, b: uint64x2_t) -> uint32x4_
 }
 #[doc = "Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2349,7 +2349,7 @@ pub fn vaddhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t {
 }
 #[doc = "Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2370,7 +2370,7 @@ pub fn vaddhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t {
 }
 #[doc = "Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2391,7 +2391,7 @@ pub fn vaddhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t {
 }
 #[doc = "Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2412,7 +2412,7 @@ pub fn vaddhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2433,7 +2433,7 @@ pub fn vaddhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddhn_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddhn))]
@@ -2454,7 +2454,7 @@ pub fn vaddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed Add Long (vector, high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2481,7 +2481,7 @@ pub fn vaddl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Add Long (vector, high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2508,7 +2508,7 @@ pub fn vaddl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Signed Add Long (vector, high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2535,7 +2535,7 @@ pub fn vaddl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Add Long (vector, high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2562,7 +2562,7 @@ pub fn vaddl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Signed Add Long (vector, high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2589,7 +2589,7 @@ pub fn vaddl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Signed Add Long (vector, high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2616,7 +2616,7 @@ pub fn vaddl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Add Long (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2641,7 +2641,7 @@ pub fn vaddl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Add Long (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2666,7 +2666,7 @@ pub fn vaddl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Add Long (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2691,7 +2691,7 @@ pub fn vaddl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t {
 }
 #[doc = "Add Long (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2716,7 +2716,7 @@ pub fn vaddl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Add Long (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2741,7 +2741,7 @@ pub fn vaddl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Add Long (vector)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))]
@@ -2766,7 +2766,7 @@ pub fn vaddl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Bitwise exclusive OR"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddq_p128)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -2787,7 +2787,7 @@ pub fn vaddq_p128(a: p128, b: p128) -> p128 {
 }
 #[doc = "Add Wide (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2812,7 +2812,7 @@ pub fn vaddw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Add Wide (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2837,7 +2837,7 @@ pub fn vaddw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Add Wide (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2862,7 +2862,7 @@ pub fn vaddw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Add Wide (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2887,7 +2887,7 @@ pub fn vaddw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Add Wide (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2912,7 +2912,7 @@ pub fn vaddw_high_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Add Wide (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2937,7 +2937,7 @@ pub fn vaddw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Add Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2961,7 +2961,7 @@ pub fn vaddw_s16(a: int32x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Add Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -2985,7 +2985,7 @@ pub fn vaddw_s32(a: int64x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Add Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -3009,7 +3009,7 @@ pub fn vaddw_s8(a: int16x8_t, b: int8x8_t) -> int16x8_t {
 }
 #[doc = "Add Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -3033,7 +3033,7 @@ pub fn vaddw_u16(a: uint32x4_t, b: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Add Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -3057,7 +3057,7 @@ pub fn vaddw_u32(a: uint64x2_t, b: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Add Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaddw_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))]
@@ -3081,7 +3081,7 @@ pub fn vaddw_u8(a: uint16x8_t, b: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "AES single round encryption."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaesdq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(aesd))]
@@ -3106,7 +3106,7 @@ pub fn vaesdq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "AES single round encryption."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaeseq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(aese))]
@@ -3131,7 +3131,7 @@ pub fn vaeseq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "AES inverse mix columns."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaesimcq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(aesimc))]
@@ -3156,7 +3156,7 @@ pub fn vaesimcq_u8(data: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "AES mix columns."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vaesmcq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(aesmc))]
@@ -3181,7 +3181,7 @@ pub fn vaesmcq_u8(data: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3202,7 +3202,7 @@ pub fn vand_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3223,7 +3223,7 @@ pub fn vandq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3244,7 +3244,7 @@ pub fn vand_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3265,7 +3265,7 @@ pub fn vandq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3286,7 +3286,7 @@ pub fn vand_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3307,7 +3307,7 @@ pub fn vandq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3328,7 +3328,7 @@ pub fn vand_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3349,7 +3349,7 @@ pub fn vandq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3370,7 +3370,7 @@ pub fn vand_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3391,7 +3391,7 @@ pub fn vandq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3412,7 +3412,7 @@ pub fn vand_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3433,7 +3433,7 @@ pub fn vandq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3454,7 +3454,7 @@ pub fn vand_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3475,7 +3475,7 @@ pub fn vandq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vand_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3496,7 +3496,7 @@ pub fn vand_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector bitwise and"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vandq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))]
@@ -3517,7 +3517,7 @@ pub fn vandq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3539,7 +3539,7 @@ pub fn vbic_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3561,7 +3561,7 @@ pub fn vbic_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3583,7 +3583,7 @@ pub fn vbic_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3605,7 +3605,7 @@ pub fn vbic_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3627,7 +3627,7 @@ pub fn vbicq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3649,7 +3649,7 @@ pub fn vbicq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3671,7 +3671,7 @@ pub fn vbicq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3693,7 +3693,7 @@ pub fn vbicq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3715,7 +3715,7 @@ pub fn vbic_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3737,7 +3737,7 @@ pub fn vbic_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3759,7 +3759,7 @@ pub fn vbic_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbic_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3781,7 +3781,7 @@ pub fn vbic_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3803,7 +3803,7 @@ pub fn vbicq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3825,7 +3825,7 @@ pub fn vbicq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3847,7 +3847,7 @@ pub fn vbicq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector bitwise bit clear."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbicq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))]
@@ -3869,7 +3869,7 @@ pub fn vbicq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -3897,7 +3897,7 @@ pub fn vbsl_f16(a: uint16x4_t, b: float16x4_t, c: float16x4_t) -> float16x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -3925,7 +3925,7 @@ pub fn vbslq_f16(a: uint16x8_t, b: float16x8_t, c: float16x8_t) -> float16x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -3952,7 +3952,7 @@ pub fn vbsl_f32(a: uint32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -3979,7 +3979,7 @@ pub fn vbsl_p16(a: uint16x4_t, b: poly16x4_t, c: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4006,7 +4006,7 @@ pub fn vbsl_p8(a: uint8x8_t, b: poly8x8_t, c: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4033,7 +4033,7 @@ pub fn vbsl_s16(a: uint16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4060,7 +4060,7 @@ pub fn vbsl_s32(a: uint32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4087,7 +4087,7 @@ pub fn vbsl_s64(a: uint64x1_t, b: int64x1_t, c: int64x1_t) -> int64x1_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4114,7 +4114,7 @@ pub fn vbsl_s8(a: uint8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4141,7 +4141,7 @@ pub fn vbslq_f32(a: uint32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4168,7 +4168,7 @@ pub fn vbslq_p16(a: uint16x8_t, b: poly16x8_t, c: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4195,7 +4195,7 @@ pub fn vbslq_p8(a: uint8x16_t, b: poly8x16_t, c: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4222,7 +4222,7 @@ pub fn vbslq_s16(a: uint16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4249,7 +4249,7 @@ pub fn vbslq_s32(a: uint32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4276,7 +4276,7 @@ pub fn vbslq_s64(a: uint64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4303,7 +4303,7 @@ pub fn vbslq_s8(a: uint8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4330,7 +4330,7 @@ pub fn vbsl_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4357,7 +4357,7 @@ pub fn vbsl_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4384,7 +4384,7 @@ pub fn vbsl_u64(a: uint64x1_t, b: uint64x1_t, c: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbsl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4411,7 +4411,7 @@ pub fn vbsl_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4438,7 +4438,7 @@ pub fn vbslq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4465,7 +4465,7 @@ pub fn vbslq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4492,7 +4492,7 @@ pub fn vbslq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Bitwise Select."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vbslq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))]
@@ -4519,7 +4519,7 @@ pub fn vbslq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcage_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f16"))]
 #[cfg_attr(
@@ -4549,7 +4549,7 @@ pub fn vcage_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcageq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f16"))]
 #[cfg_attr(
@@ -4579,7 +4579,7 @@ pub fn vcageq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcage_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))]
@@ -4608,7 +4608,7 @@ pub fn vcage_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point absolute compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcageq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))]
@@ -4637,7 +4637,7 @@ pub fn vcageq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagt_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f16"))]
 #[cfg_attr(
@@ -4667,7 +4667,7 @@ pub fn vcagt_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f16"))]
 #[cfg_attr(
@@ -4697,7 +4697,7 @@ pub fn vcagtq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagt_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))]
@@ -4726,7 +4726,7 @@ pub fn vcagt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point absolute compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcagtq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))]
@@ -4755,7 +4755,7 @@ pub fn vcagtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcale_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f16"))]
 #[cfg_attr(
@@ -4777,7 +4777,7 @@ pub fn vcale_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaleq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f16"))]
 #[cfg_attr(
@@ -4799,7 +4799,7 @@ pub fn vcaleq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcale_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))]
@@ -4820,7 +4820,7 @@ pub fn vcale_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point absolute compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaleq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))]
@@ -4841,7 +4841,7 @@ pub fn vcaleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalt_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f16"))]
 #[cfg_attr(
@@ -4863,7 +4863,7 @@ pub fn vcalt_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f16"))]
 #[cfg_attr(
@@ -4885,7 +4885,7 @@ pub fn vcaltq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcalt_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))]
@@ -4906,7 +4906,7 @@ pub fn vcalt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point absolute compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcaltq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))]
@@ -4927,7 +4927,7 @@ pub fn vcaltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f16"))]
 #[cfg_attr(
@@ -4949,7 +4949,7 @@ pub fn vceq_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f16"))]
 #[cfg_attr(
@@ -4971,7 +4971,7 @@ pub fn vceqq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))]
@@ -4992,7 +4992,7 @@ pub fn vceq_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))]
@@ -5013,7 +5013,7 @@ pub fn vceqq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))]
@@ -5034,7 +5034,7 @@ pub fn vceq_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))]
@@ -5055,7 +5055,7 @@ pub fn vceqq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))]
@@ -5076,7 +5076,7 @@ pub fn vceq_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))]
@@ -5097,7 +5097,7 @@ pub fn vceqq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))]
@@ -5118,7 +5118,7 @@ pub fn vceq_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))]
@@ -5139,7 +5139,7 @@ pub fn vceqq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))]
@@ -5160,7 +5160,7 @@ pub fn vceq_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))]
@@ -5181,7 +5181,7 @@ pub fn vceqq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))]
@@ -5202,7 +5202,7 @@ pub fn vceq_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))]
@@ -5223,7 +5223,7 @@ pub fn vceqq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))]
@@ -5244,7 +5244,7 @@ pub fn vceq_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))]
@@ -5265,7 +5265,7 @@ pub fn vceqq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))]
@@ -5286,7 +5286,7 @@ pub fn vceq_p8(a: poly8x8_t, b: poly8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare bitwise Equal (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vceqq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))]
@@ -5307,7 +5307,7 @@ pub fn vceqq_p8(a: poly8x16_t, b: poly8x16_t) -> uint8x16_t {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f16"))]
 #[cfg_attr(
@@ -5329,7 +5329,7 @@ pub fn vcge_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f16"))]
 #[cfg_attr(
@@ -5351,7 +5351,7 @@ pub fn vcgeq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))]
@@ -5372,7 +5372,7 @@ pub fn vcge_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))]
@@ -5393,7 +5393,7 @@ pub fn vcgeq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))]
@@ -5414,7 +5414,7 @@ pub fn vcge_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))]
@@ -5435,7 +5435,7 @@ pub fn vcgeq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))]
@@ -5456,7 +5456,7 @@ pub fn vcge_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))]
@@ -5477,7 +5477,7 @@ pub fn vcgeq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))]
@@ -5498,7 +5498,7 @@ pub fn vcge_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))]
@@ -5519,7 +5519,7 @@ pub fn vcgeq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))]
@@ -5540,7 +5540,7 @@ pub fn vcge_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))]
@@ -5561,7 +5561,7 @@ pub fn vcgeq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))]
@@ -5582,7 +5582,7 @@ pub fn vcge_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))]
@@ -5603,7 +5603,7 @@ pub fn vcgeq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcge_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))]
@@ -5624,7 +5624,7 @@ pub fn vcge_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare unsigned greater than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgeq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))]
@@ -5645,7 +5645,7 @@ pub fn vcgeq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgez_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f16"))]
 #[cfg_attr(
@@ -5668,7 +5668,7 @@ pub fn vcgez_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare greater than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgezq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f16"))]
 #[cfg_attr(
@@ -5691,7 +5691,7 @@ pub fn vcgezq_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f16"))]
 #[cfg_attr(
@@ -5713,7 +5713,7 @@ pub fn vcgt_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f16"))]
 #[cfg_attr(
@@ -5735,7 +5735,7 @@ pub fn vcgtq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))]
@@ -5756,7 +5756,7 @@ pub fn vcgt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))]
@@ -5777,7 +5777,7 @@ pub fn vcgtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))]
@@ -5798,7 +5798,7 @@ pub fn vcgt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))]
@@ -5819,7 +5819,7 @@ pub fn vcgtq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))]
@@ -5840,7 +5840,7 @@ pub fn vcgt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))]
@@ -5861,7 +5861,7 @@ pub fn vcgtq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))]
@@ -5882,7 +5882,7 @@ pub fn vcgt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))]
@@ -5903,7 +5903,7 @@ pub fn vcgtq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))]
@@ -5924,7 +5924,7 @@ pub fn vcgt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))]
@@ -5945,7 +5945,7 @@ pub fn vcgtq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))]
@@ -5966,7 +5966,7 @@ pub fn vcgt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))]
@@ -5987,7 +5987,7 @@ pub fn vcgtq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgt_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))]
@@ -6008,7 +6008,7 @@ pub fn vcgt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare unsigned greater than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))]
@@ -6029,7 +6029,7 @@ pub fn vcgtq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtz_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f16"))]
 #[cfg_attr(
@@ -6052,7 +6052,7 @@ pub fn vcgtz_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare greater than zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcgtzq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f16"))]
 #[cfg_attr(
@@ -6075,7 +6075,7 @@ pub fn vcgtzq_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f16"))]
 #[cfg_attr(
@@ -6097,7 +6097,7 @@ pub fn vcle_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f16"))]
 #[cfg_attr(
@@ -6119,7 +6119,7 @@ pub fn vcleq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))]
@@ -6140,7 +6140,7 @@ pub fn vcle_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))]
@@ -6161,7 +6161,7 @@ pub fn vcleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))]
@@ -6182,7 +6182,7 @@ pub fn vcle_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))]
@@ -6203,7 +6203,7 @@ pub fn vcleq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))]
@@ -6224,7 +6224,7 @@ pub fn vcle_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))]
@@ -6245,7 +6245,7 @@ pub fn vcleq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))]
@@ -6266,7 +6266,7 @@ pub fn vcle_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))]
@@ -6287,7 +6287,7 @@ pub fn vcleq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))]
@@ -6308,7 +6308,7 @@ pub fn vcle_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))]
@@ -6329,7 +6329,7 @@ pub fn vcleq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))]
@@ -6350,7 +6350,7 @@ pub fn vcle_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))]
@@ -6371,7 +6371,7 @@ pub fn vcleq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcle_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))]
@@ -6392,7 +6392,7 @@ pub fn vcle_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare unsigned less than or equal"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcleq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))]
@@ -6413,7 +6413,7 @@ pub fn vcleq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclez_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcle.f16"))]
 #[cfg_attr(
@@ -6436,7 +6436,7 @@ pub fn vclez_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare less than or equal to zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclezq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcle.f16"))]
 #[cfg_attr(
@@ -6459,7 +6459,7 @@ pub fn vclezq_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s8"))]
@@ -6488,7 +6488,7 @@ pub fn vcls_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s8"))]
@@ -6517,7 +6517,7 @@ pub fn vclsq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s16"))]
@@ -6546,7 +6546,7 @@ pub fn vcls_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s16"))]
@@ -6575,7 +6575,7 @@ pub fn vclsq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s32"))]
@@ -6604,7 +6604,7 @@ pub fn vcls_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s32"))]
@@ -6633,7 +6633,7 @@ pub fn vclsq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))]
@@ -6654,7 +6654,7 @@ pub fn vcls_u8(a: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))]
@@ -6675,7 +6675,7 @@ pub fn vclsq_u8(a: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))]
@@ -6696,7 +6696,7 @@ pub fn vcls_u16(a: uint16x4_t) -> int16x4_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))]
@@ -6717,7 +6717,7 @@ pub fn vclsq_u16(a: uint16x8_t) -> int16x8_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcls_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))]
@@ -6738,7 +6738,7 @@ pub fn vcls_u32(a: uint32x2_t) -> int32x2_t {
 }
 #[doc = "Count leading sign bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclsq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcls))]
@@ -6759,7 +6759,7 @@ pub fn vclsq_u32(a: uint32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f16"))]
 #[cfg_attr(
@@ -6781,7 +6781,7 @@ pub fn vclt_f16(a: float16x4_t, b: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f16"))]
 #[cfg_attr(
@@ -6803,7 +6803,7 @@ pub fn vcltq_f16(a: float16x8_t, b: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))]
@@ -6824,7 +6824,7 @@ pub fn vclt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))]
@@ -6845,7 +6845,7 @@ pub fn vcltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))]
@@ -6866,7 +6866,7 @@ pub fn vclt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))]
@@ -6887,7 +6887,7 @@ pub fn vcltq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))]
@@ -6908,7 +6908,7 @@ pub fn vclt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))]
@@ -6929,7 +6929,7 @@ pub fn vcltq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))]
@@ -6950,7 +6950,7 @@ pub fn vclt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare signed less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))]
@@ -6971,7 +6971,7 @@ pub fn vcltq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))]
@@ -6992,7 +6992,7 @@ pub fn vclt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))]
@@ -7013,7 +7013,7 @@ pub fn vcltq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))]
@@ -7034,7 +7034,7 @@ pub fn vclt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))]
@@ -7055,7 +7055,7 @@ pub fn vcltq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclt_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))]
@@ -7076,7 +7076,7 @@ pub fn vclt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Compare unsigned less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))]
@@ -7097,7 +7097,7 @@ pub fn vcltq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltz_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclt.f16"))]
 #[cfg_attr(
@@ -7120,7 +7120,7 @@ pub fn vcltz_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point compare less than"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcltzq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclt.f16"))]
 #[cfg_attr(
@@ -7143,7 +7143,7 @@ pub fn vcltzq_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))]
@@ -7164,7 +7164,7 @@ pub fn vclz_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))]
@@ -7185,7 +7185,7 @@ pub fn vclzq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))]
@@ -7206,7 +7206,7 @@ pub fn vclz_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))]
@@ -7227,7 +7227,7 @@ pub fn vclzq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))]
@@ -7248,7 +7248,7 @@ pub fn vclz_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))]
@@ -7269,7 +7269,7 @@ pub fn vclzq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7291,7 +7291,7 @@ pub fn vclz_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7317,7 +7317,7 @@ pub fn vclz_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7339,7 +7339,7 @@ pub fn vclzq_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7365,7 +7365,7 @@ pub fn vclzq_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7387,7 +7387,7 @@ pub fn vclz_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7413,7 +7413,7 @@ pub fn vclz_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7435,7 +7435,7 @@ pub fn vclzq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7461,7 +7461,7 @@ pub fn vclzq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7483,7 +7483,7 @@ pub fn vclz_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclz_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7509,7 +7509,7 @@ pub fn vclz_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7531,7 +7531,7 @@ pub fn vclzq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Count leading zero bits"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vclzq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7562,7 +7562,7 @@ pub fn vclzq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcnt_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))]
@@ -7583,7 +7583,7 @@ pub fn vcnt_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcntq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))]
@@ -7604,7 +7604,7 @@ pub fn vcntq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcnt_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7626,7 +7626,7 @@ pub fn vcnt_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcnt_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7652,7 +7652,7 @@ pub fn vcnt_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcntq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7674,7 +7674,7 @@ pub fn vcntq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcntq_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7705,7 +7705,7 @@ pub fn vcntq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcnt_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7727,7 +7727,7 @@ pub fn vcnt_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcnt_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7753,7 +7753,7 @@ pub fn vcnt_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcntq_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7775,7 +7775,7 @@ pub fn vcntq_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Population count per byte."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcntq_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -7806,7 +7806,7 @@ pub fn vcntq_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -7825,7 +7825,7 @@ pub fn vcombine_f16(a: float16x4_t, b: float16x4_t) -> float16x8_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7842,7 +7842,7 @@ pub fn vcombine_f32(a: float32x2_t, b: float32x2_t) -> float32x4_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7859,7 +7859,7 @@ pub fn vcombine_s8(a: int8x8_t, b: int8x8_t) -> int8x16_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7876,7 +7876,7 @@ pub fn vcombine_s16(a: int16x4_t, b: int16x4_t) -> int16x8_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7893,7 +7893,7 @@ pub fn vcombine_s32(a: int32x2_t, b: int32x2_t) -> int32x4_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7910,7 +7910,7 @@ pub fn vcombine_s64(a: int64x1_t, b: int64x1_t) -> int64x2_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7927,7 +7927,7 @@ pub fn vcombine_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x16_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7944,7 +7944,7 @@ pub fn vcombine_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x8_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7961,7 +7961,7 @@ pub fn vcombine_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x4_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7978,7 +7978,7 @@ pub fn vcombine_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x2_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -7995,7 +7995,7 @@ pub fn vcombine_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x16_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -8012,7 +8012,7 @@ pub fn vcombine_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x8_t {
 }
 #[doc = "Join two smaller vectors into a single larger vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcombine_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -8029,7 +8029,7 @@ pub fn vcombine_p64(a: poly64x1_t, b: poly64x1_t) -> poly64x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8053,7 +8053,7 @@ pub fn vcreate_f16(a: u64) -> float16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8080,7 +8080,7 @@ pub fn vcreate_f16(a: u64) -> float16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8102,7 +8102,7 @@ pub fn vcreate_f32(a: u64) -> float32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8127,7 +8127,7 @@ pub fn vcreate_f32(a: u64) -> float32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8149,7 +8149,7 @@ pub fn vcreate_s8(a: u64) -> int8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8174,7 +8174,7 @@ pub fn vcreate_s8(a: u64) -> int8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8196,7 +8196,7 @@ pub fn vcreate_s16(a: u64) -> int16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8221,7 +8221,7 @@ pub fn vcreate_s16(a: u64) -> int16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8243,7 +8243,7 @@ pub fn vcreate_s32(a: u64) -> int32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8268,7 +8268,7 @@ pub fn vcreate_s32(a: u64) -> int32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -8289,7 +8289,7 @@ pub fn vcreate_s64(a: u64) -> int64x1_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8311,7 +8311,7 @@ pub fn vcreate_u8(a: u64) -> uint8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8336,7 +8336,7 @@ pub fn vcreate_u8(a: u64) -> uint8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8358,7 +8358,7 @@ pub fn vcreate_u16(a: u64) -> uint16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8383,7 +8383,7 @@ pub fn vcreate_u16(a: u64) -> uint16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8405,7 +8405,7 @@ pub fn vcreate_u32(a: u64) -> uint32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8430,7 +8430,7 @@ pub fn vcreate_u32(a: u64) -> uint32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -8451,7 +8451,7 @@ pub fn vcreate_u64(a: u64) -> uint64x1_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8473,7 +8473,7 @@ pub fn vcreate_p8(a: u64) -> poly8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8498,7 +8498,7 @@ pub fn vcreate_p8(a: u64) -> poly8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8520,7 +8520,7 @@ pub fn vcreate_p16(a: u64) -> poly16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -8545,7 +8545,7 @@ pub fn vcreate_p16(a: u64) -> poly16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcreate_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -8566,7 +8566,7 @@ pub fn vcreate_p64(a: u64) -> poly64x1_t {
 }
 #[doc = "Floating-point convert to lower precision narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f16_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -8589,7 +8589,7 @@ pub fn vcvt_f16_f32(a: float32x4_t) -> float16x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -8611,7 +8611,7 @@ pub fn vcvt_f16_s16(a: int16x4_t) -> float16x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -8633,7 +8633,7 @@ pub fn vcvtq_f16_s16(a: int16x8_t) -> float16x8_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -8655,7 +8655,7 @@ pub fn vcvt_f16_u16(a: uint16x4_t) -> float16x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -8677,7 +8677,7 @@ pub fn vcvtq_f16_u16(a: uint16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point convert to higher precision long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -8700,7 +8700,7 @@ pub fn vcvt_f32_f16(a: float16x4_t) -> float32x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -8721,7 +8721,7 @@ pub fn vcvt_f32_s32(a: int32x2_t) -> float32x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -8742,7 +8742,7 @@ pub fn vcvtq_f32_s32(a: int32x4_t) -> float32x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -8763,7 +8763,7 @@ pub fn vcvt_f32_u32(a: uint32x2_t) -> float32x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -8784,7 +8784,7 @@ pub fn vcvtq_f32_u32(a: uint32x4_t) -> float32x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -8819,7 +8819,7 @@ pub fn vcvt_n_f16_s16(a: int16x4_t) -> float16x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -8854,7 +8854,7 @@ pub fn vcvtq_n_f16_s16(a: int16x8_t) -> float16x8_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -8889,7 +8889,7 @@ pub fn vcvt_n_f16_u16(a: uint16x4_t) -> float16x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -8924,7 +8924,7 @@ pub fn vcvtq_n_f16_u16(a: uint16x8_t) -> float16x8_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -8943,7 +8943,7 @@ pub fn vcvt_n_f32_s32(a: int32x2_t) -> float32x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -8962,7 +8962,7 @@ pub fn vcvtq_n_f32_s32(a: int32x4_t) -> float32x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
@@ -8981,7 +8981,7 @@ pub fn vcvt_n_f32_s32(a: int32x2_t) -> float32x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(scvtf, N = 2))]
@@ -9000,7 +9000,7 @@ pub fn vcvtq_n_f32_s32(a: int32x4_t) -> float32x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -9019,7 +9019,7 @@ pub fn vcvt_n_f32_u32(a: uint32x2_t) -> float32x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -9038,7 +9038,7 @@ pub fn vcvtq_n_f32_u32(a: uint32x4_t) -> float32x4_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
@@ -9057,7 +9057,7 @@ pub fn vcvt_n_f32_u32(a: uint32x2_t) -> float32x2_t {
 }
 #[doc = "Fixed-point convert to floating-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ucvtf, N = 2))]
@@ -9076,7 +9076,7 @@ pub fn vcvtq_n_f32_u32(a: uint32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point convert to signed fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -9111,7 +9111,7 @@ pub fn vcvt_n_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Floating-point convert to signed fixed-point"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -9146,7 +9146,7 @@ pub fn vcvtq_n_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -9165,7 +9165,7 @@ pub fn vcvt_n_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -9184,7 +9184,7 @@ pub fn vcvtq_n_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
@@ -9203,7 +9203,7 @@ pub fn vcvt_n_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(fcvtzs, N = 2))]
@@ -9222,7 +9222,7 @@ pub fn vcvtq_n_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Fixed-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -9257,7 +9257,7 @@ pub fn vcvt_n_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Fixed-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcvt", N = 1))]
 #[cfg_attr(
@@ -9292,7 +9292,7 @@ pub fn vcvtq_n_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -9311,7 +9311,7 @@ pub fn vcvt_n_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vcvt, N = 2))]
@@ -9330,7 +9330,7 @@ pub fn vcvtq_n_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_n_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
@@ -9349,7 +9349,7 @@ pub fn vcvt_n_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_n_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(fcvtzu, N = 2))]
@@ -9368,7 +9368,7 @@ pub fn vcvtq_n_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -9390,7 +9390,7 @@ pub fn vcvt_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Floating-point convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -9412,7 +9412,7 @@ pub fn vcvtq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Floating-point convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -9441,7 +9441,7 @@ pub fn vcvt_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Floating-point convert to signed fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -9470,7 +9470,7 @@ pub fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Floating-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -9492,7 +9492,7 @@ pub fn vcvt_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Floating-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
 #[cfg_attr(
@@ -9514,7 +9514,7 @@ pub fn vcvtq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Floating-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvt_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -9543,7 +9543,7 @@ pub fn vcvt_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Floating-point convert to unsigned fixed-point, rounding toward zero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vcvtq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))]
@@ -9572,7 +9572,7 @@ pub fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9600,7 +9600,7 @@ pub fn vdot_lane_s32(a: int32x2_t, b: int8x8_t, c: int8x8_t) ->
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9632,7 +9632,7 @@ pub fn vdot_lane_s32(a: int32x2_t, b: int8x8_t, c: int8x8_t) ->
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9661,7 +9661,7 @@ pub fn vdotq_lane_s32(a: int32x4_t, b: int8x16_t, c: int8x8_t)
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9695,7 +9695,7 @@ pub fn vdotq_lane_s32(a: int32x4_t, b: int8x16_t, c: int8x8_t)
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9723,7 +9723,7 @@ pub fn vdot_lane_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x8_t)
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9755,7 +9755,7 @@ pub fn vdot_lane_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x8_t)
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9784,7 +9784,7 @@ pub fn vdotq_lane_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x8_
 }
 #[doc = "Dot product arithmetic (indexed)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
@@ -9818,7 +9818,7 @@ pub fn vdotq_lane_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x8_
 }
 #[doc = "Dot product arithmetic (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsdot))]
@@ -9847,7 +9847,7 @@ pub fn vdot_s32(a: int32x2_t, b: int8x8_t, c: int8x8_t) -> int32x2_t {
 }
 #[doc = "Dot product arithmetic (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsdot))]
@@ -9876,7 +9876,7 @@ pub fn vdotq_s32(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t {
 }
 #[doc = "Dot product arithmetic (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdot_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vudot))]
@@ -9905,7 +9905,7 @@ pub fn vdot_u32(a: uint32x2_t, b: uint8x8_t, c: uint8x8_t) -> uint32x2_t {
 }
 #[doc = "Dot product arithmetic (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdotq_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,dotprod")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vudot))]
@@ -9934,7 +9934,7 @@ pub fn vdotq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -9959,7 +9959,7 @@ pub fn vdup_lane_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -9990,7 +9990,7 @@ pub fn vdupq_lane_f16(a: float16x4_t) -> float16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))]
@@ -10013,7 +10013,7 @@ pub fn vdup_lane_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))]
@@ -10036,7 +10036,7 @@ pub fn vdup_lane_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))]
@@ -10059,7 +10059,7 @@ pub fn vdup_lane_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))]
@@ -10082,7 +10082,7 @@ pub fn vdupq_lane_f32(a: float32x2_t) -> float32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))]
@@ -10105,7 +10105,7 @@ pub fn vdupq_lane_s32(a: int32x2_t) -> int32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))]
@@ -10128,7 +10128,7 @@ pub fn vdupq_lane_u32(a: uint32x2_t) -> uint32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -10151,7 +10151,7 @@ pub fn vdup_lane_p16(a: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -10174,7 +10174,7 @@ pub fn vdup_lane_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -10197,7 +10197,7 @@ pub fn vdup_lane_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -10226,7 +10226,7 @@ pub fn vdupq_lane_p16(a: poly16x4_t) -> poly16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -10255,7 +10255,7 @@ pub fn vdupq_lane_s16(a: int16x4_t) -> int16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))]
@@ -10284,7 +10284,7 @@ pub fn vdupq_lane_u16(a: uint16x4_t) -> uint16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))]
@@ -10313,7 +10313,7 @@ pub fn vdup_lane_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))]
@@ -10342,7 +10342,7 @@ pub fn vdup_lane_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))]
@@ -10371,7 +10371,7 @@ pub fn vdup_lane_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))]
@@ -10403,7 +10403,7 @@ pub fn vdupq_lane_p8(a: poly8x8_t) -> poly8x16_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))]
@@ -10435,7 +10435,7 @@ pub fn vdupq_lane_s8(a: int8x8_t) -> int8x16_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))]
@@ -10467,7 +10467,7 @@ pub fn vdupq_lane_u8(a: uint8x8_t) -> uint8x16_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))]
@@ -10490,7 +10490,7 @@ pub fn vdup_lane_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))]
@@ -10513,7 +10513,7 @@ pub fn vdup_lane_u64(a: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10538,7 +10538,7 @@ pub fn vdup_laneq_f16(a: float16x8_t) -> float16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10569,7 +10569,7 @@ pub fn vdupq_laneq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))]
@@ -10592,7 +10592,7 @@ pub fn vdup_laneq_f32(a: float32x4_t) -> float32x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))]
@@ -10615,7 +10615,7 @@ pub fn vdup_laneq_s32(a: int32x4_t) -> int32x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))]
@@ -10638,7 +10638,7 @@ pub fn vdup_laneq_u32(a: uint32x4_t) -> uint32x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))]
@@ -10661,7 +10661,7 @@ pub fn vdupq_laneq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))]
@@ -10684,7 +10684,7 @@ pub fn vdupq_laneq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))]
@@ -10707,7 +10707,7 @@ pub fn vdupq_laneq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10730,7 +10730,7 @@ pub fn vdup_laneq_p16(a: poly16x8_t) -> poly16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10753,7 +10753,7 @@ pub fn vdup_laneq_s16(a: int16x8_t) -> int16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10776,7 +10776,7 @@ pub fn vdup_laneq_u16(a: uint16x8_t) -> uint16x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10805,7 +10805,7 @@ pub fn vdupq_laneq_p16(a: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10834,7 +10834,7 @@ pub fn vdupq_laneq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))]
@@ -10863,7 +10863,7 @@ pub fn vdupq_laneq_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))]
@@ -10892,7 +10892,7 @@ pub fn vdup_laneq_p8(a: poly8x16_t) -> poly8x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))]
@@ -10921,7 +10921,7 @@ pub fn vdup_laneq_s8(a: int8x16_t) -> int8x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))]
@@ -10950,7 +10950,7 @@ pub fn vdup_laneq_u8(a: uint8x16_t) -> uint8x8_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))]
@@ -10982,7 +10982,7 @@ pub fn vdupq_laneq_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))]
@@ -11014,7 +11014,7 @@ pub fn vdupq_laneq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))]
@@ -11046,7 +11046,7 @@ pub fn vdupq_laneq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))]
@@ -11069,7 +11069,7 @@ pub fn vdup_laneq_s64(a: int64x2_t) -> int64x1_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_laneq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))]
@@ -11092,7 +11092,7 @@ pub fn vdup_laneq_u64(a: uint64x2_t) -> uint64x1_t {
 }
 #[doc = "Create a new vector with all lanes set to a value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11108,7 +11108,7 @@ pub fn vdup_n_f16(a: f16) -> float16x4_t {
 }
 #[doc = "Create a new vector with all lanes set to a value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11124,7 +11124,7 @@ pub fn vdupq_n_f16(a: f16) -> float16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11145,7 +11145,7 @@ pub fn vdup_n_f32(value: f32) -> float32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11166,7 +11166,7 @@ pub fn vdup_n_p16(value: p16) -> poly16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -11187,7 +11187,7 @@ pub fn vdup_n_p8(value: p8) -> poly8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11208,7 +11208,7 @@ pub fn vdup_n_s16(value: i16) -> int16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11229,7 +11229,7 @@ pub fn vdup_n_s32(value: i32) -> int32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -11250,7 +11250,7 @@ pub fn vdup_n_s64(value: i64) -> int64x1_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -11271,7 +11271,7 @@ pub fn vdup_n_s8(value: i8) -> int8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11292,7 +11292,7 @@ pub fn vdup_n_u16(value: u16) -> uint16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11313,7 +11313,7 @@ pub fn vdup_n_u32(value: u32) -> uint32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -11334,7 +11334,7 @@ pub fn vdup_n_u64(value: u64) -> uint64x1_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -11355,7 +11355,7 @@ pub fn vdup_n_u8(value: u8) -> uint8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11376,7 +11376,7 @@ pub fn vdupq_n_f32(value: f32) -> float32x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11397,7 +11397,7 @@ pub fn vdupq_n_p16(value: p16) -> poly16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -11418,7 +11418,7 @@ pub fn vdupq_n_p8(value: p8) -> poly8x16_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11439,7 +11439,7 @@ pub fn vdupq_n_s16(value: i16) -> int16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11460,7 +11460,7 @@ pub fn vdupq_n_s32(value: i32) -> int32x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -11481,7 +11481,7 @@ pub fn vdupq_n_s64(value: i64) -> int64x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -11502,7 +11502,7 @@ pub fn vdupq_n_s8(value: i8) -> int8x16_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -11523,7 +11523,7 @@ pub fn vdupq_n_u16(value: u16) -> uint16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11544,7 +11544,7 @@ pub fn vdupq_n_u32(value: u32) -> uint32x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -11565,7 +11565,7 @@ pub fn vdupq_n_u64(value: u64) -> uint64x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -11586,7 +11586,7 @@ pub fn vdupq_n_u8(value: u8) -> uint8x16_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdup_n_f32_vfp4)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11607,7 +11607,7 @@ fn vdup_n_f32_vfp4(value: f32) -> float32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_n_f32_vfp4)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -11628,7 +11628,7 @@ fn vdupq_n_f32_vfp4(value: f32) -> float32x4_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 0))]
@@ -11651,7 +11651,7 @@ pub fn vdupq_lane_s64(a: int64x1_t) -> int64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 0))]
@@ -11674,7 +11674,7 @@ pub fn vdupq_lane_u64(a: uint64x1_t) -> uint64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))]
@@ -11697,7 +11697,7 @@ pub fn vdupq_laneq_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Set all vector lanes to the same value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vdupq_laneq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))]
@@ -11720,7 +11720,7 @@ pub fn vdupq_laneq_u64(a: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11741,7 +11741,7 @@ pub fn veor_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11762,7 +11762,7 @@ pub fn veorq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11783,7 +11783,7 @@ pub fn veor_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11804,7 +11804,7 @@ pub fn veorq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11825,7 +11825,7 @@ pub fn veor_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11846,7 +11846,7 @@ pub fn veorq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11867,7 +11867,7 @@ pub fn veor_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11888,7 +11888,7 @@ pub fn veorq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11909,7 +11909,7 @@ pub fn veor_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11930,7 +11930,7 @@ pub fn veorq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11951,7 +11951,7 @@ pub fn veor_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11972,7 +11972,7 @@ pub fn veorq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -11993,7 +11993,7 @@ pub fn veor_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -12014,7 +12014,7 @@ pub fn veorq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veor_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -12035,7 +12035,7 @@ pub fn veor_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector bitwise exclusive or (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/veorq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))]
@@ -12056,7 +12056,7 @@ pub fn veorq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
 #[cfg_attr(
@@ -12088,7 +12088,7 @@ pub fn vext_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))]
@@ -12117,7 +12117,7 @@ pub fn vext_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))]
@@ -12146,7 +12146,7 @@ pub fn vext_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))]
@@ -12177,7 +12177,7 @@ pub fn vext_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))]
@@ -12202,7 +12202,7 @@ pub unsafe fn vext_s64(a: int64x1_t, _b: int64x1_t) -> int64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))]
@@ -12225,7 +12225,7 @@ pub unsafe fn vext_u64(a: uint64x1_t, _b: uint64x1_t) -> uint64x1_
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
@@ -12260,7 +12260,7 @@ pub fn vext_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
@@ -12295,7 +12295,7 @@ pub fn vextq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
@@ -12330,7 +12330,7 @@ pub fn vext_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
@@ -12365,7 +12365,7 @@ pub fn vextq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
@@ -12400,7 +12400,7 @@ pub fn vext_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
@@ -12435,7 +12435,7 @@ pub fn vextq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 7))]
 #[cfg_attr(
@@ -12471,7 +12471,7 @@ pub fn vextq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
@@ -12502,7 +12502,7 @@ pub fn vextq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
@@ -12533,7 +12533,7 @@ pub fn vext_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
@@ -12564,7 +12564,7 @@ pub fn vextq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
@@ -12595,7 +12595,7 @@ pub fn vext_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
@@ -12626,7 +12626,7 @@ pub fn vextq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vext_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 3))]
@@ -12657,7 +12657,7 @@ pub fn vext_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))]
@@ -12686,7 +12686,7 @@ pub fn vextq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))]
@@ -12715,7 +12715,7 @@ pub fn vextq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 15))]
@@ -12818,7 +12818,7 @@ pub fn vextq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 15))]
@@ -12921,7 +12921,7 @@ pub fn vextq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Extract vector from pair of vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vextq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 15))]
@@ -13024,7 +13024,7 @@ pub fn vextq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))]
 #[cfg_attr(
@@ -13046,7 +13046,7 @@ pub fn vfma_f16(a: float16x4_t, b: float16x4_t, c: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))]
 #[cfg_attr(
@@ -13068,7 +13068,7 @@ pub fn vfmaq_f16(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> float16x8_t
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))]
@@ -13089,7 +13089,7 @@ pub fn vfma_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))]
@@ -13110,7 +13110,7 @@ pub fn vfmaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfma_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))]
@@ -13131,7 +13131,7 @@ pub fn vfma_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t {
 }
 #[doc = "Floating-point fused Multiply-Add to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmaq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))]
@@ -13152,7 +13152,7 @@ pub fn vfmaq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -13178,7 +13178,7 @@ pub fn vfms_f16(a: float16x4_t, b: float16x4_t, c: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -13204,7 +13204,7 @@ pub fn vfmsq_f16(a: float16x8_t, b: float16x8_t, c: float16x8_t) -> float16x8_t
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))]
@@ -13228,7 +13228,7 @@ pub fn vfms_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point fused multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))]
@@ -13252,7 +13252,7 @@ pub fn vfmsq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t
 }
 #[doc = "Floating-point fused Multiply-subtract to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfms_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))]
@@ -13273,7 +13273,7 @@ pub fn vfms_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t {
 }
 #[doc = "Floating-point fused Multiply-subtract to accumulator(vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmsq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "vfp4"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))]
@@ -13294,7 +13294,7 @@ pub fn vfmsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t {
 }
 #[doc = "Duplicate vector element to vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -13313,7 +13313,7 @@ pub fn vget_high_f16(a: float16x8_t) -> float16x4_t {
 }
 #[doc = "Duplicate vector element to vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -13332,7 +13332,7 @@ pub fn vget_low_f16(a: float16x8_t) -> float16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13353,7 +13353,7 @@ pub fn vget_high_f32(a: float32x4_t) -> float32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13374,7 +13374,7 @@ pub fn vget_high_p16(a: poly16x8_t) -> poly16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13395,7 +13395,7 @@ pub fn vget_high_p8(a: poly8x16_t) -> poly8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13416,7 +13416,7 @@ pub fn vget_high_s16(a: int16x8_t) -> int16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13437,7 +13437,7 @@ pub fn vget_high_s32(a: int32x4_t) -> int32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13458,7 +13458,7 @@ pub fn vget_high_s8(a: int8x16_t) -> int8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13479,7 +13479,7 @@ pub fn vget_high_u16(a: uint16x8_t) -> uint16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13500,7 +13500,7 @@ pub fn vget_high_u32(a: uint32x4_t) -> uint32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13521,7 +13521,7 @@ pub fn vget_high_u8(a: uint8x16_t) -> uint8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13542,7 +13542,7 @@ pub fn vget_high_s64(a: int64x2_t) -> int64x1_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -13563,7 +13563,7 @@ pub fn vget_high_u64(a: uint64x2_t) -> uint64x1_t {
 }
 #[doc = "Duplicate vector element to scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -13581,7 +13581,7 @@ pub fn vget_lane_f16(a: float16x4_t) -> f16 {
 }
 #[doc = "Duplicate vector element to scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -13599,7 +13599,7 @@ pub fn vgetq_lane_f16(a: float16x8_t) -> f16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13618,7 +13618,7 @@ pub fn vget_lane_f32(v: float32x2_t) -> f32 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13637,7 +13637,7 @@ pub fn vget_lane_p16(v: poly16x4_t) -> p16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13656,7 +13656,7 @@ pub fn vget_lane_p8(v: poly8x8_t) -> p8 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13675,7 +13675,7 @@ pub fn vget_lane_s16(v: int16x4_t) -> i16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13694,7 +13694,7 @@ pub fn vget_lane_s32(v: int32x2_t) -> i32 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13713,7 +13713,7 @@ pub fn vget_lane_s8(v: int8x8_t) -> i8 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13732,7 +13732,7 @@ pub fn vget_lane_u16(v: uint16x4_t) -> u16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13751,7 +13751,7 @@ pub fn vget_lane_u32(v: uint32x2_t) -> u32 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13770,7 +13770,7 @@ pub fn vget_lane_u8(v: uint8x8_t) -> u8 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13789,7 +13789,7 @@ pub fn vgetq_lane_f32(v: float32x4_t) -> f32 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13808,7 +13808,7 @@ pub fn vgetq_lane_p16(v: poly16x8_t) -> p16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13827,7 +13827,7 @@ pub fn vgetq_lane_p64(v: poly64x2_t) -> p64 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13846,7 +13846,7 @@ pub fn vgetq_lane_p8(v: poly8x16_t) -> p8 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13865,7 +13865,7 @@ pub fn vgetq_lane_s16(v: int16x8_t) -> i16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13884,7 +13884,7 @@ pub fn vgetq_lane_s32(v: int32x4_t) -> i32 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13903,7 +13903,7 @@ pub fn vgetq_lane_s64(v: int64x2_t) -> i64 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13922,7 +13922,7 @@ pub fn vgetq_lane_s8(v: int8x16_t) -> i8 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13941,7 +13941,7 @@ pub fn vgetq_lane_u16(v: uint16x8_t) -> u16 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13960,7 +13960,7 @@ pub fn vgetq_lane_u32(v: uint32x4_t) -> u32 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13979,7 +13979,7 @@ pub fn vgetq_lane_u64(v: uint64x2_t) -> u64 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vgetq_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -13998,7 +13998,7 @@ pub fn vgetq_lane_u8(v: uint8x16_t) -> u8 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -14017,7 +14017,7 @@ pub fn vget_lane_p64(v: poly64x1_t) -> p64 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -14036,7 +14036,7 @@ pub fn vget_lane_s64(v: int64x1_t) -> i64 {
 }
 #[doc = "Move vector element to general-purpose register"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(1)]
@@ -14055,7 +14055,7 @@ pub fn vget_lane_u64(v: uint64x1_t) -> u64 {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14072,7 +14072,7 @@ pub fn vget_low_f32(a: float32x4_t) -> float32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14089,7 +14089,7 @@ pub fn vget_low_p16(a: poly16x8_t) -> poly16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14106,7 +14106,7 @@ pub fn vget_low_p8(a: poly8x16_t) -> poly8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14123,7 +14123,7 @@ pub fn vget_low_s16(a: int16x8_t) -> int16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14140,7 +14140,7 @@ pub fn vget_low_s32(a: int32x4_t) -> int32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14157,7 +14157,7 @@ pub fn vget_low_s8(a: int8x16_t) -> int8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14174,7 +14174,7 @@ pub fn vget_low_u16(a: uint16x8_t) -> uint16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14191,7 +14191,7 @@ pub fn vget_low_u32(a: uint32x4_t) -> uint32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14208,7 +14208,7 @@ pub fn vget_low_u8(a: uint8x16_t) -> uint8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14225,7 +14225,7 @@ pub fn vget_low_s64(a: int64x2_t) -> int64x1_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vget_low_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(nop))]
@@ -14242,7 +14242,7 @@ pub fn vget_low_u64(a: uint64x2_t) -> uint64x1_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))]
@@ -14271,7 +14271,7 @@ pub fn vhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))]
@@ -14300,7 +14300,7 @@ pub fn vhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))]
@@ -14329,7 +14329,7 @@ pub fn vhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))]
@@ -14358,7 +14358,7 @@ pub fn vhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))]
@@ -14387,7 +14387,7 @@ pub fn vhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))]
@@ -14416,7 +14416,7 @@ pub fn vhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))]
@@ -14445,7 +14445,7 @@ pub fn vhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))]
@@ -14474,7 +14474,7 @@ pub fn vhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))]
@@ -14503,7 +14503,7 @@ pub fn vhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))]
@@ -14532,7 +14532,7 @@ pub fn vhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhadd_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))]
@@ -14561,7 +14561,7 @@ pub fn vhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))]
@@ -14590,7 +14590,7 @@ pub fn vhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))]
@@ -14619,7 +14619,7 @@ pub fn vhsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))]
@@ -14648,7 +14648,7 @@ pub fn vhsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))]
@@ -14677,7 +14677,7 @@ pub fn vhsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))]
@@ -14706,7 +14706,7 @@ pub fn vhsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))]
@@ -14735,7 +14735,7 @@ pub fn vhsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))]
@@ -14764,7 +14764,7 @@ pub fn vhsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))]
@@ -14793,7 +14793,7 @@ pub fn vhsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))]
@@ -14822,7 +14822,7 @@ pub fn vhsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))]
@@ -14851,7 +14851,7 @@ pub fn vhsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))]
@@ -14880,7 +14880,7 @@ pub fn vhsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsub_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))]
@@ -14909,7 +14909,7 @@ pub fn vhsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed halving subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vhsubq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))]
@@ -14940,7 +14940,7 @@ pub fn vhsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -14959,7 +14959,7 @@ pub unsafe fn vld1_dup_f16(ptr: *const f16) -> float16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -14978,7 +14978,7 @@ pub unsafe fn vld1q_dup_f16(ptr: *const f16) -> float16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))]
@@ -15001,7 +15001,7 @@ pub unsafe fn vld1_dup_f32(ptr: *const f32) -> float32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))]
@@ -15024,7 +15024,7 @@ pub unsafe fn vld1_dup_p16(ptr: *const p16) -> poly16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))]
@@ -15047,7 +15047,7 @@ pub unsafe fn vld1_dup_p8(ptr: *const p8) -> poly8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))]
@@ -15070,7 +15070,7 @@ pub unsafe fn vld1_dup_s16(ptr: *const i16) -> int16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))]
@@ -15093,7 +15093,7 @@ pub unsafe fn vld1_dup_s32(ptr: *const i32) -> int32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))]
@@ -15116,7 +15116,7 @@ pub unsafe fn vld1_dup_s8(ptr: *const i8) -> int8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))]
@@ -15139,7 +15139,7 @@ pub unsafe fn vld1_dup_u16(ptr: *const u16) -> uint16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))]
@@ -15162,7 +15162,7 @@ pub unsafe fn vld1_dup_u32(ptr: *const u32) -> uint32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))]
@@ -15185,7 +15185,7 @@ pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))]
@@ -15208,7 +15208,7 @@ pub unsafe fn vld1q_dup_f32(ptr: *const f32) -> float32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))]
@@ -15231,7 +15231,7 @@ pub unsafe fn vld1q_dup_p16(ptr: *const p16) -> poly16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))]
@@ -15254,7 +15254,7 @@ pub unsafe fn vld1q_dup_p8(ptr: *const p8) -> poly8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))]
@@ -15277,7 +15277,7 @@ pub unsafe fn vld1q_dup_s16(ptr: *const i16) -> int16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))]
@@ -15300,7 +15300,7 @@ pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))]
@@ -15323,7 +15323,7 @@ pub unsafe fn vld1q_dup_s64(ptr: *const i64) -> int64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))]
@@ -15346,7 +15346,7 @@ pub unsafe fn vld1q_dup_s8(ptr: *const i8) -> int8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))]
@@ -15369,7 +15369,7 @@ pub unsafe fn vld1q_dup_u16(ptr: *const u16) -> uint16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))]
@@ -15392,7 +15392,7 @@ pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))]
@@ -15415,7 +15415,7 @@ pub unsafe fn vld1q_dup_u64(ptr: *const u64) -> uint64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))]
@@ -15438,7 +15438,7 @@ pub unsafe fn vld1q_dup_u8(ptr: *const u8) -> uint8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr))]
@@ -15470,7 +15470,7 @@ pub unsafe fn vld1_dup_p64(ptr: *const p64) -> poly64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr))]
@@ -15502,7 +15502,7 @@ pub unsafe fn vld1_dup_s64(ptr: *const i64) -> int64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr))]
@@ -15534,7 +15534,7 @@ pub unsafe fn vld1_dup_u64(ptr: *const u64) -> uint64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -15552,7 +15552,7 @@ pub unsafe fn vld1_f16(ptr: *const f16) -> float16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -15571,7 +15571,7 @@ pub unsafe fn vld1_f16(ptr: *const f16) -> float16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -15589,7 +15589,7 @@ pub unsafe fn vld1q_f16(ptr: *const f16) -> float16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -15608,7 +15608,7 @@ pub unsafe fn vld1q_f16(ptr: *const f16) -> float16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15634,7 +15634,7 @@ pub unsafe fn vld1_f16_x2(a: *const f16) -> float16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15660,7 +15660,7 @@ pub unsafe fn vld1_f16_x3(a: *const f16) -> float16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15686,7 +15686,7 @@ pub unsafe fn vld1_f16_x4(a: *const f16) -> float16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15712,7 +15712,7 @@ pub unsafe fn vld1q_f16_x2(a: *const f16) -> float16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15738,7 +15738,7 @@ pub unsafe fn vld1q_f16_x3(a: *const f16) -> float16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15764,7 +15764,7 @@ pub unsafe fn vld1q_f16_x4(a: *const f16) -> float16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15777,7 +15777,7 @@ pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15790,7 +15790,7 @@ pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15803,7 +15803,7 @@ pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15816,7 +15816,7 @@ pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15829,7 +15829,7 @@ pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15842,7 +15842,7 @@ pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15855,7 +15855,7 @@ pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15868,7 +15868,7 @@ pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15881,7 +15881,7 @@ pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15894,7 +15894,7 @@ pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15907,7 +15907,7 @@ pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15920,7 +15920,7 @@ pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15933,7 +15933,7 @@ pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15946,7 +15946,7 @@ pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,aes")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -15959,7 +15959,7 @@ pub unsafe fn vld1q_p64(ptr: *const p64) -> poly64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -15990,7 +15990,7 @@ pub unsafe fn vld1_f32_x2(a: *const f32) -> float32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -16021,7 +16021,7 @@ pub unsafe fn vld1_f32_x3(a: *const f32) -> float32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_f32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -16052,7 +16052,7 @@ pub unsafe fn vld1_f32_x4(a: *const f32) -> float32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -16083,7 +16083,7 @@ pub unsafe fn vld1q_f32_x2(a: *const f32) -> float32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -16114,7 +16114,7 @@ pub unsafe fn vld1q_f32_x3(a: *const f32) -> float32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_f32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -16145,7 +16145,7 @@ pub unsafe fn vld1q_f32_x4(a: *const f32) -> float32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1, LANE = 0))]
@@ -16165,7 +16165,7 @@ pub unsafe fn vld1_lane_f16(ptr: *const f16, src: float16x4_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1, LANE = 0))]
@@ -16185,7 +16185,7 @@ pub unsafe fn vld1q_lane_f16(ptr: *const f16, src: float16x8_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16210,7 +16210,7 @@ pub unsafe fn vld1_lane_f32(ptr: *const f32, src: float32x2_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16235,7 +16235,7 @@ pub unsafe fn vld1_lane_p16(ptr: *const p16, src: poly16x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16260,7 +16260,7 @@ pub unsafe fn vld1_lane_p8(ptr: *const p8, src: poly8x8_t) -> p
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16285,7 +16285,7 @@ pub unsafe fn vld1_lane_s16(ptr: *const i16, src: int16x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16310,7 +16310,7 @@ pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16335,7 +16335,7 @@ pub unsafe fn vld1_lane_s64(ptr: *const i64, src: int64x1_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16360,7 +16360,7 @@ pub unsafe fn vld1_lane_s8(ptr: *const i8, src: int8x8_t) -> in
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16385,7 +16385,7 @@ pub unsafe fn vld1_lane_u16(ptr: *const u16, src: uint16x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16410,7 +16410,7 @@ pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16435,7 +16435,7 @@ pub unsafe fn vld1_lane_u64(ptr: *const u64, src: uint64x1_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16460,7 +16460,7 @@ pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t) -> u
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16485,7 +16485,7 @@ pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16510,7 +16510,7 @@ pub unsafe fn vld1q_lane_p16(ptr: *const p16, src: poly16x8_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16535,7 +16535,7 @@ pub unsafe fn vld1q_lane_p8(ptr: *const p8, src: poly8x16_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16560,7 +16560,7 @@ pub unsafe fn vld1q_lane_s16(ptr: *const i16, src: int16x8_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16585,7 +16585,7 @@ pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16610,7 +16610,7 @@ pub unsafe fn vld1q_lane_s64(ptr: *const i64, src: int64x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16635,7 +16635,7 @@ pub unsafe fn vld1q_lane_s8(ptr: *const i8, src: int8x16_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16660,7 +16660,7 @@ pub unsafe fn vld1q_lane_u16(ptr: *const u16, src: uint16x8_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16685,7 +16685,7 @@ pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16710,7 +16710,7 @@ pub unsafe fn vld1q_lane_u64(ptr: *const u64, src: uint64x2_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16735,7 +16735,7 @@ pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16760,7 +16760,7 @@ pub unsafe fn vld1_lane_p64(ptr: *const p64, src: poly64x1_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[rustc_legacy_const_generics(2)]
@@ -16785,7 +16785,7 @@ pub unsafe fn vld1q_lane_p64(ptr: *const p64, src: poly64x2_t)
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,aes")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -16803,7 +16803,7 @@ pub unsafe fn vld1_p64(ptr: *const p64) -> poly64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -16826,7 +16826,7 @@ pub unsafe fn vld1_p64_x2(a: *const p64) -> poly64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -16849,7 +16849,7 @@ pub unsafe fn vld1_p64_x3(a: *const p64) -> poly64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -16872,7 +16872,7 @@ pub unsafe fn vld1_p64_x4(a: *const p64) -> poly64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -16896,7 +16896,7 @@ pub unsafe fn vld1q_p64_x2(a: *const p64) -> poly64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -16923,7 +16923,7 @@ pub unsafe fn vld1q_p64_x2(a: *const p64) -> poly64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -16947,7 +16947,7 @@ pub unsafe fn vld1q_p64_x3(a: *const p64) -> poly64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -16975,7 +16975,7 @@ pub unsafe fn vld1q_p64_x3(a: *const p64) -> poly64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -16999,7 +16999,7 @@ pub unsafe fn vld1q_p64_x4(a: *const p64) -> poly64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -17028,7 +17028,7 @@ pub unsafe fn vld1q_p64_x4(a: *const p64) -> poly64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17041,7 +17041,7 @@ pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17054,7 +17054,7 @@ pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17067,7 +17067,7 @@ pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17080,7 +17080,7 @@ pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17093,7 +17093,7 @@ pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17106,7 +17106,7 @@ pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17119,7 +17119,7 @@ pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -17132,7 +17132,7 @@ pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17163,7 +17163,7 @@ pub unsafe fn vld1_s8_x2(a: *const i8) -> int8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17194,7 +17194,7 @@ pub unsafe fn vld1_s8_x3(a: *const i8) -> int8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17225,7 +17225,7 @@ pub unsafe fn vld1_s8_x4(a: *const i8) -> int8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17256,7 +17256,7 @@ pub unsafe fn vld1q_s8_x2(a: *const i8) -> int8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17287,7 +17287,7 @@ pub unsafe fn vld1q_s8_x3(a: *const i8) -> int8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17318,7 +17318,7 @@ pub unsafe fn vld1q_s8_x4(a: *const i8) -> int8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17349,7 +17349,7 @@ pub unsafe fn vld1_s16_x2(a: *const i16) -> int16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17380,7 +17380,7 @@ pub unsafe fn vld1_s16_x3(a: *const i16) -> int16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17411,7 +17411,7 @@ pub unsafe fn vld1_s16_x4(a: *const i16) -> int16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17442,7 +17442,7 @@ pub unsafe fn vld1q_s16_x2(a: *const i16) -> int16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17473,7 +17473,7 @@ pub unsafe fn vld1q_s16_x3(a: *const i16) -> int16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17504,7 +17504,7 @@ pub unsafe fn vld1q_s16_x4(a: *const i16) -> int16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17535,7 +17535,7 @@ pub unsafe fn vld1_s32_x2(a: *const i32) -> int32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17566,7 +17566,7 @@ pub unsafe fn vld1_s32_x3(a: *const i32) -> int32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17597,7 +17597,7 @@ pub unsafe fn vld1_s32_x4(a: *const i32) -> int32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17628,7 +17628,7 @@ pub unsafe fn vld1q_s32_x2(a: *const i32) -> int32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17659,7 +17659,7 @@ pub unsafe fn vld1q_s32_x3(a: *const i32) -> int32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17690,7 +17690,7 @@ pub unsafe fn vld1q_s32_x4(a: *const i32) -> int32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17721,7 +17721,7 @@ pub unsafe fn vld1_s64_x2(a: *const i64) -> int64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17752,7 +17752,7 @@ pub unsafe fn vld1_s64_x3(a: *const i64) -> int64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_s64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17783,7 +17783,7 @@ pub unsafe fn vld1_s64_x4(a: *const i64) -> int64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17814,7 +17814,7 @@ pub unsafe fn vld1q_s64_x2(a: *const i64) -> int64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17845,7 +17845,7 @@ pub unsafe fn vld1q_s64_x3(a: *const i64) -> int64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_s64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -17876,7 +17876,7 @@ pub unsafe fn vld1q_s64_x4(a: *const i64) -> int64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -17900,7 +17900,7 @@ pub unsafe fn vld1_u8_x2(a: *const u8) -> uint8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -17927,7 +17927,7 @@ pub unsafe fn vld1_u8_x2(a: *const u8) -> uint8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -17951,7 +17951,7 @@ pub unsafe fn vld1_u8_x3(a: *const u8) -> uint8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -17979,7 +17979,7 @@ pub unsafe fn vld1_u8_x3(a: *const u8) -> uint8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18003,7 +18003,7 @@ pub unsafe fn vld1_u8_x4(a: *const u8) -> uint8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18032,7 +18032,7 @@ pub unsafe fn vld1_u8_x4(a: *const u8) -> uint8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18056,7 +18056,7 @@ pub unsafe fn vld1q_u8_x2(a: *const u8) -> uint8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18095,7 +18095,7 @@ pub unsafe fn vld1q_u8_x2(a: *const u8) -> uint8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18119,7 +18119,7 @@ pub unsafe fn vld1q_u8_x3(a: *const u8) -> uint8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18165,7 +18165,7 @@ pub unsafe fn vld1q_u8_x3(a: *const u8) -> uint8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18189,7 +18189,7 @@ pub unsafe fn vld1q_u8_x4(a: *const u8) -> uint8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18242,7 +18242,7 @@ pub unsafe fn vld1q_u8_x4(a: *const u8) -> uint8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18266,7 +18266,7 @@ pub unsafe fn vld1_u16_x2(a: *const u16) -> uint16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18293,7 +18293,7 @@ pub unsafe fn vld1_u16_x2(a: *const u16) -> uint16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18317,7 +18317,7 @@ pub unsafe fn vld1_u16_x3(a: *const u16) -> uint16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18345,7 +18345,7 @@ pub unsafe fn vld1_u16_x3(a: *const u16) -> uint16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18369,7 +18369,7 @@ pub unsafe fn vld1_u16_x4(a: *const u16) -> uint16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18398,7 +18398,7 @@ pub unsafe fn vld1_u16_x4(a: *const u16) -> uint16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18422,7 +18422,7 @@ pub unsafe fn vld1q_u16_x2(a: *const u16) -> uint16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18449,7 +18449,7 @@ pub unsafe fn vld1q_u16_x2(a: *const u16) -> uint16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18473,7 +18473,7 @@ pub unsafe fn vld1q_u16_x3(a: *const u16) -> uint16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18501,7 +18501,7 @@ pub unsafe fn vld1q_u16_x3(a: *const u16) -> uint16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18525,7 +18525,7 @@ pub unsafe fn vld1q_u16_x4(a: *const u16) -> uint16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18554,7 +18554,7 @@ pub unsafe fn vld1q_u16_x4(a: *const u16) -> uint16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18578,7 +18578,7 @@ pub unsafe fn vld1_u32_x2(a: *const u32) -> uint32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18605,7 +18605,7 @@ pub unsafe fn vld1_u32_x2(a: *const u32) -> uint32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18629,7 +18629,7 @@ pub unsafe fn vld1_u32_x3(a: *const u32) -> uint32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18657,7 +18657,7 @@ pub unsafe fn vld1_u32_x3(a: *const u32) -> uint32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18681,7 +18681,7 @@ pub unsafe fn vld1_u32_x4(a: *const u32) -> uint32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18710,7 +18710,7 @@ pub unsafe fn vld1_u32_x4(a: *const u32) -> uint32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18734,7 +18734,7 @@ pub unsafe fn vld1q_u32_x2(a: *const u32) -> uint32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18761,7 +18761,7 @@ pub unsafe fn vld1q_u32_x2(a: *const u32) -> uint32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18785,7 +18785,7 @@ pub unsafe fn vld1q_u32_x3(a: *const u32) -> uint32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18813,7 +18813,7 @@ pub unsafe fn vld1q_u32_x3(a: *const u32) -> uint32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18837,7 +18837,7 @@ pub unsafe fn vld1q_u32_x4(a: *const u32) -> uint32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18866,7 +18866,7 @@ pub unsafe fn vld1q_u32_x4(a: *const u32) -> uint32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -18889,7 +18889,7 @@ pub unsafe fn vld1_u64_x2(a: *const u64) -> uint64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -18912,7 +18912,7 @@ pub unsafe fn vld1_u64_x3(a: *const u64) -> uint64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_u64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld1))]
@@ -18935,7 +18935,7 @@ pub unsafe fn vld1_u64_x4(a: *const u64) -> uint64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18959,7 +18959,7 @@ pub unsafe fn vld1q_u64_x2(a: *const u64) -> uint64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -18986,7 +18986,7 @@ pub unsafe fn vld1q_u64_x2(a: *const u64) -> uint64x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19010,7 +19010,7 @@ pub unsafe fn vld1q_u64_x3(a: *const u64) -> uint64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19038,7 +19038,7 @@ pub unsafe fn vld1q_u64_x3(a: *const u64) -> uint64x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19062,7 +19062,7 @@ pub unsafe fn vld1q_u64_x4(a: *const u64) -> uint64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_u64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19091,7 +19091,7 @@ pub unsafe fn vld1q_u64_x4(a: *const u64) -> uint64x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19115,7 +19115,7 @@ pub unsafe fn vld1_p8_x2(a: *const p8) -> poly8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19142,7 +19142,7 @@ pub unsafe fn vld1_p8_x2(a: *const p8) -> poly8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19166,7 +19166,7 @@ pub unsafe fn vld1_p8_x3(a: *const p8) -> poly8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19194,7 +19194,7 @@ pub unsafe fn vld1_p8_x3(a: *const p8) -> poly8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19218,7 +19218,7 @@ pub unsafe fn vld1_p8_x4(a: *const p8) -> poly8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19247,7 +19247,7 @@ pub unsafe fn vld1_p8_x4(a: *const p8) -> poly8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19271,7 +19271,7 @@ pub unsafe fn vld1q_p8_x2(a: *const p8) -> poly8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19310,7 +19310,7 @@ pub unsafe fn vld1q_p8_x2(a: *const p8) -> poly8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19334,7 +19334,7 @@ pub unsafe fn vld1q_p8_x3(a: *const p8) -> poly8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19380,7 +19380,7 @@ pub unsafe fn vld1q_p8_x3(a: *const p8) -> poly8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19404,7 +19404,7 @@ pub unsafe fn vld1q_p8_x4(a: *const p8) -> poly8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19457,7 +19457,7 @@ pub unsafe fn vld1q_p8_x4(a: *const p8) -> poly8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19481,7 +19481,7 @@ pub unsafe fn vld1_p16_x2(a: *const p16) -> poly16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19508,7 +19508,7 @@ pub unsafe fn vld1_p16_x2(a: *const p16) -> poly16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19532,7 +19532,7 @@ pub unsafe fn vld1_p16_x3(a: *const p16) -> poly16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19560,7 +19560,7 @@ pub unsafe fn vld1_p16_x3(a: *const p16) -> poly16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19584,7 +19584,7 @@ pub unsafe fn vld1_p16_x4(a: *const p16) -> poly16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19613,7 +19613,7 @@ pub unsafe fn vld1_p16_x4(a: *const p16) -> poly16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19637,7 +19637,7 @@ pub unsafe fn vld1q_p16_x2(a: *const p16) -> poly16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19664,7 +19664,7 @@ pub unsafe fn vld1q_p16_x2(a: *const p16) -> poly16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19688,7 +19688,7 @@ pub unsafe fn vld1q_p16_x3(a: *const p16) -> poly16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19716,7 +19716,7 @@ pub unsafe fn vld1q_p16_x3(a: *const p16) -> poly16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19740,7 +19740,7 @@ pub unsafe fn vld1q_p16_x4(a: *const p16) -> poly16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_p16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -19765,7 +19765,7 @@ pub unsafe fn vld1q_p16_x4(a: *const p16) -> poly16x8x4_t {
     ret_val.3 = unsafe { simd_shuffle!(ret_val.3, ret_val.3, [7, 6, 5, 4, 3, 2, 1, 0]) };
     ret_val
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19778,7 +19778,7 @@ unsafe fn vld1_v1i64(a: *const i8) -> int64x1_t {
     }
     _vld1_v1i64(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19791,7 +19791,7 @@ unsafe fn vld1_v2f32(a: *const i8) -> float32x2_t {
     }
     _vld1_v2f32(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19804,7 +19804,7 @@ unsafe fn vld1_v2i32(a: *const i8) -> int32x2_t {
     }
     _vld1_v2i32(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19817,7 +19817,7 @@ unsafe fn vld1_v4i16(a: *const i8) -> int16x4_t {
     }
     _vld1_v4i16(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19830,7 +19830,7 @@ unsafe fn vld1_v8i8(a: *const i8) -> int8x8_t {
     }
     _vld1_v8i8(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19843,7 +19843,7 @@ unsafe fn vld1q_v16i8(a: *const i8) -> int8x16_t {
     }
     _vld1q_v16i8(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19856,7 +19856,7 @@ unsafe fn vld1q_v2i64(a: *const i8) -> int64x2_t {
     }
     _vld1q_v2i64(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19869,7 +19869,7 @@ unsafe fn vld1q_v4f32(a: *const i8) -> float32x4_t {
     }
     _vld1q_v4f32(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19882,7 +19882,7 @@ unsafe fn vld1q_v4i32(a: *const i8) -> int32x4_t {
     }
     _vld1q_v4i32(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[rustc_legacy_const_generics(1)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
@@ -19895,7 +19895,7 @@ unsafe fn vld1q_v8i16(a: *const i8) -> int16x8_t {
     }
     _vld1q_v8i16(a, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -19909,7 +19909,7 @@ unsafe fn vld1_v4f16(a: *const i8, b: i32) -> float16x4_t {
     }
     _vld1_v4f16(a, b)
 }
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -19927,7 +19927,7 @@ unsafe fn vld1q_v8f16(a: *const i8, b: i32) -> float16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr))]
@@ -19951,7 +19951,7 @@ pub unsafe fn vld1q_dup_p64(ptr: *const p64) -> poly64x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -19970,7 +19970,7 @@ pub unsafe fn vld2_dup_f16(a: *const f16) -> float16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -19989,7 +19989,7 @@ pub unsafe fn vld2q_dup_f16(a: *const f16) -> float16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -20013,7 +20013,7 @@ pub unsafe fn vld2_dup_f16(a: *const f16) -> float16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -20037,7 +20037,7 @@ pub unsafe fn vld2q_dup_f16(a: *const f16) -> float16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20053,7 +20053,7 @@ pub unsafe fn vld2_dup_f32(a: *const f32) -> float32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20069,7 +20069,7 @@ pub unsafe fn vld2q_dup_f32(a: *const f32) -> float32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20085,7 +20085,7 @@ pub unsafe fn vld2_dup_s8(a: *const i8) -> int8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20101,7 +20101,7 @@ pub unsafe fn vld2q_dup_s8(a: *const i8) -> int8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20117,7 +20117,7 @@ pub unsafe fn vld2_dup_s16(a: *const i16) -> int16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20133,7 +20133,7 @@ pub unsafe fn vld2q_dup_s16(a: *const i16) -> int16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20149,7 +20149,7 @@ pub unsafe fn vld2_dup_s32(a: *const i32) -> int32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20165,7 +20165,7 @@ pub unsafe fn vld2q_dup_s32(a: *const i32) -> int32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20184,7 +20184,7 @@ pub unsafe fn vld2_dup_f32(a: *const f32) -> float32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20203,7 +20203,7 @@ pub unsafe fn vld2q_dup_f32(a: *const f32) -> float32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20222,7 +20222,7 @@ pub unsafe fn vld2_dup_s8(a: *const i8) -> int8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20241,7 +20241,7 @@ pub unsafe fn vld2q_dup_s8(a: *const i8) -> int8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20260,7 +20260,7 @@ pub unsafe fn vld2_dup_s16(a: *const i16) -> int16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20279,7 +20279,7 @@ pub unsafe fn vld2q_dup_s16(a: *const i16) -> int16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20298,7 +20298,7 @@ pub unsafe fn vld2_dup_s32(a: *const i32) -> int32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20317,7 +20317,7 @@ pub unsafe fn vld2q_dup_s32(a: *const i32) -> int32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -20340,7 +20340,7 @@ pub unsafe fn vld2_dup_p64(a: *const p64) -> poly64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -20356,7 +20356,7 @@ pub unsafe fn vld2_dup_s64(a: *const i64) -> int64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -20375,7 +20375,7 @@ pub unsafe fn vld2_dup_s64(a: *const i64) -> int64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -20398,7 +20398,7 @@ pub unsafe fn vld2_dup_u64(a: *const u64) -> uint64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20422,7 +20422,7 @@ pub unsafe fn vld2_dup_u8(a: *const u8) -> uint8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20449,7 +20449,7 @@ pub unsafe fn vld2_dup_u8(a: *const u8) -> uint8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20473,7 +20473,7 @@ pub unsafe fn vld2q_dup_u8(a: *const u8) -> uint8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20512,7 +20512,7 @@ pub unsafe fn vld2q_dup_u8(a: *const u8) -> uint8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20536,7 +20536,7 @@ pub unsafe fn vld2_dup_u16(a: *const u16) -> uint16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20563,7 +20563,7 @@ pub unsafe fn vld2_dup_u16(a: *const u16) -> uint16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20587,7 +20587,7 @@ pub unsafe fn vld2q_dup_u16(a: *const u16) -> uint16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20614,7 +20614,7 @@ pub unsafe fn vld2q_dup_u16(a: *const u16) -> uint16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20638,7 +20638,7 @@ pub unsafe fn vld2_dup_u32(a: *const u32) -> uint32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20665,7 +20665,7 @@ pub unsafe fn vld2_dup_u32(a: *const u32) -> uint32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20689,7 +20689,7 @@ pub unsafe fn vld2q_dup_u32(a: *const u32) -> uint32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20716,7 +20716,7 @@ pub unsafe fn vld2q_dup_u32(a: *const u32) -> uint32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20740,7 +20740,7 @@ pub unsafe fn vld2_dup_p8(a: *const p8) -> poly8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20767,7 +20767,7 @@ pub unsafe fn vld2_dup_p8(a: *const p8) -> poly8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20791,7 +20791,7 @@ pub unsafe fn vld2q_dup_p8(a: *const p8) -> poly8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20830,7 +20830,7 @@ pub unsafe fn vld2q_dup_p8(a: *const p8) -> poly8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20854,7 +20854,7 @@ pub unsafe fn vld2_dup_p16(a: *const p16) -> poly16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20881,7 +20881,7 @@ pub unsafe fn vld2_dup_p16(a: *const p16) -> poly16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20905,7 +20905,7 @@ pub unsafe fn vld2q_dup_p16(a: *const p16) -> poly16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -20932,7 +20932,7 @@ pub unsafe fn vld2q_dup_p16(a: *const p16) -> poly16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -20951,7 +20951,7 @@ pub unsafe fn vld2_f16(a: *const f16) -> float16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -20970,7 +20970,7 @@ pub unsafe fn vld2q_f16(a: *const f16) -> float16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -20994,7 +20994,7 @@ pub unsafe fn vld2_f16(a: *const f16) -> float16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -21018,7 +21018,7 @@ pub unsafe fn vld2q_f16(a: *const f16) -> float16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21034,7 +21034,7 @@ pub unsafe fn vld2_f32(a: *const f32) -> float32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21050,7 +21050,7 @@ pub unsafe fn vld2q_f32(a: *const f32) -> float32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21066,7 +21066,7 @@ pub unsafe fn vld2_s8(a: *const i8) -> int8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21082,7 +21082,7 @@ pub unsafe fn vld2q_s8(a: *const i8) -> int8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21098,7 +21098,7 @@ pub unsafe fn vld2_s16(a: *const i16) -> int16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21114,7 +21114,7 @@ pub unsafe fn vld2q_s16(a: *const i16) -> int16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21130,7 +21130,7 @@ pub unsafe fn vld2_s32(a: *const i32) -> int32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21146,7 +21146,7 @@ pub unsafe fn vld2q_s32(a: *const i32) -> int32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21165,7 +21165,7 @@ pub unsafe fn vld2_f32(a: *const f32) -> float32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21184,7 +21184,7 @@ pub unsafe fn vld2q_f32(a: *const f32) -> float32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21203,7 +21203,7 @@ pub unsafe fn vld2_s8(a: *const i8) -> int8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21222,7 +21222,7 @@ pub unsafe fn vld2q_s8(a: *const i8) -> int8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21241,7 +21241,7 @@ pub unsafe fn vld2_s16(a: *const i16) -> int16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21260,7 +21260,7 @@ pub unsafe fn vld2q_s16(a: *const i16) -> int16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21279,7 +21279,7 @@ pub unsafe fn vld2_s32(a: *const i32) -> int32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21298,7 +21298,7 @@ pub unsafe fn vld2q_s32(a: *const i32) -> int32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21324,7 +21324,7 @@ pub unsafe fn vld2_lane_f16(a: *const f16, b: float16x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21350,7 +21350,7 @@ pub unsafe fn vld2q_lane_f16(a: *const f16, b: float16x8x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -21377,7 +21377,7 @@ pub unsafe fn vld2_lane_f16(a: *const f16, b: float16x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -21408,7 +21408,7 @@ pub unsafe fn vld2q_lane_f16(a: *const f16, b: float16x8x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21429,7 +21429,7 @@ pub unsafe fn vld2_lane_f32(a: *const f32, b: float32x2x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21451,7 +21451,7 @@ pub unsafe fn vld2q_lane_f32(a: *const f32, b: float32x4x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21472,7 +21472,7 @@ pub unsafe fn vld2_lane_s8(a: *const i8, b: int8x8x2_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21493,7 +21493,7 @@ pub unsafe fn vld2_lane_s16(a: *const i16, b: int16x4x2_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21514,7 +21514,7 @@ pub unsafe fn vld2q_lane_s16(a: *const i16, b: int16x8x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21535,7 +21535,7 @@ pub unsafe fn vld2_lane_s32(a: *const i32, b: int32x2x2_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld2, LANE = 0))]
@@ -21556,7 +21556,7 @@ pub unsafe fn vld2q_lane_s32(a: *const i32, b: int32x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21580,7 +21580,7 @@ pub unsafe fn vld2_lane_f32(a: *const f32, b: float32x2x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21604,7 +21604,7 @@ pub unsafe fn vld2q_lane_f32(a: *const f32, b: float32x4x2_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21628,7 +21628,7 @@ pub unsafe fn vld2q_lane_s16(a: *const i16, b: int16x8x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21652,7 +21652,7 @@ pub unsafe fn vld2q_lane_s32(a: *const i32, b: int32x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21671,7 +21671,7 @@ pub unsafe fn vld2_lane_s8(a: *const i8, b: int8x8x2_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21695,7 +21695,7 @@ pub unsafe fn vld2_lane_s16(a: *const i16, b: int16x4x2_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld2, LANE = 0))]
@@ -21719,7 +21719,7 @@ pub unsafe fn vld2_lane_s32(a: *const i32, b: int32x2x2_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21744,7 +21744,7 @@ pub unsafe fn vld2_lane_u8(a: *const u8, b: uint8x8x2_t) -> uin
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21769,7 +21769,7 @@ pub unsafe fn vld2_lane_u16(a: *const u16, b: uint16x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21794,7 +21794,7 @@ pub unsafe fn vld2q_lane_u16(a: *const u16, b: uint16x8x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21819,7 +21819,7 @@ pub unsafe fn vld2_lane_u32(a: *const u32, b: uint32x2x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21844,7 +21844,7 @@ pub unsafe fn vld2q_lane_u32(a: *const u32, b: uint32x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21869,7 +21869,7 @@ pub unsafe fn vld2_lane_p8(a: *const p8, b: poly8x8x2_t) -> pol
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21894,7 +21894,7 @@ pub unsafe fn vld2_lane_p16(a: *const p16, b: poly16x4x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld2, LANE = 0))]
@@ -21919,7 +21919,7 @@ pub unsafe fn vld2q_lane_p16(a: *const p16, b: poly16x8x2_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -21942,7 +21942,7 @@ pub unsafe fn vld2_p64(a: *const p64) -> poly64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -21958,7 +21958,7 @@ pub unsafe fn vld2_s64(a: *const i64) -> int64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -21977,7 +21977,7 @@ pub unsafe fn vld2_s64(a: *const i64) -> int64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -22000,7 +22000,7 @@ pub unsafe fn vld2_u64(a: *const u64) -> uint64x1x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22024,7 +22024,7 @@ pub unsafe fn vld2_u8(a: *const u8) -> uint8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22051,7 +22051,7 @@ pub unsafe fn vld2_u8(a: *const u8) -> uint8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22075,7 +22075,7 @@ pub unsafe fn vld2q_u8(a: *const u8) -> uint8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22114,7 +22114,7 @@ pub unsafe fn vld2q_u8(a: *const u8) -> uint8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22138,7 +22138,7 @@ pub unsafe fn vld2_u16(a: *const u16) -> uint16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22165,7 +22165,7 @@ pub unsafe fn vld2_u16(a: *const u16) -> uint16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22189,7 +22189,7 @@ pub unsafe fn vld2q_u16(a: *const u16) -> uint16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22216,7 +22216,7 @@ pub unsafe fn vld2q_u16(a: *const u16) -> uint16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22240,7 +22240,7 @@ pub unsafe fn vld2_u32(a: *const u32) -> uint32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22267,7 +22267,7 @@ pub unsafe fn vld2_u32(a: *const u32) -> uint32x2x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22291,7 +22291,7 @@ pub unsafe fn vld2q_u32(a: *const u32) -> uint32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22318,7 +22318,7 @@ pub unsafe fn vld2q_u32(a: *const u32) -> uint32x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22342,7 +22342,7 @@ pub unsafe fn vld2_p8(a: *const p8) -> poly8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22369,7 +22369,7 @@ pub unsafe fn vld2_p8(a: *const p8) -> poly8x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22393,7 +22393,7 @@ pub unsafe fn vld2q_p8(a: *const p8) -> poly8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22432,7 +22432,7 @@ pub unsafe fn vld2q_p8(a: *const p8) -> poly8x16x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22456,7 +22456,7 @@ pub unsafe fn vld2_p16(a: *const p16) -> poly16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22483,7 +22483,7 @@ pub unsafe fn vld2_p16(a: *const p16) -> poly16x4x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22507,7 +22507,7 @@ pub unsafe fn vld2q_p16(a: *const p16) -> poly16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld2q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -22534,7 +22534,7 @@ pub unsafe fn vld2q_p16(a: *const p16) -> poly16x8x2_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -22553,7 +22553,7 @@ pub unsafe fn vld3_dup_f16(a: *const f16) -> float16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -22572,7 +22572,7 @@ pub unsafe fn vld3q_dup_f16(a: *const f16) -> float16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -22596,7 +22596,7 @@ pub unsafe fn vld3_dup_f16(a: *const f16) -> float16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -22620,7 +22620,7 @@ pub unsafe fn vld3q_dup_f16(a: *const f16) -> float16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22639,7 +22639,7 @@ pub unsafe fn vld3_dup_f32(a: *const f32) -> float32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22658,7 +22658,7 @@ pub unsafe fn vld3q_dup_f32(a: *const f32) -> float32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22677,7 +22677,7 @@ pub unsafe fn vld3_dup_s8(a: *const i8) -> int8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22696,7 +22696,7 @@ pub unsafe fn vld3q_dup_s8(a: *const i8) -> int8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22715,7 +22715,7 @@ pub unsafe fn vld3_dup_s16(a: *const i16) -> int16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22734,7 +22734,7 @@ pub unsafe fn vld3q_dup_s16(a: *const i16) -> int16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22753,7 +22753,7 @@ pub unsafe fn vld3_dup_s32(a: *const i32) -> int32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22772,7 +22772,7 @@ pub unsafe fn vld3q_dup_s32(a: *const i32) -> int32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -22791,7 +22791,7 @@ pub unsafe fn vld3_dup_s64(a: *const i64) -> int64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22807,7 +22807,7 @@ pub unsafe fn vld3_dup_f32(a: *const f32) -> float32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22823,7 +22823,7 @@ pub unsafe fn vld3q_dup_f32(a: *const f32) -> float32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22839,7 +22839,7 @@ pub unsafe fn vld3_dup_s8(a: *const i8) -> int8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22855,7 +22855,7 @@ pub unsafe fn vld3q_dup_s8(a: *const i8) -> int8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22871,7 +22871,7 @@ pub unsafe fn vld3_dup_s16(a: *const i16) -> int16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22887,7 +22887,7 @@ pub unsafe fn vld3q_dup_s16(a: *const i16) -> int16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22903,7 +22903,7 @@ pub unsafe fn vld3_dup_s32(a: *const i32) -> int32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22919,7 +22919,7 @@ pub unsafe fn vld3q_dup_s32(a: *const i32) -> int32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -22942,7 +22942,7 @@ pub unsafe fn vld3_dup_p64(a: *const p64) -> poly64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -22958,7 +22958,7 @@ pub unsafe fn vld3_dup_s64(a: *const i64) -> int64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -22981,7 +22981,7 @@ pub unsafe fn vld3_dup_u64(a: *const u64) -> uint64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23005,7 +23005,7 @@ pub unsafe fn vld3_dup_u8(a: *const u8) -> uint8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23033,7 +23033,7 @@ pub unsafe fn vld3_dup_u8(a: *const u8) -> uint8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23057,7 +23057,7 @@ pub unsafe fn vld3q_dup_u8(a: *const u8) -> uint8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23103,7 +23103,7 @@ pub unsafe fn vld3q_dup_u8(a: *const u8) -> uint8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23127,7 +23127,7 @@ pub unsafe fn vld3_dup_u16(a: *const u16) -> uint16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23155,7 +23155,7 @@ pub unsafe fn vld3_dup_u16(a: *const u16) -> uint16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23179,7 +23179,7 @@ pub unsafe fn vld3q_dup_u16(a: *const u16) -> uint16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23207,7 +23207,7 @@ pub unsafe fn vld3q_dup_u16(a: *const u16) -> uint16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23231,7 +23231,7 @@ pub unsafe fn vld3_dup_u32(a: *const u32) -> uint32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23259,7 +23259,7 @@ pub unsafe fn vld3_dup_u32(a: *const u32) -> uint32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23283,7 +23283,7 @@ pub unsafe fn vld3q_dup_u32(a: *const u32) -> uint32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23311,7 +23311,7 @@ pub unsafe fn vld3q_dup_u32(a: *const u32) -> uint32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23335,7 +23335,7 @@ pub unsafe fn vld3_dup_p8(a: *const p8) -> poly8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23363,7 +23363,7 @@ pub unsafe fn vld3_dup_p8(a: *const p8) -> poly8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23387,7 +23387,7 @@ pub unsafe fn vld3q_dup_p8(a: *const p8) -> poly8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23433,7 +23433,7 @@ pub unsafe fn vld3q_dup_p8(a: *const p8) -> poly8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23457,7 +23457,7 @@ pub unsafe fn vld3_dup_p16(a: *const p16) -> poly16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23485,7 +23485,7 @@ pub unsafe fn vld3_dup_p16(a: *const p16) -> poly16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23509,7 +23509,7 @@ pub unsafe fn vld3q_dup_p16(a: *const p16) -> poly16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -23537,7 +23537,7 @@ pub unsafe fn vld3q_dup_p16(a: *const p16) -> poly16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -23556,7 +23556,7 @@ pub unsafe fn vld3_f16(a: *const f16) -> float16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
@@ -23575,7 +23575,7 @@ pub unsafe fn vld3q_f16(a: *const f16) -> float16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -23599,7 +23599,7 @@ pub unsafe fn vld3_f16(a: *const f16) -> float16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -23623,7 +23623,7 @@ pub unsafe fn vld3q_f16(a: *const f16) -> float16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23642,7 +23642,7 @@ pub unsafe fn vld3_f32(a: *const f32) -> float32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23661,7 +23661,7 @@ pub unsafe fn vld3q_f32(a: *const f32) -> float32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23680,7 +23680,7 @@ pub unsafe fn vld3_s8(a: *const i8) -> int8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23699,7 +23699,7 @@ pub unsafe fn vld3q_s8(a: *const i8) -> int8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23718,7 +23718,7 @@ pub unsafe fn vld3_s16(a: *const i16) -> int16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23737,7 +23737,7 @@ pub unsafe fn vld3q_s16(a: *const i16) -> int16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23756,7 +23756,7 @@ pub unsafe fn vld3_s32(a: *const i32) -> int32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -23775,7 +23775,7 @@ pub unsafe fn vld3q_s32(a: *const i32) -> int32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23791,7 +23791,7 @@ pub unsafe fn vld3_f32(a: *const f32) -> float32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23807,7 +23807,7 @@ pub unsafe fn vld3q_f32(a: *const f32) -> float32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23823,7 +23823,7 @@ pub unsafe fn vld3_s8(a: *const i8) -> int8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23839,7 +23839,7 @@ pub unsafe fn vld3q_s8(a: *const i8) -> int8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23855,7 +23855,7 @@ pub unsafe fn vld3_s16(a: *const i16) -> int16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23871,7 +23871,7 @@ pub unsafe fn vld3q_s16(a: *const i16) -> int16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23887,7 +23887,7 @@ pub unsafe fn vld3_s32(a: *const i32) -> int32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -23903,7 +23903,7 @@ pub unsafe fn vld3q_s32(a: *const i32) -> int32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -23930,7 +23930,7 @@ pub unsafe fn vld3_lane_f16(a: *const f16, b: float16x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -23957,7 +23957,7 @@ pub unsafe fn vld3q_lane_f16(a: *const f16, b: float16x8x3_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -23989,7 +23989,7 @@ pub unsafe fn vld3_lane_f16(a: *const f16, b: float16x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
@@ -24021,7 +24021,7 @@ pub unsafe fn vld3q_lane_f16(a: *const f16, b: float16x8x3_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24048,7 +24048,7 @@ pub unsafe fn vld3_lane_f32(a: *const f32, b: float32x2x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24075,7 +24075,7 @@ pub unsafe fn vld3q_lane_f32(a: *const f32, b: float32x4x3_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -24100,7 +24100,7 @@ pub unsafe fn vld3_lane_f32(a: *const f32, b: float32x2x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24127,7 +24127,7 @@ pub unsafe fn vld3_lane_s8(a: *const i8, b: int8x8x3_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24154,7 +24154,7 @@ pub unsafe fn vld3_lane_s16(a: *const i16, b: int16x4x3_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24181,7 +24181,7 @@ pub unsafe fn vld3q_lane_s16(a: *const i16, b: int16x8x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24208,7 +24208,7 @@ pub unsafe fn vld3_lane_s32(a: *const i32, b: int32x2x3_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld3, LANE = 0))]
@@ -24235,7 +24235,7 @@ pub unsafe fn vld3q_lane_s32(a: *const i32, b: int32x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -24260,7 +24260,7 @@ pub unsafe fn vld3_lane_s8(a: *const i8, b: int8x8x3_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -24285,7 +24285,7 @@ pub unsafe fn vld3_lane_s16(a: *const i16, b: int16x4x3_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -24310,7 +24310,7 @@ pub unsafe fn vld3q_lane_s16(a: *const i16, b: int16x8x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -24335,7 +24335,7 @@ pub unsafe fn vld3_lane_s32(a: *const i32, b: int32x2x3_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -24360,7 +24360,7 @@ pub unsafe fn vld3q_lane_s32(a: *const i32, b: int32x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24385,7 +24385,7 @@ pub unsafe fn vld3_lane_u8(a: *const u8, b: uint8x8x3_t) -> uin
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24410,7 +24410,7 @@ pub unsafe fn vld3_lane_u16(a: *const u16, b: uint16x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24435,7 +24435,7 @@ pub unsafe fn vld3q_lane_u16(a: *const u16, b: uint16x8x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24460,7 +24460,7 @@ pub unsafe fn vld3_lane_u32(a: *const u32, b: uint32x2x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24485,7 +24485,7 @@ pub unsafe fn vld3q_lane_u32(a: *const u32, b: uint32x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24510,7 +24510,7 @@ pub unsafe fn vld3_lane_p8(a: *const p8, b: poly8x8x3_t) -> pol
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24535,7 +24535,7 @@ pub unsafe fn vld3_lane_p16(a: *const p16, b: poly16x4x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld3, LANE = 0))]
@@ -24560,7 +24560,7 @@ pub unsafe fn vld3q_lane_p16(a: *const p16, b: poly16x8x3_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -24583,7 +24583,7 @@ pub unsafe fn vld3_p64(a: *const p64) -> poly64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
 #[cfg(not(target_arch = "arm"))]
@@ -24602,7 +24602,7 @@ pub unsafe fn vld3_s64(a: *const i64) -> int64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -24618,7 +24618,7 @@ pub unsafe fn vld3_s64(a: *const i64) -> int64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -24641,7 +24641,7 @@ pub unsafe fn vld3_u64(a: *const u64) -> uint64x1x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24665,7 +24665,7 @@ pub unsafe fn vld3_u8(a: *const u8) -> uint8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24693,7 +24693,7 @@ pub unsafe fn vld3_u8(a: *const u8) -> uint8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24717,7 +24717,7 @@ pub unsafe fn vld3q_u8(a: *const u8) -> uint8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24763,7 +24763,7 @@ pub unsafe fn vld3q_u8(a: *const u8) -> uint8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24787,7 +24787,7 @@ pub unsafe fn vld3_u16(a: *const u16) -> uint16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24815,7 +24815,7 @@ pub unsafe fn vld3_u16(a: *const u16) -> uint16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24839,7 +24839,7 @@ pub unsafe fn vld3q_u16(a: *const u16) -> uint16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24867,7 +24867,7 @@ pub unsafe fn vld3q_u16(a: *const u16) -> uint16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24891,7 +24891,7 @@ pub unsafe fn vld3_u32(a: *const u32) -> uint32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24919,7 +24919,7 @@ pub unsafe fn vld3_u32(a: *const u32) -> uint32x2x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24943,7 +24943,7 @@ pub unsafe fn vld3q_u32(a: *const u32) -> uint32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24971,7 +24971,7 @@ pub unsafe fn vld3q_u32(a: *const u32) -> uint32x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -24995,7 +24995,7 @@ pub unsafe fn vld3_p8(a: *const p8) -> poly8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25023,7 +25023,7 @@ pub unsafe fn vld3_p8(a: *const p8) -> poly8x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25047,7 +25047,7 @@ pub unsafe fn vld3q_p8(a: *const p8) -> poly8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25093,7 +25093,7 @@ pub unsafe fn vld3q_p8(a: *const p8) -> poly8x16x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25117,7 +25117,7 @@ pub unsafe fn vld3_p16(a: *const p16) -> poly16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25145,7 +25145,7 @@ pub unsafe fn vld3_p16(a: *const p16) -> poly16x4x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25169,7 +25169,7 @@ pub unsafe fn vld3q_p16(a: *const p16) -> poly16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25197,7 +25197,7 @@ pub unsafe fn vld3q_p16(a: *const p16) -> poly16x8x3_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld3q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld3, LANE = 0))]
@@ -25222,7 +25222,7 @@ pub unsafe fn vld3q_lane_f32(a: *const f32, b: float32x4x3_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))]
@@ -25240,7 +25240,7 @@ pub unsafe fn vld4_dup_f16(a: *const f16) -> float16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))]
@@ -25258,7 +25258,7 @@ pub unsafe fn vld4q_dup_f16(a: *const f16) -> float16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
     all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
@@ -25281,7 +25281,7 @@ pub unsafe fn vld4_dup_f16(a: *const f16) -> float16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
     all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
@@ -25304,7 +25304,7 @@ pub unsafe fn vld4q_dup_f16(a: *const f16) -> float16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25320,7 +25320,7 @@ pub unsafe fn vld4_dup_f32(a: *const f32) -> float32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25336,7 +25336,7 @@ pub unsafe fn vld4q_dup_f32(a: *const f32) -> float32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25352,7 +25352,7 @@ pub unsafe fn vld4_dup_s8(a: *const i8) -> int8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25368,7 +25368,7 @@ pub unsafe fn vld4q_dup_s8(a: *const i8) -> int8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25384,7 +25384,7 @@ pub unsafe fn vld4_dup_s16(a: *const i16) -> int16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25400,7 +25400,7 @@ pub unsafe fn vld4q_dup_s16(a: *const i16) -> int16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25416,7 +25416,7 @@ pub unsafe fn vld4_dup_s32(a: *const i32) -> int32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vld4))]
@@ -25432,7 +25432,7 @@ pub unsafe fn vld4q_dup_s32(a: *const i32) -> int32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25451,7 +25451,7 @@ pub unsafe fn vld4_dup_f32(a: *const f32) -> float32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25470,7 +25470,7 @@ pub unsafe fn vld4q_dup_f32(a: *const f32) -> float32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25489,7 +25489,7 @@ pub unsafe fn vld4_dup_s8(a: *const i8) -> int8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25508,7 +25508,7 @@ pub unsafe fn vld4q_dup_s8(a: *const i8) -> int8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25527,7 +25527,7 @@ pub unsafe fn vld4_dup_s16(a: *const i16) -> int16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25546,7 +25546,7 @@ pub unsafe fn vld4q_dup_s16(a: *const i16) -> int16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25565,7 +25565,7 @@ pub unsafe fn vld4_dup_s32(a: *const i32) -> int32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25584,7 +25584,7 @@ pub unsafe fn vld4q_dup_s32(a: *const i32) -> int32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4r))]
@@ -25603,7 +25603,7 @@ pub unsafe fn vld4_dup_s64(a: *const i64) -> int64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -25626,7 +25626,7 @@ pub unsafe fn vld4_dup_p64(a: *const p64) -> poly64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(nop))]
@@ -25642,7 +25642,7 @@ pub unsafe fn vld4_dup_s64(a: *const i64) -> int64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -25665,7 +25665,7 @@ pub unsafe fn vld4_dup_u64(a: *const u64) -> uint64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25689,7 +25689,7 @@ pub unsafe fn vld4_dup_u8(a: *const u8) -> uint8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25718,7 +25718,7 @@ pub unsafe fn vld4_dup_u8(a: *const u8) -> uint8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25742,7 +25742,7 @@ pub unsafe fn vld4q_dup_u8(a: *const u8) -> uint8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25795,7 +25795,7 @@ pub unsafe fn vld4q_dup_u8(a: *const u8) -> uint8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25819,7 +25819,7 @@ pub unsafe fn vld4_dup_u16(a: *const u16) -> uint16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25848,7 +25848,7 @@ pub unsafe fn vld4_dup_u16(a: *const u16) -> uint16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25872,7 +25872,7 @@ pub unsafe fn vld4q_dup_u16(a: *const u16) -> uint16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25901,7 +25901,7 @@ pub unsafe fn vld4q_dup_u16(a: *const u16) -> uint16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25925,7 +25925,7 @@ pub unsafe fn vld4_dup_u32(a: *const u32) -> uint32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25954,7 +25954,7 @@ pub unsafe fn vld4_dup_u32(a: *const u32) -> uint32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -25978,7 +25978,7 @@ pub unsafe fn vld4q_dup_u32(a: *const u32) -> uint32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26007,7 +26007,7 @@ pub unsafe fn vld4q_dup_u32(a: *const u32) -> uint32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26031,7 +26031,7 @@ pub unsafe fn vld4_dup_p8(a: *const p8) -> poly8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26060,7 +26060,7 @@ pub unsafe fn vld4_dup_p8(a: *const p8) -> poly8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26084,7 +26084,7 @@ pub unsafe fn vld4q_dup_p8(a: *const p8) -> poly8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26137,7 +26137,7 @@ pub unsafe fn vld4q_dup_p8(a: *const p8) -> poly8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26161,7 +26161,7 @@ pub unsafe fn vld4_dup_p16(a: *const p16) -> poly16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26190,7 +26190,7 @@ pub unsafe fn vld4_dup_p16(a: *const p16) -> poly16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26214,7 +26214,7 @@ pub unsafe fn vld4q_dup_p16(a: *const p16) -> poly16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_dup_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -26243,7 +26243,7 @@ pub unsafe fn vld4q_dup_p16(a: *const p16) -> poly16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))]
@@ -26261,7 +26261,7 @@ pub unsafe fn vld4_f16(a: *const f16) -> float16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4))]
@@ -26279,7 +26279,7 @@ pub unsafe fn vld4q_f16(a: *const f16) -> float16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
     all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
@@ -26302,7 +26302,7 @@ pub unsafe fn vld4_f16(a: *const f16) -> float16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
     all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
@@ -26325,7 +26325,7 @@ pub unsafe fn vld4q_f16(a: *const f16) -> float16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26344,7 +26344,7 @@ pub unsafe fn vld4_f32(a: *const f32) -> float32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26363,7 +26363,7 @@ pub unsafe fn vld4q_f32(a: *const f32) -> float32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26382,7 +26382,7 @@ pub unsafe fn vld4_s8(a: *const i8) -> int8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26401,7 +26401,7 @@ pub unsafe fn vld4q_s8(a: *const i8) -> int8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26420,7 +26420,7 @@ pub unsafe fn vld4_s16(a: *const i16) -> int16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26439,7 +26439,7 @@ pub unsafe fn vld4q_s16(a: *const i16) -> int16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26458,7 +26458,7 @@ pub unsafe fn vld4_s32(a: *const i32) -> int32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -26477,7 +26477,7 @@ pub unsafe fn vld4q_s32(a: *const i32) -> int32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26493,7 +26493,7 @@ pub unsafe fn vld4_f32(a: *const f32) -> float32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26509,7 +26509,7 @@ pub unsafe fn vld4q_f32(a: *const f32) -> float32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26525,7 +26525,7 @@ pub unsafe fn vld4_s8(a: *const i8) -> int8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26541,7 +26541,7 @@ pub unsafe fn vld4q_s8(a: *const i8) -> int8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26557,7 +26557,7 @@ pub unsafe fn vld4_s16(a: *const i16) -> int16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26573,7 +26573,7 @@ pub unsafe fn vld4q_s16(a: *const i16) -> int16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26589,7 +26589,7 @@ pub unsafe fn vld4_s32(a: *const i32) -> int32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -26605,7 +26605,7 @@ pub unsafe fn vld4q_s32(a: *const i32) -> int32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -26633,7 +26633,7 @@ pub unsafe fn vld4_lane_f16(a: *const f16, b: float16x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -26661,7 +26661,7 @@ pub unsafe fn vld4q_lane_f16(a: *const f16, b: float16x8x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
     all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
@@ -26693,7 +26693,7 @@ pub unsafe fn vld4_lane_f16(a: *const f16, b: float16x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(
     all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
@@ -26725,7 +26725,7 @@ pub unsafe fn vld4q_lane_f16(a: *const f16, b: float16x8x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26753,7 +26753,7 @@ pub unsafe fn vld4_lane_f32(a: *const f32, b: float32x2x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26781,7 +26781,7 @@ pub unsafe fn vld4q_lane_f32(a: *const f32, b: float32x4x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26809,7 +26809,7 @@ pub unsafe fn vld4_lane_s8(a: *const i8, b: int8x8x4_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26837,7 +26837,7 @@ pub unsafe fn vld4_lane_s16(a: *const i16, b: int16x4x4_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26865,7 +26865,7 @@ pub unsafe fn vld4q_lane_s16(a: *const i16, b: int16x8x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26893,7 +26893,7 @@ pub unsafe fn vld4_lane_s32(a: *const i32, b: int32x2x4_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(ld4, LANE = 0))]
@@ -26921,7 +26921,7 @@ pub unsafe fn vld4q_lane_s32(a: *const i32, b: int32x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -26947,7 +26947,7 @@ pub unsafe fn vld4_lane_f32(a: *const f32, b: float32x2x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -26973,7 +26973,7 @@ pub unsafe fn vld4q_lane_f32(a: *const f32, b: float32x4x4_t) -
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -26999,7 +26999,7 @@ pub unsafe fn vld4_lane_s8(a: *const i8, b: int8x8x4_t) -> int8
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -27025,7 +27025,7 @@ pub unsafe fn vld4_lane_s16(a: *const i16, b: int16x4x4_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -27051,7 +27051,7 @@ pub unsafe fn vld4q_lane_s16(a: *const i16, b: int16x8x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -27077,7 +27077,7 @@ pub unsafe fn vld4_lane_s32(a: *const i32, b: int32x2x4_t) -> i
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(test, assert_instr(vld4, LANE = 0))]
@@ -27103,7 +27103,7 @@ pub unsafe fn vld4q_lane_s32(a: *const i32, b: int32x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27128,7 +27128,7 @@ pub unsafe fn vld4_lane_u8(a: *const u8, b: uint8x8x4_t) -> uin
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27153,7 +27153,7 @@ pub unsafe fn vld4_lane_u16(a: *const u16, b: uint16x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27178,7 +27178,7 @@ pub unsafe fn vld4q_lane_u16(a: *const u16, b: uint16x8x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27203,7 +27203,7 @@ pub unsafe fn vld4_lane_u32(a: *const u32, b: uint32x2x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27228,7 +27228,7 @@ pub unsafe fn vld4q_lane_u32(a: *const u32, b: uint32x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27253,7 +27253,7 @@ pub unsafe fn vld4_lane_p8(a: *const p8, b: poly8x8x4_t) -> pol
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27278,7 +27278,7 @@ pub unsafe fn vld4_lane_p16(a: *const p16, b: poly16x4x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vld4, LANE = 0))]
@@ -27303,7 +27303,7 @@ pub unsafe fn vld4q_lane_p16(a: *const p16, b: poly16x8x4_t) ->
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -27326,7 +27326,7 @@ pub unsafe fn vld4_p64(a: *const p64) -> poly64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -27345,7 +27345,7 @@ pub unsafe fn vld4_s64(a: *const i64) -> int64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -27361,7 +27361,7 @@ pub unsafe fn vld4_s64(a: *const i64) -> int64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -27384,7 +27384,7 @@ pub unsafe fn vld4_u64(a: *const u64) -> uint64x1x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27408,7 +27408,7 @@ pub unsafe fn vld4_u8(a: *const u8) -> uint8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27437,7 +27437,7 @@ pub unsafe fn vld4_u8(a: *const u8) -> uint8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27461,7 +27461,7 @@ pub unsafe fn vld4q_u8(a: *const u8) -> uint8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27514,7 +27514,7 @@ pub unsafe fn vld4q_u8(a: *const u8) -> uint8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27538,7 +27538,7 @@ pub unsafe fn vld4_u16(a: *const u16) -> uint16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27567,7 +27567,7 @@ pub unsafe fn vld4_u16(a: *const u16) -> uint16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27591,7 +27591,7 @@ pub unsafe fn vld4q_u16(a: *const u16) -> uint16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27620,7 +27620,7 @@ pub unsafe fn vld4q_u16(a: *const u16) -> uint16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27644,7 +27644,7 @@ pub unsafe fn vld4_u32(a: *const u32) -> uint32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27673,7 +27673,7 @@ pub unsafe fn vld4_u32(a: *const u32) -> uint32x2x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27697,7 +27697,7 @@ pub unsafe fn vld4q_u32(a: *const u32) -> uint32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27726,7 +27726,7 @@ pub unsafe fn vld4q_u32(a: *const u32) -> uint32x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27750,7 +27750,7 @@ pub unsafe fn vld4_p8(a: *const p8) -> poly8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27779,7 +27779,7 @@ pub unsafe fn vld4_p8(a: *const p8) -> poly8x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27803,7 +27803,7 @@ pub unsafe fn vld4q_p8(a: *const p8) -> poly8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27856,7 +27856,7 @@ pub unsafe fn vld4q_p8(a: *const p8) -> poly8x16x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27880,7 +27880,7 @@ pub unsafe fn vld4_p16(a: *const p16) -> poly16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27909,7 +27909,7 @@ pub unsafe fn vld4_p16(a: *const p16) -> poly16x4x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27933,7 +27933,7 @@ pub unsafe fn vld4q_p16(a: *const p16) -> poly16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld4q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -27962,7 +27962,7 @@ pub unsafe fn vld4q_p16(a: *const p16) -> poly16x8x4_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vldrq_p128)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -27983,7 +27983,7 @@ pub unsafe fn vldrq_p128(a: *const p128) -> p128 {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
 #[cfg_attr(
@@ -28013,7 +28013,7 @@ pub fn vmax_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
 #[cfg_attr(
@@ -28043,7 +28043,7 @@ pub fn vmaxq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28072,7 +28072,7 @@ pub fn vmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28101,7 +28101,7 @@ pub fn vmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28125,7 +28125,7 @@ pub fn vmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28149,7 +28149,7 @@ pub fn vmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28173,7 +28173,7 @@ pub fn vmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28197,7 +28197,7 @@ pub fn vmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28221,7 +28221,7 @@ pub fn vmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28245,7 +28245,7 @@ pub fn vmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28269,7 +28269,7 @@ pub fn vmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28293,7 +28293,7 @@ pub fn vmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28317,7 +28317,7 @@ pub fn vmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28341,7 +28341,7 @@ pub fn vmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmax_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28365,7 +28365,7 @@ pub fn vmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Maximum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))]
@@ -28389,7 +28389,7 @@ pub fn vmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point Maximum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnm_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))]
 #[cfg_attr(
@@ -28411,7 +28411,7 @@ pub fn vmaxnm_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point Maximum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))]
 #[cfg_attr(
@@ -28433,7 +28433,7 @@ pub fn vmaxnmq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point Maximum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnm_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))]
@@ -28454,7 +28454,7 @@ pub fn vmaxnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point Maximum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmaxnmq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))]
@@ -28475,7 +28475,7 @@ pub fn vmaxnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
 #[cfg_attr(
@@ -28505,7 +28505,7 @@ pub fn vmin_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
 #[cfg_attr(
@@ -28535,7 +28535,7 @@ pub fn vminq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28564,7 +28564,7 @@ pub fn vmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28593,7 +28593,7 @@ pub fn vminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28617,7 +28617,7 @@ pub fn vmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28641,7 +28641,7 @@ pub fn vminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28665,7 +28665,7 @@ pub fn vmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28689,7 +28689,7 @@ pub fn vminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28713,7 +28713,7 @@ pub fn vmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28737,7 +28737,7 @@ pub fn vminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28761,7 +28761,7 @@ pub fn vmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28785,7 +28785,7 @@ pub fn vminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28809,7 +28809,7 @@ pub fn vmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28833,7 +28833,7 @@ pub fn vminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmin_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28857,7 +28857,7 @@ pub fn vmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Minimum (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))]
@@ -28881,7 +28881,7 @@ pub fn vminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point Minimum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnm_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))]
 #[cfg_attr(
@@ -28903,7 +28903,7 @@ pub fn vminnm_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point Minimum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))]
 #[cfg_attr(
@@ -28925,7 +28925,7 @@ pub fn vminnmq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point Minimum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnm_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))]
@@ -28946,7 +28946,7 @@ pub fn vminnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point Minimum Number (vector)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vminnmq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))]
@@ -28967,7 +28967,7 @@ pub fn vminnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Floating-point multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))]
@@ -28988,7 +28988,7 @@ pub fn vmla_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))]
@@ -29009,7 +29009,7 @@ pub fn vmlaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))]
@@ -29036,7 +29036,7 @@ pub fn vmla_lane_f32(
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))]
@@ -29063,7 +29063,7 @@ pub fn vmla_laneq_f32(
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))]
@@ -29096,7 +29096,7 @@ pub fn vmlaq_lane_f32(
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))]
@@ -29129,7 +29129,7 @@ pub fn vmlaq_laneq_f32(
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29158,7 +29158,7 @@ pub fn vmla_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t)
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29187,7 +29187,7 @@ pub fn vmla_lane_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29216,7 +29216,7 @@ pub fn vmla_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t)
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29245,7 +29245,7 @@ pub fn vmla_laneq_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x8
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29287,7 +29287,7 @@ pub fn vmlaq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t)
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29329,7 +29329,7 @@ pub fn vmlaq_lane_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x4
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29371,7 +29371,7 @@ pub fn vmlaq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))]
@@ -29413,7 +29413,7 @@ pub fn vmlaq_laneq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29436,7 +29436,7 @@ pub fn vmla_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t)
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29459,7 +29459,7 @@ pub fn vmla_lane_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29482,7 +29482,7 @@ pub fn vmla_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t)
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29505,7 +29505,7 @@ pub fn vmla_laneq_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x4
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29534,7 +29534,7 @@ pub fn vmlaq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t)
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29563,7 +29563,7 @@ pub fn vmlaq_lane_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x2
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29592,7 +29592,7 @@ pub fn vmlaq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))]
@@ -29621,7 +29621,7 @@ pub fn vmlaq_laneq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))]
@@ -29642,7 +29642,7 @@ pub fn vmla_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))]
@@ -29663,7 +29663,7 @@ pub fn vmlaq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -29684,7 +29684,7 @@ pub fn vmla_n_s16(a: int16x4_t, b: int16x4_t, c: i16) -> int16x4_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -29705,7 +29705,7 @@ pub fn vmlaq_n_s16(a: int16x8_t, b: int16x8_t, c: i16) -> int16x8_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -29726,7 +29726,7 @@ pub fn vmla_n_u16(a: uint16x4_t, b: uint16x4_t, c: u16) -> uint16x4_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -29747,7 +29747,7 @@ pub fn vmlaq_n_u16(a: uint16x8_t, b: uint16x8_t, c: u16) -> uint16x8_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -29768,7 +29768,7 @@ pub fn vmla_n_s32(a: int32x2_t, b: int32x2_t, c: i32) -> int32x2_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -29789,7 +29789,7 @@ pub fn vmlaq_n_s32(a: int32x4_t, b: int32x4_t, c: i32) -> int32x4_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -29810,7 +29810,7 @@ pub fn vmla_n_u32(a: uint32x2_t, b: uint32x2_t, c: u32) -> uint32x2_t {
 }
 #[doc = "Vector multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -29831,7 +29831,7 @@ pub fn vmlaq_n_u32(a: uint32x4_t, b: uint32x4_t, c: u32) -> uint32x4_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))]
@@ -29852,7 +29852,7 @@ pub fn vmla_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))]
@@ -29873,7 +29873,7 @@ pub fn vmlaq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -29894,7 +29894,7 @@ pub fn vmla_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -29915,7 +29915,7 @@ pub fn vmlaq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -29936,7 +29936,7 @@ pub fn vmla_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -29957,7 +29957,7 @@ pub fn vmlaq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))]
@@ -29978,7 +29978,7 @@ pub fn vmla_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))]
@@ -29999,7 +29999,7 @@ pub fn vmlaq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -30020,7 +30020,7 @@ pub fn vmla_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))]
@@ -30041,7 +30041,7 @@ pub fn vmlaq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmla_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -30062,7 +30062,7 @@ pub fn vmla_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Multiply-add to accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlaq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))]
@@ -30083,7 +30083,7 @@ pub fn vmlaq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16", LANE = 1))]
@@ -30112,7 +30112,7 @@ pub fn vmlal_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t)
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16", LANE = 1))]
@@ -30141,7 +30141,7 @@ pub fn vmlal_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32", LANE = 1))]
@@ -30164,7 +30164,7 @@ pub fn vmlal_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t)
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32", LANE = 1))]
@@ -30187,7 +30187,7 @@ pub fn vmlal_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16", LANE = 1))]
@@ -30216,7 +30216,7 @@ pub fn vmlal_lane_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16", LANE = 1))]
@@ -30245,7 +30245,7 @@ pub fn vmlal_laneq_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32", LANE = 1))]
@@ -30268,7 +30268,7 @@ pub fn vmlal_lane_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32", LANE = 1))]
@@ -30291,7 +30291,7 @@ pub fn vmlal_laneq_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16"))]
@@ -30312,7 +30312,7 @@ pub fn vmlal_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t {
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32"))]
@@ -30333,7 +30333,7 @@ pub fn vmlal_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t {
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16"))]
@@ -30354,7 +30354,7 @@ pub fn vmlal_n_u16(a: uint32x4_t, b: uint16x4_t, c: u16) -> uint32x4_t {
 }
 #[doc = "Vector widening multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32"))]
@@ -30375,7 +30375,7 @@ pub fn vmlal_n_u32(a: uint64x2_t, b: uint32x2_t, c: u32) -> uint64x2_t {
 }
 #[doc = "Signed multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s8"))]
@@ -30396,7 +30396,7 @@ pub fn vmlal_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16"))]
@@ -30417,7 +30417,7 @@ pub fn vmlal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32"))]
@@ -30438,7 +30438,7 @@ pub fn vmlal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u8"))]
@@ -30459,7 +30459,7 @@ pub fn vmlal_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16"))]
@@ -30480,7 +30480,7 @@ pub fn vmlal_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlal_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32"))]
@@ -30501,7 +30501,7 @@ pub fn vmlal_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Floating-point multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))]
@@ -30522,7 +30522,7 @@ pub fn vmls_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))]
@@ -30543,7 +30543,7 @@ pub fn vmlsq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))]
@@ -30570,7 +30570,7 @@ pub fn vmls_lane_f32(
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))]
@@ -30597,7 +30597,7 @@ pub fn vmls_laneq_f32(
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))]
@@ -30630,7 +30630,7 @@ pub fn vmlsq_lane_f32(
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))]
@@ -30663,7 +30663,7 @@ pub fn vmlsq_laneq_f32(
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30692,7 +30692,7 @@ pub fn vmls_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t)
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30721,7 +30721,7 @@ pub fn vmls_lane_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30750,7 +30750,7 @@ pub fn vmls_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t)
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30779,7 +30779,7 @@ pub fn vmls_laneq_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x8
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30821,7 +30821,7 @@ pub fn vmlsq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t)
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30863,7 +30863,7 @@ pub fn vmlsq_lane_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x4
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30905,7 +30905,7 @@ pub fn vmlsq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))]
@@ -30947,7 +30947,7 @@ pub fn vmlsq_laneq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -30970,7 +30970,7 @@ pub fn vmls_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t)
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -30993,7 +30993,7 @@ pub fn vmls_lane_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -31016,7 +31016,7 @@ pub fn vmls_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t)
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -31039,7 +31039,7 @@ pub fn vmls_laneq_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x4
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -31068,7 +31068,7 @@ pub fn vmlsq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t)
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -31097,7 +31097,7 @@ pub fn vmlsq_lane_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x2
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -31126,7 +31126,7 @@ pub fn vmlsq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))]
@@ -31155,7 +31155,7 @@ pub fn vmlsq_laneq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))]
@@ -31176,7 +31176,7 @@ pub fn vmls_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))]
@@ -31197,7 +31197,7 @@ pub fn vmlsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31218,7 +31218,7 @@ pub fn vmls_n_s16(a: int16x4_t, b: int16x4_t, c: i16) -> int16x4_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31239,7 +31239,7 @@ pub fn vmlsq_n_s16(a: int16x8_t, b: int16x8_t, c: i16) -> int16x8_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31260,7 +31260,7 @@ pub fn vmls_n_u16(a: uint16x4_t, b: uint16x4_t, c: u16) -> uint16x4_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31281,7 +31281,7 @@ pub fn vmlsq_n_u16(a: uint16x8_t, b: uint16x8_t, c: u16) -> uint16x8_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31302,7 +31302,7 @@ pub fn vmls_n_s32(a: int32x2_t, b: int32x2_t, c: i32) -> int32x2_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31323,7 +31323,7 @@ pub fn vmlsq_n_s32(a: int32x4_t, b: int32x4_t, c: i32) -> int32x4_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31344,7 +31344,7 @@ pub fn vmls_n_u32(a: uint32x2_t, b: uint32x2_t, c: u32) -> uint32x2_t {
 }
 #[doc = "Vector multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31365,7 +31365,7 @@ pub fn vmlsq_n_u32(a: uint32x4_t, b: uint32x4_t, c: u32) -> uint32x4_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))]
@@ -31386,7 +31386,7 @@ pub fn vmls_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))]
@@ -31407,7 +31407,7 @@ pub fn vmlsq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31428,7 +31428,7 @@ pub fn vmls_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31449,7 +31449,7 @@ pub fn vmlsq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31470,7 +31470,7 @@ pub fn vmls_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31491,7 +31491,7 @@ pub fn vmlsq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))]
@@ -31512,7 +31512,7 @@ pub fn vmls_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))]
@@ -31533,7 +31533,7 @@ pub fn vmlsq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31554,7 +31554,7 @@ pub fn vmls_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))]
@@ -31575,7 +31575,7 @@ pub fn vmlsq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmls_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31596,7 +31596,7 @@ pub fn vmls_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Multiply-subtract from accumulator"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))]
@@ -31617,7 +31617,7 @@ pub fn vmlsq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16", LANE = 1))]
@@ -31646,7 +31646,7 @@ pub fn vmlsl_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t)
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16", LANE = 1))]
@@ -31675,7 +31675,7 @@ pub fn vmlsl_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32", LANE = 1))]
@@ -31698,7 +31698,7 @@ pub fn vmlsl_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t)
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32", LANE = 1))]
@@ -31721,7 +31721,7 @@ pub fn vmlsl_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16", LANE = 1))]
@@ -31750,7 +31750,7 @@ pub fn vmlsl_lane_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16", LANE = 1))]
@@ -31779,7 +31779,7 @@ pub fn vmlsl_laneq_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32", LANE = 1))]
@@ -31802,7 +31802,7 @@ pub fn vmlsl_lane_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32", LANE = 1))]
@@ -31825,7 +31825,7 @@ pub fn vmlsl_laneq_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16"))]
@@ -31846,7 +31846,7 @@ pub fn vmlsl_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t {
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32"))]
@@ -31867,7 +31867,7 @@ pub fn vmlsl_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t {
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16"))]
@@ -31888,7 +31888,7 @@ pub fn vmlsl_n_u16(a: uint32x4_t, b: uint16x4_t, c: u16) -> uint32x4_t {
 }
 #[doc = "Vector widening multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32"))]
@@ -31909,7 +31909,7 @@ pub fn vmlsl_n_u32(a: uint64x2_t, b: uint32x2_t, c: u32) -> uint64x2_t {
 }
 #[doc = "Signed multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s8"))]
@@ -31930,7 +31930,7 @@ pub fn vmlsl_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16"))]
@@ -31951,7 +31951,7 @@ pub fn vmlsl_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32"))]
@@ -31972,7 +31972,7 @@ pub fn vmlsl_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u8"))]
@@ -31993,7 +31993,7 @@ pub fn vmlsl_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16"))]
@@ -32014,7 +32014,7 @@ pub fn vmlsl_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmlsl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32"))]
@@ -32035,7 +32035,7 @@ pub fn vmlsl_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "8-bit integer matrix multiply-accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmmlaq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -32064,7 +32064,7 @@ pub fn vmmlaq_s32(a: int32x4_t, b: int8x16_t, c: int8x16_t) -> int32x4_t {
 }
 #[doc = "8-bit integer matrix multiply-accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmmlaq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -32093,7 +32093,7 @@ pub fn vmmlaq_u32(a: uint32x4_t, b: uint8x16_t, c: uint8x16_t) -> uint32x4_t {
 }
 #[doc = "Duplicate element to vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32109,7 +32109,7 @@ pub fn vmov_n_f16(a: f16) -> float16x4_t {
 }
 #[doc = "Duplicate element to vector"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_f16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32125,7 +32125,7 @@ pub fn vmovq_n_f16(a: f16) -> float16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -32146,7 +32146,7 @@ pub fn vmov_n_f32(value: f32) -> float32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32167,7 +32167,7 @@ pub fn vmov_n_p16(value: p16) -> poly16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -32188,7 +32188,7 @@ pub fn vmov_n_p8(value: p8) -> poly8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32209,7 +32209,7 @@ pub fn vmov_n_s16(value: i16) -> int16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -32230,7 +32230,7 @@ pub fn vmov_n_s32(value: i32) -> int32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -32251,7 +32251,7 @@ pub fn vmov_n_s64(value: i64) -> int64x1_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -32272,7 +32272,7 @@ pub fn vmov_n_s8(value: i8) -> int8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32293,7 +32293,7 @@ pub fn vmov_n_u16(value: u16) -> uint16x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -32314,7 +32314,7 @@ pub fn vmov_n_u32(value: u32) -> uint32x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -32335,7 +32335,7 @@ pub fn vmov_n_u64(value: u64) -> uint64x1_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmov_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -32356,7 +32356,7 @@ pub fn vmov_n_u8(value: u8) -> uint8x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -32377,7 +32377,7 @@ pub fn vmovq_n_f32(value: f32) -> float32x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32398,7 +32398,7 @@ pub fn vmovq_n_p16(value: p16) -> poly16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -32419,7 +32419,7 @@ pub fn vmovq_n_p8(value: p8) -> poly8x16_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32440,7 +32440,7 @@ pub fn vmovq_n_s16(value: i16) -> int16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -32461,7 +32461,7 @@ pub fn vmovq_n_s32(value: i32) -> int32x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -32482,7 +32482,7 @@ pub fn vmovq_n_s64(value: i64) -> int64x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -32503,7 +32503,7 @@ pub fn vmovq_n_s8(value: i8) -> int8x16_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))]
@@ -32524,7 +32524,7 @@ pub fn vmovq_n_u16(value: u16) -> uint16x8_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))]
@@ -32545,7 +32545,7 @@ pub fn vmovq_n_u32(value: u32) -> uint32x4_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))]
@@ -32566,7 +32566,7 @@ pub fn vmovq_n_u64(value: u64) -> uint64x2_t {
 }
 #[doc = "Duplicate vector element to vector or scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))]
@@ -32587,7 +32587,7 @@ pub fn vmovq_n_u8(value: u8) -> uint8x16_t {
 }
 #[doc = "Vector long move."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))]
@@ -32608,7 +32608,7 @@ pub fn vmovl_s16(a: int16x4_t) -> int32x4_t {
 }
 #[doc = "Vector long move."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))]
@@ -32629,7 +32629,7 @@ pub fn vmovl_s32(a: int32x2_t) -> int64x2_t {
 }
 #[doc = "Vector long move."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))]
@@ -32650,7 +32650,7 @@ pub fn vmovl_s8(a: int8x8_t) -> int16x8_t {
 }
 #[doc = "Vector long move."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))]
@@ -32671,7 +32671,7 @@ pub fn vmovl_u16(a: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Vector long move."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))]
@@ -32692,7 +32692,7 @@ pub fn vmovl_u32(a: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Vector long move."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovl))]
@@ -32713,7 +32713,7 @@ pub fn vmovl_u8(a: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Vector narrow integer."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))]
@@ -32734,7 +32734,7 @@ pub fn vmovn_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Vector narrow integer."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))]
@@ -32755,7 +32755,7 @@ pub fn vmovn_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Vector narrow integer."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))]
@@ -32776,7 +32776,7 @@ pub fn vmovn_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Vector narrow integer."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))]
@@ -32797,7 +32797,7 @@ pub fn vmovn_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Vector narrow integer."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))]
@@ -32818,7 +32818,7 @@ pub fn vmovn_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Vector narrow integer."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmovn_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))]
@@ -32839,7 +32839,7 @@ pub fn vmovn_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f16"))]
 #[cfg_attr(
@@ -32861,7 +32861,7 @@ pub fn vmul_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f16"))]
 #[cfg_attr(
@@ -32883,7 +32883,7 @@ pub fn vmulq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))]
@@ -32904,7 +32904,7 @@ pub fn vmul_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))]
@@ -32925,7 +32925,7 @@ pub fn vmulq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
 #[cfg_attr(
@@ -32954,7 +32954,7 @@ pub fn vmul_lane_f16(a: float16x4_t, v: float16x4_t) -> float16
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
 #[cfg_attr(
@@ -32996,7 +32996,7 @@ pub fn vmulq_lane_f16(a: float16x8_t, v: float16x4_t) -> float1
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))]
@@ -33019,7 +33019,7 @@ pub fn vmul_lane_f32(a: float32x2_t, b: float32x2_t) -> float32
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))]
@@ -33042,7 +33042,7 @@ pub fn vmul_laneq_f32(a: float32x2_t, b: float32x4_t) -> float3
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))]
@@ -33070,7 +33070,7 @@ pub fn vmulq_lane_f32(a: float32x4_t, b: float32x2_t) -> float3
 }
 #[doc = "Floating-point multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))]
@@ -33098,7 +33098,7 @@ pub fn vmulq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33126,7 +33126,7 @@ pub fn vmul_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33167,7 +33167,7 @@ pub fn vmulq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33190,7 +33190,7 @@ pub fn vmul_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33218,7 +33218,7 @@ pub fn vmulq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33246,7 +33246,7 @@ pub fn vmul_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33287,7 +33287,7 @@ pub fn vmulq_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint16x8
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33310,7 +33310,7 @@ pub fn vmul_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33338,7 +33338,7 @@ pub fn vmulq_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint32x4
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33366,7 +33366,7 @@ pub fn vmul_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33407,7 +33407,7 @@ pub fn vmulq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33430,7 +33430,7 @@ pub fn vmul_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33458,7 +33458,7 @@ pub fn vmulq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33486,7 +33486,7 @@ pub fn vmul_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint16x4
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33527,7 +33527,7 @@ pub fn vmulq_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33550,7 +33550,7 @@ pub fn vmul_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint32x2
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))]
@@ -33578,7 +33578,7 @@ pub fn vmulq_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
 #[cfg_attr(
@@ -33593,7 +33593,7 @@ pub fn vmul_n_f16(a: float16x4_t, b: f16) -> float16x4_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
 #[cfg_attr(
@@ -33608,7 +33608,7 @@ pub fn vmulq_n_f16(a: float16x8_t, b: f16) -> float16x8_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33629,7 +33629,7 @@ pub fn vmul_n_f32(a: float32x2_t, b: f32) -> float32x2_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33650,7 +33650,7 @@ pub fn vmulq_n_f32(a: float32x4_t, b: f32) -> float32x4_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33671,7 +33671,7 @@ pub fn vmul_n_s16(a: int16x4_t, b: i16) -> int16x4_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33692,7 +33692,7 @@ pub fn vmulq_n_s16(a: int16x8_t, b: i16) -> int16x8_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33713,7 +33713,7 @@ pub fn vmul_n_s32(a: int32x2_t, b: i32) -> int32x2_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33734,7 +33734,7 @@ pub fn vmulq_n_s32(a: int32x4_t, b: i32) -> int32x4_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33755,7 +33755,7 @@ pub fn vmul_n_u16(a: uint16x4_t, b: u16) -> uint16x4_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33776,7 +33776,7 @@ pub fn vmulq_n_u16(a: uint16x8_t, b: u16) -> uint16x8_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33797,7 +33797,7 @@ pub fn vmul_n_u32(a: uint32x2_t, b: u32) -> uint32x2_t {
 }
 #[doc = "Vector multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33818,7 +33818,7 @@ pub fn vmulq_n_u32(a: uint32x4_t, b: u32) -> uint32x4_t {
 }
 #[doc = "Polynomial multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33847,7 +33847,7 @@ pub fn vmul_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Polynomial multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))]
@@ -33876,7 +33876,7 @@ pub fn vmulq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))]
@@ -33897,7 +33897,7 @@ pub fn vmul_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))]
@@ -33918,7 +33918,7 @@ pub fn vmulq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))]
@@ -33939,7 +33939,7 @@ pub fn vmul_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))]
@@ -33960,7 +33960,7 @@ pub fn vmulq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))]
@@ -33981,7 +33981,7 @@ pub fn vmul_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))]
@@ -34002,7 +34002,7 @@ pub fn vmulq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))]
@@ -34023,7 +34023,7 @@ pub fn vmul_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))]
@@ -34044,7 +34044,7 @@ pub fn vmulq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))]
@@ -34065,7 +34065,7 @@ pub fn vmul_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))]
@@ -34086,7 +34086,7 @@ pub fn vmulq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmul_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))]
@@ -34107,7 +34107,7 @@ pub fn vmul_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Multiply"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmulq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))]
@@ -34128,7 +34128,7 @@ pub fn vmulq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34156,7 +34156,7 @@ pub fn vmull_lane_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34184,7 +34184,7 @@ pub fn vmull_laneq_s16(a: int16x4_t, b: int16x8_t) -> int32x4_t
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34207,7 +34207,7 @@ pub fn vmull_lane_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34230,7 +34230,7 @@ pub fn vmull_laneq_s32(a: int32x2_t, b: int32x4_t) -> int64x2_t
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34258,7 +34258,7 @@ pub fn vmull_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34286,7 +34286,7 @@ pub fn vmull_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint32x
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34309,7 +34309,7 @@ pub fn vmull_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2
 }
 #[doc = "Vector long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_laneq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))]
@@ -34332,7 +34332,7 @@ pub fn vmull_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint64x
 }
 #[doc = "Vector long multiply with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))]
@@ -34353,7 +34353,7 @@ pub fn vmull_n_s16(a: int16x4_t, b: i16) -> int32x4_t {
 }
 #[doc = "Vector long multiply with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))]
@@ -34374,7 +34374,7 @@ pub fn vmull_n_s32(a: int32x2_t, b: i32) -> int64x2_t {
 }
 #[doc = "Vector long multiply with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))]
@@ -34395,7 +34395,7 @@ pub fn vmull_n_u16(a: uint16x4_t, b: u16) -> uint32x4_t {
 }
 #[doc = "Vector long multiply with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))]
@@ -34416,7 +34416,7 @@ pub fn vmull_n_u32(a: uint32x2_t, b: u32) -> uint64x2_t {
 }
 #[doc = "Polynomial multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.p8"))]
@@ -34445,7 +34445,7 @@ pub fn vmull_p8(a: poly8x8_t, b: poly8x8_t) -> poly16x8_t {
 }
 #[doc = "Signed multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s16"))]
@@ -34474,7 +34474,7 @@ pub fn vmull_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s32"))]
@@ -34503,7 +34503,7 @@ pub fn vmull_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Signed multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s8"))]
@@ -34532,7 +34532,7 @@ pub fn vmull_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t {
 }
 #[doc = "Unsigned multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u8"))]
@@ -34561,7 +34561,7 @@ pub fn vmull_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u16"))]
@@ -34590,7 +34590,7 @@ pub fn vmull_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmull_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u32"))]
@@ -34619,7 +34619,7 @@ pub fn vmull_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34641,7 +34641,7 @@ pub fn vmvn_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34663,7 +34663,7 @@ pub fn vmvn_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34685,7 +34685,7 @@ pub fn vmvn_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34707,7 +34707,7 @@ pub fn vmvn_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34729,7 +34729,7 @@ pub fn vmvn_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34751,7 +34751,7 @@ pub fn vmvn_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvn_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34773,7 +34773,7 @@ pub fn vmvn_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34795,7 +34795,7 @@ pub fn vmvnq_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34817,7 +34817,7 @@ pub fn vmvnq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34839,7 +34839,7 @@ pub fn vmvnq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34861,7 +34861,7 @@ pub fn vmvnq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34883,7 +34883,7 @@ pub fn vmvnq_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34905,7 +34905,7 @@ pub fn vmvnq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector bitwise not."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vmvnq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmvn))]
@@ -34927,7 +34927,7 @@ pub fn vmvnq_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f16"))]
 #[cfg_attr(
@@ -34949,7 +34949,7 @@ pub fn vneg_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f16"))]
 #[cfg_attr(
@@ -34971,7 +34971,7 @@ pub fn vnegq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f32"))]
@@ -34992,7 +34992,7 @@ pub fn vneg_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f32"))]
@@ -35013,7 +35013,7 @@ pub fn vnegq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s8"))]
@@ -35034,7 +35034,7 @@ pub fn vneg_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s8"))]
@@ -35055,7 +35055,7 @@ pub fn vnegq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s16"))]
@@ -35076,7 +35076,7 @@ pub fn vneg_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s16"))]
@@ -35097,7 +35097,7 @@ pub fn vnegq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vneg_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s32"))]
@@ -35118,7 +35118,7 @@ pub fn vneg_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vnegq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s32"))]
@@ -35139,7 +35139,7 @@ pub fn vnegq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35161,7 +35161,7 @@ pub fn vorn_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35183,7 +35183,7 @@ pub fn vorn_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35205,7 +35205,7 @@ pub fn vorn_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35227,7 +35227,7 @@ pub fn vorn_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35249,7 +35249,7 @@ pub fn vornq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35271,7 +35271,7 @@ pub fn vornq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35293,7 +35293,7 @@ pub fn vornq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35315,7 +35315,7 @@ pub fn vornq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35337,7 +35337,7 @@ pub fn vorn_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35359,7 +35359,7 @@ pub fn vorn_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35381,7 +35381,7 @@ pub fn vorn_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorn_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35403,7 +35403,7 @@ pub fn vorn_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35425,7 +35425,7 @@ pub fn vornq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35447,7 +35447,7 @@ pub fn vornq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35469,7 +35469,7 @@ pub fn vornq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector bitwise inclusive OR NOT"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vornq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))]
@@ -35491,7 +35491,7 @@ pub fn vornq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35512,7 +35512,7 @@ pub fn vorr_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35533,7 +35533,7 @@ pub fn vorrq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35554,7 +35554,7 @@ pub fn vorr_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35575,7 +35575,7 @@ pub fn vorrq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35596,7 +35596,7 @@ pub fn vorr_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35617,7 +35617,7 @@ pub fn vorrq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35638,7 +35638,7 @@ pub fn vorr_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35659,7 +35659,7 @@ pub fn vorrq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35680,7 +35680,7 @@ pub fn vorr_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35701,7 +35701,7 @@ pub fn vorrq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35722,7 +35722,7 @@ pub fn vorr_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35743,7 +35743,7 @@ pub fn vorrq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35764,7 +35764,7 @@ pub fn vorr_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35785,7 +35785,7 @@ pub fn vorrq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorr_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35806,7 +35806,7 @@ pub fn vorr_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector bitwise or (immediate, inclusive)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vorrq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -35827,7 +35827,7 @@ pub fn vorrq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadal_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.s8"))]
@@ -35857,7 +35857,7 @@ pub fn vpadal_s8(a: int16x4_t, b: int8x8_t) -> int16x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadalq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.s8"))]
@@ -35887,7 +35887,7 @@ pub fn vpadalq_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadal_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.s16"))]
@@ -35917,7 +35917,7 @@ pub fn vpadal_s16(a: int32x2_t, b: int16x4_t) -> int32x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadalq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.s16"))]
@@ -35947,7 +35947,7 @@ pub fn vpadalq_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadal_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.s32"))]
@@ -35977,7 +35977,7 @@ pub fn vpadal_s32(a: int64x1_t, b: int32x2_t) -> int64x1_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadalq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.s32"))]
@@ -36007,7 +36007,7 @@ pub fn vpadalq_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadal_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.u8"))]
@@ -36037,7 +36037,7 @@ pub fn vpadal_u8(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadalq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.u8"))]
@@ -36067,7 +36067,7 @@ pub fn vpadalq_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadal_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.u16"))]
@@ -36097,7 +36097,7 @@ pub fn vpadal_u16(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadalq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.u16"))]
@@ -36127,7 +36127,7 @@ pub fn vpadalq_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadal_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.u32"))]
@@ -36157,7 +36157,7 @@ pub fn vpadal_u32(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadalq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpadal.u32"))]
@@ -36187,7 +36187,7 @@ pub fn vpadalq_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))]
 #[cfg_attr(
@@ -36217,7 +36217,7 @@ pub fn vpadd_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point add pairwise"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))]
@@ -36246,7 +36246,7 @@ pub fn vpadd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))]
@@ -36275,7 +36275,7 @@ pub fn vpadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))]
@@ -36304,7 +36304,7 @@ pub fn vpadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadd))]
@@ -36333,7 +36333,7 @@ pub fn vpadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -36355,7 +36355,7 @@ pub fn vpadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -36382,7 +36382,7 @@ pub fn vpadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -36404,7 +36404,7 @@ pub fn vpadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -36431,7 +36431,7 @@ pub fn vpadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -36453,7 +36453,7 @@ pub fn vpadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Add pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpadd_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -36480,7 +36480,7 @@ pub fn vpadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.s8"))]
@@ -36509,7 +36509,7 @@ pub fn vpaddl_s8(a: int8x8_t) -> int16x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddlq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.s8"))]
@@ -36538,7 +36538,7 @@ pub fn vpaddlq_s8(a: int8x16_t) -> int16x8_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.s16"))]
@@ -36567,7 +36567,7 @@ pub fn vpaddl_s16(a: int16x4_t) -> int32x2_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddlq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.s16"))]
@@ -36596,7 +36596,7 @@ pub fn vpaddlq_s16(a: int16x8_t) -> int32x4_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.s32"))]
@@ -36625,7 +36625,7 @@ pub fn vpaddl_s32(a: int32x2_t) -> int64x1_t {
 }
 #[doc = "Signed Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddlq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.s32"))]
@@ -36654,7 +36654,7 @@ pub fn vpaddlq_s32(a: int32x4_t) -> int64x2_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.u8"))]
@@ -36683,7 +36683,7 @@ pub fn vpaddl_u8(a: uint8x8_t) -> uint16x4_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddlq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.u8"))]
@@ -36712,7 +36712,7 @@ pub fn vpaddlq_u8(a: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.u16"))]
@@ -36741,7 +36741,7 @@ pub fn vpaddl_u16(a: uint16x4_t) -> uint32x2_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddlq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.u16"))]
@@ -36770,7 +36770,7 @@ pub fn vpaddlq_u16(a: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.u32"))]
@@ -36799,7 +36799,7 @@ pub fn vpaddl_u32(a: uint32x2_t) -> uint64x1_t {
 }
 #[doc = "Unsigned Add and Accumulate Long Pairwise."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpaddlq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vpaddl.u32"))]
@@ -36828,7 +36828,7 @@ pub fn vpaddlq_u32(a: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -36857,7 +36857,7 @@ pub fn vpmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -36886,7 +36886,7 @@ pub fn vpmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -36915,7 +36915,7 @@ pub fn vpmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -36944,7 +36944,7 @@ pub fn vpmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -36973,7 +36973,7 @@ pub fn vpmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -37002,7 +37002,7 @@ pub fn vpmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Folding maximum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmax_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))]
@@ -37031,7 +37031,7 @@ pub fn vpmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37060,7 +37060,7 @@ pub fn vpmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37089,7 +37089,7 @@ pub fn vpmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37118,7 +37118,7 @@ pub fn vpmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37147,7 +37147,7 @@ pub fn vpmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37176,7 +37176,7 @@ pub fn vpmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37205,7 +37205,7 @@ pub fn vpmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Folding minimum of adjacent pairs"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vpmin_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))]
@@ -37234,7 +37234,7 @@ pub fn vpmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s8"))]
@@ -37263,7 +37263,7 @@ pub fn vqabs_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s8"))]
@@ -37292,7 +37292,7 @@ pub fn vqabsq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s16"))]
@@ -37321,7 +37321,7 @@ pub fn vqabs_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s16"))]
@@ -37350,7 +37350,7 @@ pub fn vqabsq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabs_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s32"))]
@@ -37379,7 +37379,7 @@ pub fn vqabs_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating Absolute value"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqabsq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s32"))]
@@ -37408,7 +37408,7 @@ pub fn vqabsq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))]
@@ -37429,7 +37429,7 @@ pub fn vqadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))]
@@ -37450,7 +37450,7 @@ pub fn vqaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))]
@@ -37471,7 +37471,7 @@ pub fn vqadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))]
@@ -37492,7 +37492,7 @@ pub fn vqaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))]
@@ -37513,7 +37513,7 @@ pub fn vqadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))]
@@ -37534,7 +37534,7 @@ pub fn vqaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s64"))]
@@ -37555,7 +37555,7 @@ pub fn vqadd_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s64"))]
@@ -37576,7 +37576,7 @@ pub fn vqaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))]
@@ -37597,7 +37597,7 @@ pub fn vqadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))]
@@ -37618,7 +37618,7 @@ pub fn vqaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))]
@@ -37639,7 +37639,7 @@ pub fn vqadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))]
@@ -37660,7 +37660,7 @@ pub fn vqaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))]
@@ -37681,7 +37681,7 @@ pub fn vqadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))]
@@ -37702,7 +37702,7 @@ pub fn vqaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqadd_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u64"))]
@@ -37723,7 +37723,7 @@ pub fn vqadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Saturating add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqaddq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u64"))]
@@ -37744,7 +37744,7 @@ pub fn vqaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector widening saturating doubling multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal, N = 2))]
@@ -37767,7 +37767,7 @@ pub fn vqdmlal_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t)
 }
 #[doc = "Vector widening saturating doubling multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal, N = 1))]
@@ -37790,7 +37790,7 @@ pub fn vqdmlal_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t)
 }
 #[doc = "Vector widening saturating doubling multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))]
@@ -37811,7 +37811,7 @@ pub fn vqdmlal_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t {
 }
 #[doc = "Vector widening saturating doubling multiply accumulate with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))]
@@ -37832,7 +37832,7 @@ pub fn vqdmlal_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))]
@@ -37853,7 +37853,7 @@ pub fn vqdmlal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply-add long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlal_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))]
@@ -37874,7 +37874,7 @@ pub fn vqdmlal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t {
 }
 #[doc = "Vector widening saturating doubling multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl, N = 2))]
@@ -37897,7 +37897,7 @@ pub fn vqdmlsl_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t)
 }
 #[doc = "Vector widening saturating doubling multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl, N = 1))]
@@ -37920,7 +37920,7 @@ pub fn vqdmlsl_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t)
 }
 #[doc = "Vector widening saturating doubling multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))]
@@ -37941,7 +37941,7 @@ pub fn vqdmlsl_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t {
 }
 #[doc = "Vector widening saturating doubling multiply subtract with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))]
@@ -37962,7 +37962,7 @@ pub fn vqdmlsl_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))]
@@ -37983,7 +37983,7 @@ pub fn vqdmlsl_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply-subtract long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmlsl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))]
@@ -38004,7 +38004,7 @@ pub fn vqdmlsl_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t {
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))]
@@ -38027,7 +38027,7 @@ pub fn vqdmulh_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))]
@@ -38050,7 +38050,7 @@ pub fn vqdmulhq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))]
@@ -38073,7 +38073,7 @@ pub fn vqdmulh_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2
 }
 #[doc = "Vector saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh, LANE = 0))]
@@ -38096,7 +38096,7 @@ pub fn vqdmulhq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x
 }
 #[doc = "Vector saturating doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38118,7 +38118,7 @@ pub fn vqdmulh_n_s16(a: int16x4_t, b: i16) -> int16x4_t {
 }
 #[doc = "Vector saturating doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38140,7 +38140,7 @@ pub fn vqdmulhq_n_s16(a: int16x8_t, b: i16) -> int16x8_t {
 }
 #[doc = "Vector saturating doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38162,7 +38162,7 @@ pub fn vqdmulh_n_s32(a: int32x2_t, b: i32) -> int32x2_t {
 }
 #[doc = "Vector saturating doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38184,7 +38184,7 @@ pub fn vqdmulhq_n_s32(a: int32x4_t, b: i32) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38213,7 +38213,7 @@ pub fn vqdmulh_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38242,7 +38242,7 @@ pub fn vqdmulhq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulh_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38271,7 +38271,7 @@ pub fn vqdmulh_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmulhq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))]
@@ -38300,7 +38300,7 @@ pub fn vqdmulhq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector saturating doubling long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull, N = 2))]
@@ -38326,7 +38326,7 @@ pub fn vqdmull_lane_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Vector saturating doubling long multiply by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull, N = 1))]
@@ -38352,7 +38352,7 @@ pub fn vqdmull_lane_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Vector saturating doubling long multiply with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))]
@@ -38373,7 +38373,7 @@ pub fn vqdmull_n_s16(a: int16x4_t, b: i16) -> int32x4_t {
 }
 #[doc = "Vector saturating doubling long multiply with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))]
@@ -38394,7 +38394,7 @@ pub fn vqdmull_n_s32(a: int32x2_t, b: i32) -> int64x2_t {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))]
@@ -38423,7 +38423,7 @@ pub fn vqdmull_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating doubling multiply long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqdmull_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))]
@@ -38452,7 +38452,7 @@ pub fn vqdmull_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))]
@@ -38481,7 +38481,7 @@ pub fn vqmovn_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))]
@@ -38510,7 +38510,7 @@ pub fn vqmovn_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))]
@@ -38539,7 +38539,7 @@ pub fn vqmovn_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Unsigned saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))]
@@ -38568,7 +38568,7 @@ pub fn vqmovn_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))]
@@ -38597,7 +38597,7 @@ pub fn vqmovn_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned saturating extract narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovn_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))]
@@ -38626,7 +38626,7 @@ pub fn vqmovn_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))]
@@ -38655,7 +38655,7 @@ pub fn vqmovun_s16(a: int16x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))]
@@ -38684,7 +38684,7 @@ pub fn vqmovun_s32(a: int32x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating extract unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqmovun_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))]
@@ -38713,7 +38713,7 @@ pub fn vqmovun_s64(a: int64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s8"))]
@@ -38742,7 +38742,7 @@ pub fn vqneg_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s8"))]
@@ -38771,7 +38771,7 @@ pub fn vqnegq_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s16"))]
@@ -38800,7 +38800,7 @@ pub fn vqneg_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s16"))]
@@ -38829,7 +38829,7 @@ pub fn vqnegq_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqneg_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s32"))]
@@ -38858,7 +38858,7 @@ pub fn vqneg_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating negate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqnegq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s32"))]
@@ -38887,7 +38887,7 @@ pub fn vqnegq_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -38914,7 +38914,7 @@ pub fn vqrdmulh_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -38940,7 +38940,7 @@ pub fn vqrdmulh_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -38967,7 +38967,7 @@ pub fn vqrdmulh_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -38993,7 +38993,7 @@ pub fn vqrdmulh_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -39032,7 +39032,7 @@ pub fn vqrdmulhq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -39059,7 +39059,7 @@ pub fn vqrdmulhq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_laneq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -39098,7 +39098,7 @@ pub fn vqrdmulhq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16
 }
 #[doc = "Vector rounding saturating doubling multiply high by scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_laneq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))]
@@ -39125,7 +39125,7 @@ pub fn vqrdmulhq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32
 }
 #[doc = "Vector saturating rounding doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39146,7 +39146,7 @@ pub fn vqrdmulh_n_s16(a: int16x4_t, b: i16) -> int16x4_t {
 }
 #[doc = "Vector saturating rounding doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39167,7 +39167,7 @@ pub fn vqrdmulhq_n_s16(a: int16x8_t, b: i16) -> int16x8_t {
 }
 #[doc = "Vector saturating rounding doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39188,7 +39188,7 @@ pub fn vqrdmulh_n_s32(a: int32x2_t, b: i32) -> int32x2_t {
 }
 #[doc = "Vector saturating rounding doubling multiply high with scalar"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39209,7 +39209,7 @@ pub fn vqrdmulhq_n_s32(a: int32x4_t, b: i32) -> int32x4_t {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39238,7 +39238,7 @@ pub fn vqrdmulh_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39267,7 +39267,7 @@ pub fn vqrdmulhq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulh_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39296,7 +39296,7 @@ pub fn vqrdmulh_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating rounding doubling multiply returning high half"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrdmulhq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))]
@@ -39325,7 +39325,7 @@ pub fn vqrdmulhq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39354,7 +39354,7 @@ pub fn vqrshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39383,7 +39383,7 @@ pub fn vqrshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39412,7 +39412,7 @@ pub fn vqrshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39441,7 +39441,7 @@ pub fn vqrshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39470,7 +39470,7 @@ pub fn vqrshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39499,7 +39499,7 @@ pub fn vqrshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39528,7 +39528,7 @@ pub fn vqrshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39557,7 +39557,7 @@ pub fn vqrshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39586,7 +39586,7 @@ pub fn vqrshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39615,7 +39615,7 @@ pub fn vqrshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39644,7 +39644,7 @@ pub fn vqrshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39673,7 +39673,7 @@ pub fn vqrshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39702,7 +39702,7 @@ pub fn vqrshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39731,7 +39731,7 @@ pub fn vqrshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshl_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39760,7 +39760,7 @@ pub fn vqrshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned signed saturating rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshlq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))]
@@ -39789,7 +39789,7 @@ pub fn vqrshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrn, N = 2))]
@@ -39805,7 +39805,7 @@ pub fn vqrshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrn, N = 2))]
@@ -39821,7 +39821,7 @@ pub fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrn, N = 2))]
@@ -39837,7 +39837,7 @@ pub fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqrshrn, N = 2))]
@@ -39856,7 +39856,7 @@ pub fn vqrshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqrshrn, N = 2))]
@@ -39875,7 +39875,7 @@ pub fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqrshrn, N = 2))]
@@ -39894,7 +39894,7 @@ pub fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Unsigned signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrn, N = 2))]
@@ -39920,7 +39920,7 @@ pub fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrn, N = 2))]
@@ -39941,7 +39941,7 @@ pub fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrn, N = 2))]
@@ -39957,7 +39957,7 @@ pub fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(uqrshrn, N = 2))]
@@ -39976,7 +39976,7 @@ pub fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(uqrshrn, N = 2))]
@@ -39995,7 +39995,7 @@ pub fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned signed saturating rounded shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrn_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(uqrshrn, N = 2))]
@@ -40014,7 +40014,7 @@ pub fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrun, N = 2))]
@@ -40030,7 +40030,7 @@ pub fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrun, N = 2))]
@@ -40046,7 +40046,7 @@ pub fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqrshrun, N = 2))]
@@ -40062,7 +40062,7 @@ pub fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqrshrun, N = 2))]
@@ -40081,7 +40081,7 @@ pub fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqrshrun, N = 2))]
@@ -40100,7 +40100,7 @@ pub fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating rounded shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqrshrun_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqrshrun, N = 2))]
@@ -40119,7 +40119,7 @@ pub fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40142,7 +40142,7 @@ pub fn vqshl_n_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40165,7 +40165,7 @@ pub fn vqshlq_n_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40188,7 +40188,7 @@ pub fn vqshl_n_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40211,7 +40211,7 @@ pub fn vqshlq_n_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40234,7 +40234,7 @@ pub fn vqshl_n_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40257,7 +40257,7 @@ pub fn vqshlq_n_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40280,7 +40280,7 @@ pub fn vqshl_n_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40303,7 +40303,7 @@ pub fn vqshlq_n_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40326,7 +40326,7 @@ pub fn vqshl_n_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40349,7 +40349,7 @@ pub fn vqshlq_n_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40372,7 +40372,7 @@ pub fn vqshl_n_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40395,7 +40395,7 @@ pub fn vqshlq_n_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40418,7 +40418,7 @@ pub fn vqshl_n_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40441,7 +40441,7 @@ pub fn vqshlq_n_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40464,7 +40464,7 @@ pub fn vqshl_n_u64(a: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))]
@@ -40487,7 +40487,7 @@ pub fn vqshlq_n_u64(a: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40516,7 +40516,7 @@ pub fn vqshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40545,7 +40545,7 @@ pub fn vqshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40574,7 +40574,7 @@ pub fn vqshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40603,7 +40603,7 @@ pub fn vqshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40632,7 +40632,7 @@ pub fn vqshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40661,7 +40661,7 @@ pub fn vqshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40690,7 +40690,7 @@ pub fn vqshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40719,7 +40719,7 @@ pub fn vqshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40748,7 +40748,7 @@ pub fn vqshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40777,7 +40777,7 @@ pub fn vqshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40806,7 +40806,7 @@ pub fn vqshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40835,7 +40835,7 @@ pub fn vqshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40864,7 +40864,7 @@ pub fn vqshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40893,7 +40893,7 @@ pub fn vqshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshl_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40922,7 +40922,7 @@ pub fn vqshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned saturating shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))]
@@ -40951,7 +40951,7 @@ pub fn vqshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -40967,7 +40967,7 @@ pub fn vqshlu_n_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -40983,7 +40983,7 @@ pub fn vqshluq_n_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -40999,7 +40999,7 @@ pub fn vqshlu_n_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -41015,7 +41015,7 @@ pub fn vqshluq_n_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -41031,7 +41031,7 @@ pub fn vqshlu_n_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -41047,7 +41047,7 @@ pub fn vqshluq_n_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -41063,7 +41063,7 @@ pub fn vqshlu_n_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshlu, N = 2))]
@@ -41079,7 +41079,7 @@ pub fn vqshluq_n_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41098,7 +41098,7 @@ pub fn vqshlu_n_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41117,7 +41117,7 @@ pub fn vqshluq_n_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41136,7 +41136,7 @@ pub fn vqshlu_n_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41155,7 +41155,7 @@ pub fn vqshluq_n_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41174,7 +41174,7 @@ pub fn vqshlu_n_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41193,7 +41193,7 @@ pub fn vqshluq_n_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshlu_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41212,7 +41212,7 @@ pub fn vqshlu_n_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Signed saturating shift left unsigned"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshluq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshlu, N = 2))]
@@ -41231,7 +41231,7 @@ pub fn vqshluq_n_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrn, N = 2))]
@@ -41247,7 +41247,7 @@ pub fn vqshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrn, N = 2))]
@@ -41263,7 +41263,7 @@ pub fn vqshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrn, N = 2))]
@@ -41279,7 +41279,7 @@ pub fn vqshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshrn, N = 2))]
@@ -41298,7 +41298,7 @@ pub fn vqshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshrn, N = 2))]
@@ -41317,7 +41317,7 @@ pub fn vqshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Signed saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshrn, N = 2))]
@@ -41336,7 +41336,7 @@ pub fn vqshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrn, N = 2))]
@@ -41362,7 +41362,7 @@ pub fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrn, N = 2))]
@@ -41383,7 +41383,7 @@ pub fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrn, N = 2))]
@@ -41399,7 +41399,7 @@ pub fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(uqshrn, N = 2))]
@@ -41418,7 +41418,7 @@ pub fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(uqshrn, N = 2))]
@@ -41437,7 +41437,7 @@ pub fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned saturating shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrn_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(uqshrn, N = 2))]
@@ -41456,7 +41456,7 @@ pub fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrun, N = 2))]
@@ -41472,7 +41472,7 @@ pub fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrun, N = 2))]
@@ -41488,7 +41488,7 @@ pub fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vqshrun, N = 2))]
@@ -41504,7 +41504,7 @@ pub fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshrun, N = 2))]
@@ -41523,7 +41523,7 @@ pub fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshrun, N = 2))]
@@ -41542,7 +41542,7 @@ pub fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t {
 }
 #[doc = "Signed saturating shift right unsigned narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqshrun_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(sqshrun, N = 2))]
@@ -41561,7 +41561,7 @@ pub fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))]
@@ -41582,7 +41582,7 @@ pub fn vqsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))]
@@ -41603,7 +41603,7 @@ pub fn vqsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))]
@@ -41624,7 +41624,7 @@ pub fn vqsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))]
@@ -41645,7 +41645,7 @@ pub fn vqsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))]
@@ -41666,7 +41666,7 @@ pub fn vqsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))]
@@ -41687,7 +41687,7 @@ pub fn vqsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s64"))]
@@ -41708,7 +41708,7 @@ pub fn vqsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s64"))]
@@ -41729,7 +41729,7 @@ pub fn vqsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))]
@@ -41750,7 +41750,7 @@ pub fn vqsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))]
@@ -41771,7 +41771,7 @@ pub fn vqsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))]
@@ -41792,7 +41792,7 @@ pub fn vqsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))]
@@ -41813,7 +41813,7 @@ pub fn vqsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))]
@@ -41834,7 +41834,7 @@ pub fn vqsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))]
@@ -41855,7 +41855,7 @@ pub fn vqsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsub_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u64"))]
@@ -41876,7 +41876,7 @@ pub fn vqsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Saturating subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vqsubq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u64"))]
@@ -41897,7 +41897,7 @@ pub fn vqsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Rounding Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i16"))]
@@ -41919,7 +41919,7 @@ pub fn vraddhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t {
 }
 #[doc = "Rounding Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i32"))]
@@ -41941,7 +41941,7 @@ pub fn vraddhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t {
 }
 #[doc = "Rounding Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i64"))]
@@ -41963,7 +41963,7 @@ pub fn vraddhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t {
 }
 #[doc = "Rounding Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i16"))]
@@ -41987,7 +41987,7 @@ pub fn vraddhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_
 }
 #[doc = "Rounding Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i32"))]
@@ -42011,7 +42011,7 @@ pub fn vraddhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8
 }
 #[doc = "Rounding Add returning High Narrow (high half)."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i64"))]
@@ -42035,7 +42035,7 @@ pub fn vraddhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i16"))]
@@ -42064,7 +42064,7 @@ pub fn vraddhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i32"))]
@@ -42093,7 +42093,7 @@ pub fn vraddhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vraddhn.i64"))]
@@ -42122,7 +42122,7 @@ pub fn vraddhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -42144,7 +42144,7 @@ pub fn vraddhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -42171,7 +42171,7 @@ pub fn vraddhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -42193,7 +42193,7 @@ pub fn vraddhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -42220,7 +42220,7 @@ pub fn vraddhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -42242,7 +42242,7 @@ pub fn vraddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Rounding Add returning High Narrow."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vraddhn_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -42269,7 +42269,7 @@ pub fn vraddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))]
 #[cfg_attr(
@@ -42299,7 +42299,7 @@ pub fn vrecpe_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))]
 #[cfg_attr(
@@ -42329,7 +42329,7 @@ pub fn vrecpeq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))]
@@ -42358,7 +42358,7 @@ pub fn vrecpe_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Reciprocal estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))]
@@ -42387,7 +42387,7 @@ pub fn vrecpeq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Unsigned reciprocal estimate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpe_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))]
@@ -42416,7 +42416,7 @@ pub fn vrecpe_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned reciprocal estimate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpeq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))]
@@ -42445,7 +42445,7 @@ pub fn vrecpeq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecps_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecps))]
 #[cfg_attr(
@@ -42475,7 +42475,7 @@ pub fn vrecps_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecps))]
 #[cfg_attr(
@@ -42505,7 +42505,7 @@ pub fn vrecpsq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecps_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecps))]
@@ -42534,7 +42534,7 @@ pub fn vrecps_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point reciprocal step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrecpsq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecps))]
@@ -42563,7 +42563,7 @@ pub fn vrecpsq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42586,7 +42586,7 @@ pub fn vreinterpret_f32_f16(a: float16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42613,7 +42613,7 @@ pub fn vreinterpret_f32_f16(a: float16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42636,7 +42636,7 @@ pub fn vreinterpret_s8_f16(a: float16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42663,7 +42663,7 @@ pub fn vreinterpret_s8_f16(a: float16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42686,7 +42686,7 @@ pub fn vreinterpret_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42713,7 +42713,7 @@ pub fn vreinterpret_s16_f16(a: float16x4_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42736,7 +42736,7 @@ pub fn vreinterpret_s32_f16(a: float16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42763,7 +42763,7 @@ pub fn vreinterpret_s32_f16(a: float16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42786,7 +42786,7 @@ pub fn vreinterpret_s64_f16(a: float16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42810,7 +42810,7 @@ pub fn vreinterpret_s64_f16(a: float16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42833,7 +42833,7 @@ pub fn vreinterpret_u8_f16(a: float16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42860,7 +42860,7 @@ pub fn vreinterpret_u8_f16(a: float16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42883,7 +42883,7 @@ pub fn vreinterpret_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42910,7 +42910,7 @@ pub fn vreinterpret_u16_f16(a: float16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42933,7 +42933,7 @@ pub fn vreinterpret_u32_f16(a: float16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42960,7 +42960,7 @@ pub fn vreinterpret_u32_f16(a: float16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -42983,7 +42983,7 @@ pub fn vreinterpret_u64_f16(a: float16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43007,7 +43007,7 @@ pub fn vreinterpret_u64_f16(a: float16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43030,7 +43030,7 @@ pub fn vreinterpret_p8_f16(a: float16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43057,7 +43057,7 @@ pub fn vreinterpret_p8_f16(a: float16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43080,7 +43080,7 @@ pub fn vreinterpret_p16_f16(a: float16x4_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43107,7 +43107,7 @@ pub fn vreinterpret_p16_f16(a: float16x4_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43130,7 +43130,7 @@ pub fn vreinterpretq_f32_f16(a: float16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43157,7 +43157,7 @@ pub fn vreinterpretq_f32_f16(a: float16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43180,7 +43180,7 @@ pub fn vreinterpretq_s8_f16(a: float16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43211,7 +43211,7 @@ pub fn vreinterpretq_s8_f16(a: float16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43234,7 +43234,7 @@ pub fn vreinterpretq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43261,7 +43261,7 @@ pub fn vreinterpretq_s16_f16(a: float16x8_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43284,7 +43284,7 @@ pub fn vreinterpretq_s32_f16(a: float16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43311,7 +43311,7 @@ pub fn vreinterpretq_s32_f16(a: float16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43334,7 +43334,7 @@ pub fn vreinterpretq_s64_f16(a: float16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43361,7 +43361,7 @@ pub fn vreinterpretq_s64_f16(a: float16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43384,7 +43384,7 @@ pub fn vreinterpretq_u8_f16(a: float16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43415,7 +43415,7 @@ pub fn vreinterpretq_u8_f16(a: float16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43438,7 +43438,7 @@ pub fn vreinterpretq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43465,7 +43465,7 @@ pub fn vreinterpretq_u16_f16(a: float16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43488,7 +43488,7 @@ pub fn vreinterpretq_u32_f16(a: float16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43515,7 +43515,7 @@ pub fn vreinterpretq_u32_f16(a: float16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43538,7 +43538,7 @@ pub fn vreinterpretq_u64_f16(a: float16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43565,7 +43565,7 @@ pub fn vreinterpretq_u64_f16(a: float16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43588,7 +43588,7 @@ pub fn vreinterpretq_p8_f16(a: float16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43619,7 +43619,7 @@ pub fn vreinterpretq_p8_f16(a: float16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43642,7 +43642,7 @@ pub fn vreinterpretq_p16_f16(a: float16x8_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43669,7 +43669,7 @@ pub fn vreinterpretq_p16_f16(a: float16x8_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43692,7 +43692,7 @@ pub fn vreinterpret_f16_f32(a: float32x2_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43719,7 +43719,7 @@ pub fn vreinterpret_f16_f32(a: float32x2_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43742,7 +43742,7 @@ pub fn vreinterpretq_f16_f32(a: float32x4_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43769,7 +43769,7 @@ pub fn vreinterpretq_f16_f32(a: float32x4_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43792,7 +43792,7 @@ pub fn vreinterpret_f16_s8(a: int8x8_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43819,7 +43819,7 @@ pub fn vreinterpret_f16_s8(a: int8x8_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43842,7 +43842,7 @@ pub fn vreinterpretq_f16_s8(a: int8x16_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43870,7 +43870,7 @@ pub fn vreinterpretq_f16_s8(a: int8x16_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43893,7 +43893,7 @@ pub fn vreinterpret_f16_s16(a: int16x4_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43920,7 +43920,7 @@ pub fn vreinterpret_f16_s16(a: int16x4_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43943,7 +43943,7 @@ pub fn vreinterpretq_f16_s16(a: int16x8_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43970,7 +43970,7 @@ pub fn vreinterpretq_f16_s16(a: int16x8_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -43993,7 +43993,7 @@ pub fn vreinterpret_f16_s32(a: int32x2_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44020,7 +44020,7 @@ pub fn vreinterpret_f16_s32(a: int32x2_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44043,7 +44043,7 @@ pub fn vreinterpretq_f16_s32(a: int32x4_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44070,7 +44070,7 @@ pub fn vreinterpretq_f16_s32(a: int32x4_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44093,7 +44093,7 @@ pub fn vreinterpret_f16_s64(a: int64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44119,7 +44119,7 @@ pub fn vreinterpret_f16_s64(a: int64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44142,7 +44142,7 @@ pub fn vreinterpretq_f16_s64(a: int64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44169,7 +44169,7 @@ pub fn vreinterpretq_f16_s64(a: int64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44192,7 +44192,7 @@ pub fn vreinterpret_f16_u8(a: uint8x8_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44219,7 +44219,7 @@ pub fn vreinterpret_f16_u8(a: uint8x8_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44242,7 +44242,7 @@ pub fn vreinterpretq_f16_u8(a: uint8x16_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44270,7 +44270,7 @@ pub fn vreinterpretq_f16_u8(a: uint8x16_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44293,7 +44293,7 @@ pub fn vreinterpret_f16_u16(a: uint16x4_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44320,7 +44320,7 @@ pub fn vreinterpret_f16_u16(a: uint16x4_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44343,7 +44343,7 @@ pub fn vreinterpretq_f16_u16(a: uint16x8_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44370,7 +44370,7 @@ pub fn vreinterpretq_f16_u16(a: uint16x8_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44393,7 +44393,7 @@ pub fn vreinterpret_f16_u32(a: uint32x2_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44420,7 +44420,7 @@ pub fn vreinterpret_f16_u32(a: uint32x2_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44443,7 +44443,7 @@ pub fn vreinterpretq_f16_u32(a: uint32x4_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44470,7 +44470,7 @@ pub fn vreinterpretq_f16_u32(a: uint32x4_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44493,7 +44493,7 @@ pub fn vreinterpret_f16_u64(a: uint64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44519,7 +44519,7 @@ pub fn vreinterpret_f16_u64(a: uint64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44542,7 +44542,7 @@ pub fn vreinterpretq_f16_u64(a: uint64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44569,7 +44569,7 @@ pub fn vreinterpretq_f16_u64(a: uint64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44592,7 +44592,7 @@ pub fn vreinterpret_f16_p8(a: poly8x8_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44619,7 +44619,7 @@ pub fn vreinterpret_f16_p8(a: poly8x8_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44642,7 +44642,7 @@ pub fn vreinterpretq_f16_p8(a: poly8x16_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44670,7 +44670,7 @@ pub fn vreinterpretq_f16_p8(a: poly8x16_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44693,7 +44693,7 @@ pub fn vreinterpret_f16_p16(a: poly16x4_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44720,7 +44720,7 @@ pub fn vreinterpret_f16_p16(a: poly16x4_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44743,7 +44743,7 @@ pub fn vreinterpretq_f16_p16(a: poly16x8_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44770,7 +44770,7 @@ pub fn vreinterpretq_f16_p16(a: poly16x8_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44793,7 +44793,7 @@ pub fn vreinterpretq_f16_p128(a: p128) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44819,7 +44819,7 @@ pub fn vreinterpretq_f16_p128(a: p128) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44842,7 +44842,7 @@ pub fn vreinterpret_p64_f16(a: float16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44866,7 +44866,7 @@ pub fn vreinterpret_p64_f16(a: float16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44889,7 +44889,7 @@ pub fn vreinterpretq_p128_f16(a: float16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44913,7 +44913,7 @@ pub fn vreinterpretq_p128_f16(a: float16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44936,7 +44936,7 @@ pub fn vreinterpretq_p64_f16(a: float16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44963,7 +44963,7 @@ pub fn vreinterpretq_p64_f16(a: float16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -44986,7 +44986,7 @@ pub fn vreinterpret_f16_p64(a: poly64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -45012,7 +45012,7 @@ pub fn vreinterpret_f16_p64(a: poly64x1_t) -> float16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -45035,7 +45035,7 @@ pub fn vreinterpretq_f16_p64(a: poly64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -45062,7 +45062,7 @@ pub fn vreinterpretq_f16_p64(a: poly64x2_t) -> float16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45084,7 +45084,7 @@ pub fn vreinterpretq_f32_p128(a: p128) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45109,7 +45109,7 @@ pub fn vreinterpretq_f32_p128(a: p128) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45131,7 +45131,7 @@ pub fn vreinterpret_s8_f32(a: float32x2_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45157,7 +45157,7 @@ pub fn vreinterpret_s8_f32(a: float32x2_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45179,7 +45179,7 @@ pub fn vreinterpret_s16_f32(a: float32x2_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45205,7 +45205,7 @@ pub fn vreinterpret_s16_f32(a: float32x2_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45227,7 +45227,7 @@ pub fn vreinterpret_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45253,7 +45253,7 @@ pub fn vreinterpret_s32_f32(a: float32x2_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45275,7 +45275,7 @@ pub fn vreinterpret_s64_f32(a: float32x2_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45298,7 +45298,7 @@ pub fn vreinterpret_s64_f32(a: float32x2_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45320,7 +45320,7 @@ pub fn vreinterpret_u8_f32(a: float32x2_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45346,7 +45346,7 @@ pub fn vreinterpret_u8_f32(a: float32x2_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45368,7 +45368,7 @@ pub fn vreinterpret_u16_f32(a: float32x2_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45394,7 +45394,7 @@ pub fn vreinterpret_u16_f32(a: float32x2_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45416,7 +45416,7 @@ pub fn vreinterpret_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45442,7 +45442,7 @@ pub fn vreinterpret_u32_f32(a: float32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45464,7 +45464,7 @@ pub fn vreinterpret_u64_f32(a: float32x2_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45487,7 +45487,7 @@ pub fn vreinterpret_u64_f32(a: float32x2_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45509,7 +45509,7 @@ pub fn vreinterpret_p8_f32(a: float32x2_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45535,7 +45535,7 @@ pub fn vreinterpret_p8_f32(a: float32x2_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45557,7 +45557,7 @@ pub fn vreinterpret_p16_f32(a: float32x2_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45583,7 +45583,7 @@ pub fn vreinterpret_p16_f32(a: float32x2_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45605,7 +45605,7 @@ pub fn vreinterpretq_p128_f32(a: float32x4_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45628,7 +45628,7 @@ pub fn vreinterpretq_p128_f32(a: float32x4_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45650,7 +45650,7 @@ pub fn vreinterpretq_s8_f32(a: float32x4_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45680,7 +45680,7 @@ pub fn vreinterpretq_s8_f32(a: float32x4_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45702,7 +45702,7 @@ pub fn vreinterpretq_s16_f32(a: float32x4_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45728,7 +45728,7 @@ pub fn vreinterpretq_s16_f32(a: float32x4_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45750,7 +45750,7 @@ pub fn vreinterpretq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45776,7 +45776,7 @@ pub fn vreinterpretq_s32_f32(a: float32x4_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45798,7 +45798,7 @@ pub fn vreinterpretq_s64_f32(a: float32x4_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45824,7 +45824,7 @@ pub fn vreinterpretq_s64_f32(a: float32x4_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45846,7 +45846,7 @@ pub fn vreinterpretq_u8_f32(a: float32x4_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45876,7 +45876,7 @@ pub fn vreinterpretq_u8_f32(a: float32x4_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45898,7 +45898,7 @@ pub fn vreinterpretq_u16_f32(a: float32x4_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45924,7 +45924,7 @@ pub fn vreinterpretq_u16_f32(a: float32x4_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45946,7 +45946,7 @@ pub fn vreinterpretq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45972,7 +45972,7 @@ pub fn vreinterpretq_u32_f32(a: float32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -45994,7 +45994,7 @@ pub fn vreinterpretq_u64_f32(a: float32x4_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46020,7 +46020,7 @@ pub fn vreinterpretq_u64_f32(a: float32x4_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46042,7 +46042,7 @@ pub fn vreinterpretq_p8_f32(a: float32x4_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46072,7 +46072,7 @@ pub fn vreinterpretq_p8_f32(a: float32x4_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46094,7 +46094,7 @@ pub fn vreinterpretq_p16_f32(a: float32x4_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_f32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46120,7 +46120,7 @@ pub fn vreinterpretq_p16_f32(a: float32x4_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46142,7 +46142,7 @@ pub fn vreinterpret_f32_s8(a: int8x8_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46168,7 +46168,7 @@ pub fn vreinterpret_f32_s8(a: int8x8_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46190,7 +46190,7 @@ pub fn vreinterpret_s16_s8(a: int8x8_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46216,7 +46216,7 @@ pub fn vreinterpret_s16_s8(a: int8x8_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46238,7 +46238,7 @@ pub fn vreinterpret_s32_s8(a: int8x8_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46264,7 +46264,7 @@ pub fn vreinterpret_s32_s8(a: int8x8_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46286,7 +46286,7 @@ pub fn vreinterpret_s64_s8(a: int8x8_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46309,7 +46309,7 @@ pub fn vreinterpret_s64_s8(a: int8x8_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46331,7 +46331,7 @@ pub fn vreinterpret_u8_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46357,7 +46357,7 @@ pub fn vreinterpret_u8_s8(a: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46379,7 +46379,7 @@ pub fn vreinterpret_u16_s8(a: int8x8_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46405,7 +46405,7 @@ pub fn vreinterpret_u16_s8(a: int8x8_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46427,7 +46427,7 @@ pub fn vreinterpret_u32_s8(a: int8x8_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46453,7 +46453,7 @@ pub fn vreinterpret_u32_s8(a: int8x8_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46475,7 +46475,7 @@ pub fn vreinterpret_u64_s8(a: int8x8_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46498,7 +46498,7 @@ pub fn vreinterpret_u64_s8(a: int8x8_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46520,7 +46520,7 @@ pub fn vreinterpret_p8_s8(a: int8x8_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46546,7 +46546,7 @@ pub fn vreinterpret_p8_s8(a: int8x8_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46568,7 +46568,7 @@ pub fn vreinterpret_p16_s8(a: int8x8_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46594,7 +46594,7 @@ pub fn vreinterpret_p16_s8(a: int8x8_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46616,7 +46616,7 @@ pub fn vreinterpretq_f32_s8(a: int8x16_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46643,7 +46643,7 @@ pub fn vreinterpretq_f32_s8(a: int8x16_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46665,7 +46665,7 @@ pub fn vreinterpretq_s16_s8(a: int8x16_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46692,7 +46692,7 @@ pub fn vreinterpretq_s16_s8(a: int8x16_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46714,7 +46714,7 @@ pub fn vreinterpretq_s32_s8(a: int8x16_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46741,7 +46741,7 @@ pub fn vreinterpretq_s32_s8(a: int8x16_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46763,7 +46763,7 @@ pub fn vreinterpretq_s64_s8(a: int8x16_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46790,7 +46790,7 @@ pub fn vreinterpretq_s64_s8(a: int8x16_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46812,7 +46812,7 @@ pub fn vreinterpretq_u8_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46843,7 +46843,7 @@ pub fn vreinterpretq_u8_s8(a: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46865,7 +46865,7 @@ pub fn vreinterpretq_u16_s8(a: int8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46892,7 +46892,7 @@ pub fn vreinterpretq_u16_s8(a: int8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46914,7 +46914,7 @@ pub fn vreinterpretq_u32_s8(a: int8x16_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46941,7 +46941,7 @@ pub fn vreinterpretq_u32_s8(a: int8x16_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46963,7 +46963,7 @@ pub fn vreinterpretq_u64_s8(a: int8x16_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -46990,7 +46990,7 @@ pub fn vreinterpretq_u64_s8(a: int8x16_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47012,7 +47012,7 @@ pub fn vreinterpretq_p8_s8(a: int8x16_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47043,7 +47043,7 @@ pub fn vreinterpretq_p8_s8(a: int8x16_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47065,7 +47065,7 @@ pub fn vreinterpretq_p16_s8(a: int8x16_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47092,7 +47092,7 @@ pub fn vreinterpretq_p16_s8(a: int8x16_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47114,7 +47114,7 @@ pub fn vreinterpret_f32_s16(a: int16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47140,7 +47140,7 @@ pub fn vreinterpret_f32_s16(a: int16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47162,7 +47162,7 @@ pub fn vreinterpret_s8_s16(a: int16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47188,7 +47188,7 @@ pub fn vreinterpret_s8_s16(a: int16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47210,7 +47210,7 @@ pub fn vreinterpret_s32_s16(a: int16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47236,7 +47236,7 @@ pub fn vreinterpret_s32_s16(a: int16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47258,7 +47258,7 @@ pub fn vreinterpret_s64_s16(a: int16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47281,7 +47281,7 @@ pub fn vreinterpret_s64_s16(a: int16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47303,7 +47303,7 @@ pub fn vreinterpret_u8_s16(a: int16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47329,7 +47329,7 @@ pub fn vreinterpret_u8_s16(a: int16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47351,7 +47351,7 @@ pub fn vreinterpret_u16_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47377,7 +47377,7 @@ pub fn vreinterpret_u16_s16(a: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47399,7 +47399,7 @@ pub fn vreinterpret_u32_s16(a: int16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47425,7 +47425,7 @@ pub fn vreinterpret_u32_s16(a: int16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47447,7 +47447,7 @@ pub fn vreinterpret_u64_s16(a: int16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47470,7 +47470,7 @@ pub fn vreinterpret_u64_s16(a: int16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47492,7 +47492,7 @@ pub fn vreinterpret_p8_s16(a: int16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47518,7 +47518,7 @@ pub fn vreinterpret_p8_s16(a: int16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47540,7 +47540,7 @@ pub fn vreinterpret_p16_s16(a: int16x4_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47566,7 +47566,7 @@ pub fn vreinterpret_p16_s16(a: int16x4_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47588,7 +47588,7 @@ pub fn vreinterpretq_f32_s16(a: int16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47614,7 +47614,7 @@ pub fn vreinterpretq_f32_s16(a: int16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47636,7 +47636,7 @@ pub fn vreinterpretq_s8_s16(a: int16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47666,7 +47666,7 @@ pub fn vreinterpretq_s8_s16(a: int16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47688,7 +47688,7 @@ pub fn vreinterpretq_s32_s16(a: int16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47714,7 +47714,7 @@ pub fn vreinterpretq_s32_s16(a: int16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47736,7 +47736,7 @@ pub fn vreinterpretq_s64_s16(a: int16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47762,7 +47762,7 @@ pub fn vreinterpretq_s64_s16(a: int16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47784,7 +47784,7 @@ pub fn vreinterpretq_u8_s16(a: int16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47814,7 +47814,7 @@ pub fn vreinterpretq_u8_s16(a: int16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47836,7 +47836,7 @@ pub fn vreinterpretq_u16_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47862,7 +47862,7 @@ pub fn vreinterpretq_u16_s16(a: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47884,7 +47884,7 @@ pub fn vreinterpretq_u32_s16(a: int16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47910,7 +47910,7 @@ pub fn vreinterpretq_u32_s16(a: int16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47932,7 +47932,7 @@ pub fn vreinterpretq_u64_s16(a: int16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47958,7 +47958,7 @@ pub fn vreinterpretq_u64_s16(a: int16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -47980,7 +47980,7 @@ pub fn vreinterpretq_p8_s16(a: int16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48010,7 +48010,7 @@ pub fn vreinterpretq_p8_s16(a: int16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48032,7 +48032,7 @@ pub fn vreinterpretq_p16_s16(a: int16x8_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48058,7 +48058,7 @@ pub fn vreinterpretq_p16_s16(a: int16x8_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48080,7 +48080,7 @@ pub fn vreinterpret_f32_s32(a: int32x2_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48106,7 +48106,7 @@ pub fn vreinterpret_f32_s32(a: int32x2_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48128,7 +48128,7 @@ pub fn vreinterpret_s8_s32(a: int32x2_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48154,7 +48154,7 @@ pub fn vreinterpret_s8_s32(a: int32x2_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48176,7 +48176,7 @@ pub fn vreinterpret_s16_s32(a: int32x2_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48202,7 +48202,7 @@ pub fn vreinterpret_s16_s32(a: int32x2_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48224,7 +48224,7 @@ pub fn vreinterpret_s64_s32(a: int32x2_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48247,7 +48247,7 @@ pub fn vreinterpret_s64_s32(a: int32x2_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48269,7 +48269,7 @@ pub fn vreinterpret_u8_s32(a: int32x2_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48295,7 +48295,7 @@ pub fn vreinterpret_u8_s32(a: int32x2_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48317,7 +48317,7 @@ pub fn vreinterpret_u16_s32(a: int32x2_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48343,7 +48343,7 @@ pub fn vreinterpret_u16_s32(a: int32x2_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48365,7 +48365,7 @@ pub fn vreinterpret_u32_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48391,7 +48391,7 @@ pub fn vreinterpret_u32_s32(a: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48413,7 +48413,7 @@ pub fn vreinterpret_u64_s32(a: int32x2_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48436,7 +48436,7 @@ pub fn vreinterpret_u64_s32(a: int32x2_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48458,7 +48458,7 @@ pub fn vreinterpret_p8_s32(a: int32x2_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48484,7 +48484,7 @@ pub fn vreinterpret_p8_s32(a: int32x2_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48506,7 +48506,7 @@ pub fn vreinterpret_p16_s32(a: int32x2_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48532,7 +48532,7 @@ pub fn vreinterpret_p16_s32(a: int32x2_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48554,7 +48554,7 @@ pub fn vreinterpretq_f32_s32(a: int32x4_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48580,7 +48580,7 @@ pub fn vreinterpretq_f32_s32(a: int32x4_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48602,7 +48602,7 @@ pub fn vreinterpretq_s8_s32(a: int32x4_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48632,7 +48632,7 @@ pub fn vreinterpretq_s8_s32(a: int32x4_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48654,7 +48654,7 @@ pub fn vreinterpretq_s16_s32(a: int32x4_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48680,7 +48680,7 @@ pub fn vreinterpretq_s16_s32(a: int32x4_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48702,7 +48702,7 @@ pub fn vreinterpretq_s64_s32(a: int32x4_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48728,7 +48728,7 @@ pub fn vreinterpretq_s64_s32(a: int32x4_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48750,7 +48750,7 @@ pub fn vreinterpretq_u8_s32(a: int32x4_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48780,7 +48780,7 @@ pub fn vreinterpretq_u8_s32(a: int32x4_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48802,7 +48802,7 @@ pub fn vreinterpretq_u16_s32(a: int32x4_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48828,7 +48828,7 @@ pub fn vreinterpretq_u16_s32(a: int32x4_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48850,7 +48850,7 @@ pub fn vreinterpretq_u32_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48876,7 +48876,7 @@ pub fn vreinterpretq_u32_s32(a: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48898,7 +48898,7 @@ pub fn vreinterpretq_u64_s32(a: int32x4_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48924,7 +48924,7 @@ pub fn vreinterpretq_u64_s32(a: int32x4_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48946,7 +48946,7 @@ pub fn vreinterpretq_p8_s32(a: int32x4_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48976,7 +48976,7 @@ pub fn vreinterpretq_p8_s32(a: int32x4_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -48998,7 +48998,7 @@ pub fn vreinterpretq_p16_s32(a: int32x4_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49024,7 +49024,7 @@ pub fn vreinterpretq_p16_s32(a: int32x4_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49046,7 +49046,7 @@ pub fn vreinterpret_f32_s64(a: int64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49071,7 +49071,7 @@ pub fn vreinterpret_f32_s64(a: int64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49093,7 +49093,7 @@ pub fn vreinterpret_s8_s64(a: int64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49118,7 +49118,7 @@ pub fn vreinterpret_s8_s64(a: int64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49140,7 +49140,7 @@ pub fn vreinterpret_s16_s64(a: int64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49165,7 +49165,7 @@ pub fn vreinterpret_s16_s64(a: int64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49187,7 +49187,7 @@ pub fn vreinterpret_s32_s64(a: int64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49212,7 +49212,7 @@ pub fn vreinterpret_s32_s64(a: int64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49234,7 +49234,7 @@ pub fn vreinterpret_u8_s64(a: int64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49259,7 +49259,7 @@ pub fn vreinterpret_u8_s64(a: int64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49281,7 +49281,7 @@ pub fn vreinterpret_u16_s64(a: int64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49306,7 +49306,7 @@ pub fn vreinterpret_u16_s64(a: int64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49328,7 +49328,7 @@ pub fn vreinterpret_u32_s64(a: int64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49353,7 +49353,7 @@ pub fn vreinterpret_u32_s64(a: int64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -49374,7 +49374,7 @@ pub fn vreinterpret_u64_s64(a: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49396,7 +49396,7 @@ pub fn vreinterpret_p8_s64(a: int64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49421,7 +49421,7 @@ pub fn vreinterpret_p8_s64(a: int64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49443,7 +49443,7 @@ pub fn vreinterpret_p16_s64(a: int64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49468,7 +49468,7 @@ pub fn vreinterpret_p16_s64(a: int64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49490,7 +49490,7 @@ pub fn vreinterpretq_f32_s64(a: int64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49516,7 +49516,7 @@ pub fn vreinterpretq_f32_s64(a: int64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49538,7 +49538,7 @@ pub fn vreinterpretq_s8_s64(a: int64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49568,7 +49568,7 @@ pub fn vreinterpretq_s8_s64(a: int64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49590,7 +49590,7 @@ pub fn vreinterpretq_s16_s64(a: int64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49616,7 +49616,7 @@ pub fn vreinterpretq_s16_s64(a: int64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49638,7 +49638,7 @@ pub fn vreinterpretq_s32_s64(a: int64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49664,7 +49664,7 @@ pub fn vreinterpretq_s32_s64(a: int64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49686,7 +49686,7 @@ pub fn vreinterpretq_u8_s64(a: int64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49716,7 +49716,7 @@ pub fn vreinterpretq_u8_s64(a: int64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49738,7 +49738,7 @@ pub fn vreinterpretq_u16_s64(a: int64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49764,7 +49764,7 @@ pub fn vreinterpretq_u16_s64(a: int64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49786,7 +49786,7 @@ pub fn vreinterpretq_u32_s64(a: int64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49812,7 +49812,7 @@ pub fn vreinterpretq_u32_s64(a: int64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49834,7 +49834,7 @@ pub fn vreinterpretq_u64_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49860,7 +49860,7 @@ pub fn vreinterpretq_u64_s64(a: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49882,7 +49882,7 @@ pub fn vreinterpretq_p8_s64(a: int64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49912,7 +49912,7 @@ pub fn vreinterpretq_p8_s64(a: int64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49934,7 +49934,7 @@ pub fn vreinterpretq_p16_s64(a: int64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49960,7 +49960,7 @@ pub fn vreinterpretq_p16_s64(a: int64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -49982,7 +49982,7 @@ pub fn vreinterpret_f32_u8(a: uint8x8_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50008,7 +50008,7 @@ pub fn vreinterpret_f32_u8(a: uint8x8_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50030,7 +50030,7 @@ pub fn vreinterpret_s8_u8(a: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50056,7 +50056,7 @@ pub fn vreinterpret_s8_u8(a: uint8x8_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50078,7 +50078,7 @@ pub fn vreinterpret_s16_u8(a: uint8x8_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50104,7 +50104,7 @@ pub fn vreinterpret_s16_u8(a: uint8x8_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50126,7 +50126,7 @@ pub fn vreinterpret_s32_u8(a: uint8x8_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50152,7 +50152,7 @@ pub fn vreinterpret_s32_u8(a: uint8x8_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50174,7 +50174,7 @@ pub fn vreinterpret_s64_u8(a: uint8x8_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50197,7 +50197,7 @@ pub fn vreinterpret_s64_u8(a: uint8x8_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50219,7 +50219,7 @@ pub fn vreinterpret_u16_u8(a: uint8x8_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50245,7 +50245,7 @@ pub fn vreinterpret_u16_u8(a: uint8x8_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50267,7 +50267,7 @@ pub fn vreinterpret_u32_u8(a: uint8x8_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50293,7 +50293,7 @@ pub fn vreinterpret_u32_u8(a: uint8x8_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50315,7 +50315,7 @@ pub fn vreinterpret_u64_u8(a: uint8x8_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50338,7 +50338,7 @@ pub fn vreinterpret_u64_u8(a: uint8x8_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50360,7 +50360,7 @@ pub fn vreinterpret_p8_u8(a: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50386,7 +50386,7 @@ pub fn vreinterpret_p8_u8(a: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50408,7 +50408,7 @@ pub fn vreinterpret_p16_u8(a: uint8x8_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50434,7 +50434,7 @@ pub fn vreinterpret_p16_u8(a: uint8x8_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50456,7 +50456,7 @@ pub fn vreinterpretq_f32_u8(a: uint8x16_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50483,7 +50483,7 @@ pub fn vreinterpretq_f32_u8(a: uint8x16_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50505,7 +50505,7 @@ pub fn vreinterpretq_s8_u8(a: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50536,7 +50536,7 @@ pub fn vreinterpretq_s8_u8(a: uint8x16_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50558,7 +50558,7 @@ pub fn vreinterpretq_s16_u8(a: uint8x16_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50585,7 +50585,7 @@ pub fn vreinterpretq_s16_u8(a: uint8x16_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50607,7 +50607,7 @@ pub fn vreinterpretq_s32_u8(a: uint8x16_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50634,7 +50634,7 @@ pub fn vreinterpretq_s32_u8(a: uint8x16_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50656,7 +50656,7 @@ pub fn vreinterpretq_s64_u8(a: uint8x16_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50683,7 +50683,7 @@ pub fn vreinterpretq_s64_u8(a: uint8x16_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50705,7 +50705,7 @@ pub fn vreinterpretq_u16_u8(a: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50732,7 +50732,7 @@ pub fn vreinterpretq_u16_u8(a: uint8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50754,7 +50754,7 @@ pub fn vreinterpretq_u32_u8(a: uint8x16_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50781,7 +50781,7 @@ pub fn vreinterpretq_u32_u8(a: uint8x16_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50803,7 +50803,7 @@ pub fn vreinterpretq_u64_u8(a: uint8x16_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50830,7 +50830,7 @@ pub fn vreinterpretq_u64_u8(a: uint8x16_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50852,7 +50852,7 @@ pub fn vreinterpretq_p8_u8(a: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50883,7 +50883,7 @@ pub fn vreinterpretq_p8_u8(a: uint8x16_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50905,7 +50905,7 @@ pub fn vreinterpretq_p16_u8(a: uint8x16_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50932,7 +50932,7 @@ pub fn vreinterpretq_p16_u8(a: uint8x16_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50954,7 +50954,7 @@ pub fn vreinterpret_f32_u16(a: uint16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -50980,7 +50980,7 @@ pub fn vreinterpret_f32_u16(a: uint16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51002,7 +51002,7 @@ pub fn vreinterpret_s8_u16(a: uint16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51028,7 +51028,7 @@ pub fn vreinterpret_s8_u16(a: uint16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51050,7 +51050,7 @@ pub fn vreinterpret_s16_u16(a: uint16x4_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51076,7 +51076,7 @@ pub fn vreinterpret_s16_u16(a: uint16x4_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51098,7 +51098,7 @@ pub fn vreinterpret_s32_u16(a: uint16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51124,7 +51124,7 @@ pub fn vreinterpret_s32_u16(a: uint16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51146,7 +51146,7 @@ pub fn vreinterpret_s64_u16(a: uint16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51169,7 +51169,7 @@ pub fn vreinterpret_s64_u16(a: uint16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51191,7 +51191,7 @@ pub fn vreinterpret_u8_u16(a: uint16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51217,7 +51217,7 @@ pub fn vreinterpret_u8_u16(a: uint16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51239,7 +51239,7 @@ pub fn vreinterpret_u32_u16(a: uint16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51265,7 +51265,7 @@ pub fn vreinterpret_u32_u16(a: uint16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51287,7 +51287,7 @@ pub fn vreinterpret_u64_u16(a: uint16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51310,7 +51310,7 @@ pub fn vreinterpret_u64_u16(a: uint16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51332,7 +51332,7 @@ pub fn vreinterpret_p8_u16(a: uint16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51358,7 +51358,7 @@ pub fn vreinterpret_p8_u16(a: uint16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51380,7 +51380,7 @@ pub fn vreinterpret_p16_u16(a: uint16x4_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51406,7 +51406,7 @@ pub fn vreinterpret_p16_u16(a: uint16x4_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51428,7 +51428,7 @@ pub fn vreinterpretq_f32_u16(a: uint16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51454,7 +51454,7 @@ pub fn vreinterpretq_f32_u16(a: uint16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51476,7 +51476,7 @@ pub fn vreinterpretq_s8_u16(a: uint16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51506,7 +51506,7 @@ pub fn vreinterpretq_s8_u16(a: uint16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51528,7 +51528,7 @@ pub fn vreinterpretq_s16_u16(a: uint16x8_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51554,7 +51554,7 @@ pub fn vreinterpretq_s16_u16(a: uint16x8_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51576,7 +51576,7 @@ pub fn vreinterpretq_s32_u16(a: uint16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51602,7 +51602,7 @@ pub fn vreinterpretq_s32_u16(a: uint16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51624,7 +51624,7 @@ pub fn vreinterpretq_s64_u16(a: uint16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51650,7 +51650,7 @@ pub fn vreinterpretq_s64_u16(a: uint16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51672,7 +51672,7 @@ pub fn vreinterpretq_u8_u16(a: uint16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51702,7 +51702,7 @@ pub fn vreinterpretq_u8_u16(a: uint16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51724,7 +51724,7 @@ pub fn vreinterpretq_u32_u16(a: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51750,7 +51750,7 @@ pub fn vreinterpretq_u32_u16(a: uint16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51772,7 +51772,7 @@ pub fn vreinterpretq_u64_u16(a: uint16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51798,7 +51798,7 @@ pub fn vreinterpretq_u64_u16(a: uint16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51820,7 +51820,7 @@ pub fn vreinterpretq_p8_u16(a: uint16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51850,7 +51850,7 @@ pub fn vreinterpretq_p8_u16(a: uint16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51872,7 +51872,7 @@ pub fn vreinterpretq_p16_u16(a: uint16x8_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51898,7 +51898,7 @@ pub fn vreinterpretq_p16_u16(a: uint16x8_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51920,7 +51920,7 @@ pub fn vreinterpret_f32_u32(a: uint32x2_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51946,7 +51946,7 @@ pub fn vreinterpret_f32_u32(a: uint32x2_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51968,7 +51968,7 @@ pub fn vreinterpret_s8_u32(a: uint32x2_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -51994,7 +51994,7 @@ pub fn vreinterpret_s8_u32(a: uint32x2_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52016,7 +52016,7 @@ pub fn vreinterpret_s16_u32(a: uint32x2_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52042,7 +52042,7 @@ pub fn vreinterpret_s16_u32(a: uint32x2_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52064,7 +52064,7 @@ pub fn vreinterpret_s32_u32(a: uint32x2_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52090,7 +52090,7 @@ pub fn vreinterpret_s32_u32(a: uint32x2_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52112,7 +52112,7 @@ pub fn vreinterpret_s64_u32(a: uint32x2_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52135,7 +52135,7 @@ pub fn vreinterpret_s64_u32(a: uint32x2_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52157,7 +52157,7 @@ pub fn vreinterpret_u8_u32(a: uint32x2_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52183,7 +52183,7 @@ pub fn vreinterpret_u8_u32(a: uint32x2_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52205,7 +52205,7 @@ pub fn vreinterpret_u16_u32(a: uint32x2_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52231,7 +52231,7 @@ pub fn vreinterpret_u16_u32(a: uint32x2_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52253,7 +52253,7 @@ pub fn vreinterpret_u64_u32(a: uint32x2_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52276,7 +52276,7 @@ pub fn vreinterpret_u64_u32(a: uint32x2_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52298,7 +52298,7 @@ pub fn vreinterpret_p8_u32(a: uint32x2_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52324,7 +52324,7 @@ pub fn vreinterpret_p8_u32(a: uint32x2_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52346,7 +52346,7 @@ pub fn vreinterpret_p16_u32(a: uint32x2_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52372,7 +52372,7 @@ pub fn vreinterpret_p16_u32(a: uint32x2_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52394,7 +52394,7 @@ pub fn vreinterpretq_f32_u32(a: uint32x4_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52420,7 +52420,7 @@ pub fn vreinterpretq_f32_u32(a: uint32x4_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52442,7 +52442,7 @@ pub fn vreinterpretq_s8_u32(a: uint32x4_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52472,7 +52472,7 @@ pub fn vreinterpretq_s8_u32(a: uint32x4_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52494,7 +52494,7 @@ pub fn vreinterpretq_s16_u32(a: uint32x4_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52520,7 +52520,7 @@ pub fn vreinterpretq_s16_u32(a: uint32x4_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52542,7 +52542,7 @@ pub fn vreinterpretq_s32_u32(a: uint32x4_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52568,7 +52568,7 @@ pub fn vreinterpretq_s32_u32(a: uint32x4_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52590,7 +52590,7 @@ pub fn vreinterpretq_s64_u32(a: uint32x4_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52616,7 +52616,7 @@ pub fn vreinterpretq_s64_u32(a: uint32x4_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52638,7 +52638,7 @@ pub fn vreinterpretq_u8_u32(a: uint32x4_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52668,7 +52668,7 @@ pub fn vreinterpretq_u8_u32(a: uint32x4_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52690,7 +52690,7 @@ pub fn vreinterpretq_u16_u32(a: uint32x4_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52716,7 +52716,7 @@ pub fn vreinterpretq_u16_u32(a: uint32x4_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52738,7 +52738,7 @@ pub fn vreinterpretq_u64_u32(a: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52764,7 +52764,7 @@ pub fn vreinterpretq_u64_u32(a: uint32x4_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52786,7 +52786,7 @@ pub fn vreinterpretq_p8_u32(a: uint32x4_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52816,7 +52816,7 @@ pub fn vreinterpretq_p8_u32(a: uint32x4_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52838,7 +52838,7 @@ pub fn vreinterpretq_p16_u32(a: uint32x4_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52864,7 +52864,7 @@ pub fn vreinterpretq_p16_u32(a: uint32x4_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52886,7 +52886,7 @@ pub fn vreinterpret_f32_u64(a: uint64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52911,7 +52911,7 @@ pub fn vreinterpret_f32_u64(a: uint64x1_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52933,7 +52933,7 @@ pub fn vreinterpret_s8_u64(a: uint64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52958,7 +52958,7 @@ pub fn vreinterpret_s8_u64(a: uint64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -52980,7 +52980,7 @@ pub fn vreinterpret_s16_u64(a: uint64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53005,7 +53005,7 @@ pub fn vreinterpret_s16_u64(a: uint64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53027,7 +53027,7 @@ pub fn vreinterpret_s32_u64(a: uint64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53052,7 +53052,7 @@ pub fn vreinterpret_s32_u64(a: uint64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -53073,7 +53073,7 @@ pub fn vreinterpret_s64_u64(a: uint64x1_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53095,7 +53095,7 @@ pub fn vreinterpret_u8_u64(a: uint64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53120,7 +53120,7 @@ pub fn vreinterpret_u8_u64(a: uint64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53142,7 +53142,7 @@ pub fn vreinterpret_u16_u64(a: uint64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53167,7 +53167,7 @@ pub fn vreinterpret_u16_u64(a: uint64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53189,7 +53189,7 @@ pub fn vreinterpret_u32_u64(a: uint64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53214,7 +53214,7 @@ pub fn vreinterpret_u32_u64(a: uint64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53236,7 +53236,7 @@ pub fn vreinterpret_p8_u64(a: uint64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53261,7 +53261,7 @@ pub fn vreinterpret_p8_u64(a: uint64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53283,7 +53283,7 @@ pub fn vreinterpret_p16_u64(a: uint64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53308,7 +53308,7 @@ pub fn vreinterpret_p16_u64(a: uint64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53330,7 +53330,7 @@ pub fn vreinterpretq_f32_u64(a: uint64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53356,7 +53356,7 @@ pub fn vreinterpretq_f32_u64(a: uint64x2_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53378,7 +53378,7 @@ pub fn vreinterpretq_s8_u64(a: uint64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53408,7 +53408,7 @@ pub fn vreinterpretq_s8_u64(a: uint64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53430,7 +53430,7 @@ pub fn vreinterpretq_s16_u64(a: uint64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53456,7 +53456,7 @@ pub fn vreinterpretq_s16_u64(a: uint64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53478,7 +53478,7 @@ pub fn vreinterpretq_s32_u64(a: uint64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53504,7 +53504,7 @@ pub fn vreinterpretq_s32_u64(a: uint64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53526,7 +53526,7 @@ pub fn vreinterpretq_s64_u64(a: uint64x2_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53552,7 +53552,7 @@ pub fn vreinterpretq_s64_u64(a: uint64x2_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53574,7 +53574,7 @@ pub fn vreinterpretq_u8_u64(a: uint64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53604,7 +53604,7 @@ pub fn vreinterpretq_u8_u64(a: uint64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53626,7 +53626,7 @@ pub fn vreinterpretq_u16_u64(a: uint64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53652,7 +53652,7 @@ pub fn vreinterpretq_u16_u64(a: uint64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53674,7 +53674,7 @@ pub fn vreinterpretq_u32_u64(a: uint64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53700,7 +53700,7 @@ pub fn vreinterpretq_u32_u64(a: uint64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53722,7 +53722,7 @@ pub fn vreinterpretq_p8_u64(a: uint64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53752,7 +53752,7 @@ pub fn vreinterpretq_p8_u64(a: uint64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53774,7 +53774,7 @@ pub fn vreinterpretq_p16_u64(a: uint64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53800,7 +53800,7 @@ pub fn vreinterpretq_p16_u64(a: uint64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53822,7 +53822,7 @@ pub fn vreinterpret_f32_p8(a: poly8x8_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53848,7 +53848,7 @@ pub fn vreinterpret_f32_p8(a: poly8x8_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53870,7 +53870,7 @@ pub fn vreinterpret_s8_p8(a: poly8x8_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53896,7 +53896,7 @@ pub fn vreinterpret_s8_p8(a: poly8x8_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53918,7 +53918,7 @@ pub fn vreinterpret_s16_p8(a: poly8x8_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53944,7 +53944,7 @@ pub fn vreinterpret_s16_p8(a: poly8x8_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53966,7 +53966,7 @@ pub fn vreinterpret_s32_p8(a: poly8x8_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -53992,7 +53992,7 @@ pub fn vreinterpret_s32_p8(a: poly8x8_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54014,7 +54014,7 @@ pub fn vreinterpret_s64_p8(a: poly8x8_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54037,7 +54037,7 @@ pub fn vreinterpret_s64_p8(a: poly8x8_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54059,7 +54059,7 @@ pub fn vreinterpret_u8_p8(a: poly8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54085,7 +54085,7 @@ pub fn vreinterpret_u8_p8(a: poly8x8_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54107,7 +54107,7 @@ pub fn vreinterpret_u16_p8(a: poly8x8_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54133,7 +54133,7 @@ pub fn vreinterpret_u16_p8(a: poly8x8_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54155,7 +54155,7 @@ pub fn vreinterpret_u32_p8(a: poly8x8_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54181,7 +54181,7 @@ pub fn vreinterpret_u32_p8(a: poly8x8_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54203,7 +54203,7 @@ pub fn vreinterpret_u64_p8(a: poly8x8_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54226,7 +54226,7 @@ pub fn vreinterpret_u64_p8(a: poly8x8_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54248,7 +54248,7 @@ pub fn vreinterpret_p16_p8(a: poly8x8_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54274,7 +54274,7 @@ pub fn vreinterpret_p16_p8(a: poly8x8_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54296,7 +54296,7 @@ pub fn vreinterpretq_f32_p8(a: poly8x16_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54323,7 +54323,7 @@ pub fn vreinterpretq_f32_p8(a: poly8x16_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54345,7 +54345,7 @@ pub fn vreinterpretq_s8_p8(a: poly8x16_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54376,7 +54376,7 @@ pub fn vreinterpretq_s8_p8(a: poly8x16_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54398,7 +54398,7 @@ pub fn vreinterpretq_s16_p8(a: poly8x16_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54425,7 +54425,7 @@ pub fn vreinterpretq_s16_p8(a: poly8x16_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54447,7 +54447,7 @@ pub fn vreinterpretq_s32_p8(a: poly8x16_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54474,7 +54474,7 @@ pub fn vreinterpretq_s32_p8(a: poly8x16_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54496,7 +54496,7 @@ pub fn vreinterpretq_s64_p8(a: poly8x16_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54523,7 +54523,7 @@ pub fn vreinterpretq_s64_p8(a: poly8x16_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54545,7 +54545,7 @@ pub fn vreinterpretq_u8_p8(a: poly8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54576,7 +54576,7 @@ pub fn vreinterpretq_u8_p8(a: poly8x16_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54598,7 +54598,7 @@ pub fn vreinterpretq_u16_p8(a: poly8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54625,7 +54625,7 @@ pub fn vreinterpretq_u16_p8(a: poly8x16_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54647,7 +54647,7 @@ pub fn vreinterpretq_u32_p8(a: poly8x16_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54674,7 +54674,7 @@ pub fn vreinterpretq_u32_p8(a: poly8x16_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54696,7 +54696,7 @@ pub fn vreinterpretq_u64_p8(a: poly8x16_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54723,7 +54723,7 @@ pub fn vreinterpretq_u64_p8(a: poly8x16_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54745,7 +54745,7 @@ pub fn vreinterpretq_p16_p8(a: poly8x16_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54772,7 +54772,7 @@ pub fn vreinterpretq_p16_p8(a: poly8x16_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54794,7 +54794,7 @@ pub fn vreinterpret_f32_p16(a: poly16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_f32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54820,7 +54820,7 @@ pub fn vreinterpret_f32_p16(a: poly16x4_t) -> float32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54842,7 +54842,7 @@ pub fn vreinterpret_s8_p16(a: poly16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54868,7 +54868,7 @@ pub fn vreinterpret_s8_p16(a: poly16x4_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54890,7 +54890,7 @@ pub fn vreinterpret_s16_p16(a: poly16x4_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54916,7 +54916,7 @@ pub fn vreinterpret_s16_p16(a: poly16x4_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54938,7 +54938,7 @@ pub fn vreinterpret_s32_p16(a: poly16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54964,7 +54964,7 @@ pub fn vreinterpret_s32_p16(a: poly16x4_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -54986,7 +54986,7 @@ pub fn vreinterpret_s64_p16(a: poly16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55009,7 +55009,7 @@ pub fn vreinterpret_s64_p16(a: poly16x4_t) -> int64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55031,7 +55031,7 @@ pub fn vreinterpret_u8_p16(a: poly16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55057,7 +55057,7 @@ pub fn vreinterpret_u8_p16(a: poly16x4_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55079,7 +55079,7 @@ pub fn vreinterpret_u16_p16(a: poly16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55105,7 +55105,7 @@ pub fn vreinterpret_u16_p16(a: poly16x4_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55127,7 +55127,7 @@ pub fn vreinterpret_u32_p16(a: poly16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55153,7 +55153,7 @@ pub fn vreinterpret_u32_p16(a: poly16x4_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55175,7 +55175,7 @@ pub fn vreinterpret_u64_p16(a: poly16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55198,7 +55198,7 @@ pub fn vreinterpret_u64_p16(a: poly16x4_t) -> uint64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55220,7 +55220,7 @@ pub fn vreinterpret_p8_p16(a: poly16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55246,7 +55246,7 @@ pub fn vreinterpret_p8_p16(a: poly16x4_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55268,7 +55268,7 @@ pub fn vreinterpretq_f32_p16(a: poly16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_f32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55294,7 +55294,7 @@ pub fn vreinterpretq_f32_p16(a: poly16x8_t) -> float32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55316,7 +55316,7 @@ pub fn vreinterpretq_s8_p16(a: poly16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55346,7 +55346,7 @@ pub fn vreinterpretq_s8_p16(a: poly16x8_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55368,7 +55368,7 @@ pub fn vreinterpretq_s16_p16(a: poly16x8_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55394,7 +55394,7 @@ pub fn vreinterpretq_s16_p16(a: poly16x8_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55416,7 +55416,7 @@ pub fn vreinterpretq_s32_p16(a: poly16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55442,7 +55442,7 @@ pub fn vreinterpretq_s32_p16(a: poly16x8_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55464,7 +55464,7 @@ pub fn vreinterpretq_s64_p16(a: poly16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55490,7 +55490,7 @@ pub fn vreinterpretq_s64_p16(a: poly16x8_t) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55512,7 +55512,7 @@ pub fn vreinterpretq_u8_p16(a: poly16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55542,7 +55542,7 @@ pub fn vreinterpretq_u8_p16(a: poly16x8_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55564,7 +55564,7 @@ pub fn vreinterpretq_u16_p16(a: poly16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55590,7 +55590,7 @@ pub fn vreinterpretq_u16_p16(a: poly16x8_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55612,7 +55612,7 @@ pub fn vreinterpretq_u32_p16(a: poly16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55638,7 +55638,7 @@ pub fn vreinterpretq_u32_p16(a: poly16x8_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55660,7 +55660,7 @@ pub fn vreinterpretq_u64_p16(a: poly16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55686,7 +55686,7 @@ pub fn vreinterpretq_u64_p16(a: poly16x8_t) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55708,7 +55708,7 @@ pub fn vreinterpretq_p8_p16(a: poly16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -55738,7 +55738,7 @@ pub fn vreinterpretq_p8_p16(a: poly16x8_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55760,7 +55760,7 @@ pub fn vreinterpretq_s8_p128(a: p128) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55789,7 +55789,7 @@ pub fn vreinterpretq_s8_p128(a: p128) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55811,7 +55811,7 @@ pub fn vreinterpretq_s16_p128(a: p128) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55836,7 +55836,7 @@ pub fn vreinterpretq_s16_p128(a: p128) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55858,7 +55858,7 @@ pub fn vreinterpretq_s32_p128(a: p128) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55883,7 +55883,7 @@ pub fn vreinterpretq_s32_p128(a: p128) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55905,7 +55905,7 @@ pub fn vreinterpretq_s64_p128(a: p128) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55930,7 +55930,7 @@ pub fn vreinterpretq_s64_p128(a: p128) -> int64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55952,7 +55952,7 @@ pub fn vreinterpretq_u8_p128(a: p128) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -55981,7 +55981,7 @@ pub fn vreinterpretq_u8_p128(a: p128) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56003,7 +56003,7 @@ pub fn vreinterpretq_u16_p128(a: p128) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56028,7 +56028,7 @@ pub fn vreinterpretq_u16_p128(a: p128) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56050,7 +56050,7 @@ pub fn vreinterpretq_u32_p128(a: p128) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56075,7 +56075,7 @@ pub fn vreinterpretq_u32_p128(a: p128) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56097,7 +56097,7 @@ pub fn vreinterpretq_u64_p128(a: p128) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56122,7 +56122,7 @@ pub fn vreinterpretq_u64_p128(a: p128) -> uint64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56144,7 +56144,7 @@ pub fn vreinterpretq_p8_p128(a: p128) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56173,7 +56173,7 @@ pub fn vreinterpretq_p8_p128(a: p128) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56195,7 +56195,7 @@ pub fn vreinterpretq_p16_p128(a: p128) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56220,7 +56220,7 @@ pub fn vreinterpretq_p16_p128(a: p128) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56242,7 +56242,7 @@ pub fn vreinterpretq_p64_p128(a: p128) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p128)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56267,7 +56267,7 @@ pub fn vreinterpretq_p64_p128(a: p128) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56289,7 +56289,7 @@ pub fn vreinterpret_p64_s8(a: int8x8_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56312,7 +56312,7 @@ pub fn vreinterpret_p64_s8(a: int8x8_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56334,7 +56334,7 @@ pub fn vreinterpretq_p128_s8(a: int8x16_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56358,7 +56358,7 @@ pub fn vreinterpretq_p128_s8(a: int8x16_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56380,7 +56380,7 @@ pub fn vreinterpretq_p64_s8(a: int8x16_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56407,7 +56407,7 @@ pub fn vreinterpretq_p64_s8(a: int8x16_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56429,7 +56429,7 @@ pub fn vreinterpret_p64_s16(a: int16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56452,7 +56452,7 @@ pub fn vreinterpret_p64_s16(a: int16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56474,7 +56474,7 @@ pub fn vreinterpretq_p128_s16(a: int16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56497,7 +56497,7 @@ pub fn vreinterpretq_p128_s16(a: int16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56519,7 +56519,7 @@ pub fn vreinterpretq_p64_s16(a: int16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56545,7 +56545,7 @@ pub fn vreinterpretq_p64_s16(a: int16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56567,7 +56567,7 @@ pub fn vreinterpret_p64_s32(a: int32x2_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56590,7 +56590,7 @@ pub fn vreinterpret_p64_s32(a: int32x2_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56612,7 +56612,7 @@ pub fn vreinterpretq_p128_s32(a: int32x4_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56635,7 +56635,7 @@ pub fn vreinterpretq_p128_s32(a: int32x4_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56657,7 +56657,7 @@ pub fn vreinterpretq_p64_s32(a: int32x4_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56683,7 +56683,7 @@ pub fn vreinterpretq_p64_s32(a: int32x4_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56705,7 +56705,7 @@ pub fn vreinterpretq_p128_s64(a: int64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56728,7 +56728,7 @@ pub fn vreinterpretq_p128_s64(a: int64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56750,7 +56750,7 @@ pub fn vreinterpret_p64_u8(a: uint8x8_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56773,7 +56773,7 @@ pub fn vreinterpret_p64_u8(a: uint8x8_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56795,7 +56795,7 @@ pub fn vreinterpretq_p128_u8(a: uint8x16_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56819,7 +56819,7 @@ pub fn vreinterpretq_p128_u8(a: uint8x16_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56841,7 +56841,7 @@ pub fn vreinterpretq_p64_u8(a: uint8x16_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56868,7 +56868,7 @@ pub fn vreinterpretq_p64_u8(a: uint8x16_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56890,7 +56890,7 @@ pub fn vreinterpret_p64_u16(a: uint16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56913,7 +56913,7 @@ pub fn vreinterpret_p64_u16(a: uint16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56935,7 +56935,7 @@ pub fn vreinterpretq_p128_u16(a: uint16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56958,7 +56958,7 @@ pub fn vreinterpretq_p128_u16(a: uint16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -56980,7 +56980,7 @@ pub fn vreinterpretq_p64_u16(a: uint16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57006,7 +57006,7 @@ pub fn vreinterpretq_p64_u16(a: uint16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57028,7 +57028,7 @@ pub fn vreinterpret_p64_u32(a: uint32x2_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57051,7 +57051,7 @@ pub fn vreinterpret_p64_u32(a: uint32x2_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57073,7 +57073,7 @@ pub fn vreinterpretq_p128_u32(a: uint32x4_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57096,7 +57096,7 @@ pub fn vreinterpretq_p128_u32(a: uint32x4_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57118,7 +57118,7 @@ pub fn vreinterpretq_p64_u32(a: uint32x4_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57144,7 +57144,7 @@ pub fn vreinterpretq_p64_u32(a: uint32x4_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57166,7 +57166,7 @@ pub fn vreinterpretq_p128_u64(a: uint64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57189,7 +57189,7 @@ pub fn vreinterpretq_p128_u64(a: uint64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57211,7 +57211,7 @@ pub fn vreinterpret_p64_p8(a: poly8x8_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57234,7 +57234,7 @@ pub fn vreinterpret_p64_p8(a: poly8x8_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57256,7 +57256,7 @@ pub fn vreinterpretq_p128_p8(a: poly8x16_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57280,7 +57280,7 @@ pub fn vreinterpretq_p128_p8(a: poly8x16_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57302,7 +57302,7 @@ pub fn vreinterpretq_p64_p8(a: poly8x16_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57329,7 +57329,7 @@ pub fn vreinterpretq_p64_p8(a: poly8x16_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57351,7 +57351,7 @@ pub fn vreinterpret_p64_p16(a: poly16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57374,7 +57374,7 @@ pub fn vreinterpret_p64_p16(a: poly16x4_t) -> poly64x1_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57396,7 +57396,7 @@ pub fn vreinterpretq_p128_p16(a: poly16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57419,7 +57419,7 @@ pub fn vreinterpretq_p128_p16(a: poly16x8_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57441,7 +57441,7 @@ pub fn vreinterpretq_p64_p16(a: poly16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p64_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57467,7 +57467,7 @@ pub fn vreinterpretq_p64_p16(a: poly16x8_t) -> poly64x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57489,7 +57489,7 @@ pub fn vreinterpret_s8_p64(a: poly64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57514,7 +57514,7 @@ pub fn vreinterpret_s8_p64(a: poly64x1_t) -> int8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57536,7 +57536,7 @@ pub fn vreinterpret_s16_p64(a: poly64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57561,7 +57561,7 @@ pub fn vreinterpret_s16_p64(a: poly64x1_t) -> int16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57583,7 +57583,7 @@ pub fn vreinterpret_s32_p64(a: poly64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_s32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57608,7 +57608,7 @@ pub fn vreinterpret_s32_p64(a: poly64x1_t) -> int32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57630,7 +57630,7 @@ pub fn vreinterpret_u8_p64(a: poly64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57655,7 +57655,7 @@ pub fn vreinterpret_u8_p64(a: poly64x1_t) -> uint8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57677,7 +57677,7 @@ pub fn vreinterpret_u16_p64(a: poly64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57702,7 +57702,7 @@ pub fn vreinterpret_u16_p64(a: poly64x1_t) -> uint16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57724,7 +57724,7 @@ pub fn vreinterpret_u32_p64(a: poly64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_u32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57749,7 +57749,7 @@ pub fn vreinterpret_u32_p64(a: poly64x1_t) -> uint32x2_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57771,7 +57771,7 @@ pub fn vreinterpret_p8_p64(a: poly64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57796,7 +57796,7 @@ pub fn vreinterpret_p8_p64(a: poly64x1_t) -> poly8x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57818,7 +57818,7 @@ pub fn vreinterpret_p16_p64(a: poly64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpret_p16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57843,7 +57843,7 @@ pub fn vreinterpret_p16_p64(a: poly64x1_t) -> poly16x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57865,7 +57865,7 @@ pub fn vreinterpretq_p128_p64(a: poly64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p128_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57888,7 +57888,7 @@ pub fn vreinterpretq_p128_p64(a: poly64x2_t) -> p128 {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57910,7 +57910,7 @@ pub fn vreinterpretq_s8_p64(a: poly64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57940,7 +57940,7 @@ pub fn vreinterpretq_s8_p64(a: poly64x2_t) -> int8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57962,7 +57962,7 @@ pub fn vreinterpretq_s16_p64(a: poly64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -57988,7 +57988,7 @@ pub fn vreinterpretq_s16_p64(a: poly64x2_t) -> int16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58010,7 +58010,7 @@ pub fn vreinterpretq_s32_p64(a: poly64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_s32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58036,7 +58036,7 @@ pub fn vreinterpretq_s32_p64(a: poly64x2_t) -> int32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58058,7 +58058,7 @@ pub fn vreinterpretq_u8_p64(a: poly64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58088,7 +58088,7 @@ pub fn vreinterpretq_u8_p64(a: poly64x2_t) -> uint8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58110,7 +58110,7 @@ pub fn vreinterpretq_u16_p64(a: poly64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58136,7 +58136,7 @@ pub fn vreinterpretq_u16_p64(a: poly64x2_t) -> uint16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58158,7 +58158,7 @@ pub fn vreinterpretq_u32_p64(a: poly64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_u32_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58184,7 +58184,7 @@ pub fn vreinterpretq_u32_p64(a: poly64x2_t) -> uint32x4_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58206,7 +58206,7 @@ pub fn vreinterpretq_p8_p64(a: poly64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p8_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58236,7 +58236,7 @@ pub fn vreinterpretq_p8_p64(a: poly64x2_t) -> poly8x16_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58258,7 +58258,7 @@ pub fn vreinterpretq_p16_p64(a: poly64x2_t) -> poly16x8_t {
 }
 #[doc = "Vector reinterpret cast operation"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vreinterpretq_p16_p64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -58284,7 +58284,7 @@ pub fn vreinterpretq_p16_p64(a: poly64x2_t) -> poly16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev16_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))]
@@ -58305,7 +58305,7 @@ pub fn vrev16_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev16_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))]
@@ -58326,7 +58326,7 @@ pub fn vrev16_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev16_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))]
@@ -58347,7 +58347,7 @@ pub fn vrev16_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev16q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))]
@@ -58368,7 +58368,7 @@ pub fn vrev16q_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev16q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))]
@@ -58389,7 +58389,7 @@ pub fn vrev16q_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev16q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))]
@@ -58410,7 +58410,7 @@ pub fn vrev16q_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))]
@@ -58431,7 +58431,7 @@ pub fn vrev32_p16(a: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))]
@@ -58452,7 +58452,7 @@ pub fn vrev32_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))]
@@ -58473,7 +58473,7 @@ pub fn vrev32_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))]
@@ -58494,7 +58494,7 @@ pub fn vrev32_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))]
@@ -58515,7 +58515,7 @@ pub fn vrev32_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))]
@@ -58536,7 +58536,7 @@ pub fn vrev32_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))]
@@ -58557,7 +58557,7 @@ pub fn vrev32q_p16(a: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))]
@@ -58578,7 +58578,7 @@ pub fn vrev32q_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))]
@@ -58599,7 +58599,7 @@ pub fn vrev32q_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))]
@@ -58620,7 +58620,7 @@ pub fn vrev32q_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))]
@@ -58641,7 +58641,7 @@ pub fn vrev32q_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev32q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))]
@@ -58662,7 +58662,7 @@ pub fn vrev32q_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))]
@@ -58683,7 +58683,7 @@ pub fn vrev64_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))]
@@ -58704,7 +58704,7 @@ pub fn vrev64_p16(a: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))]
@@ -58725,7 +58725,7 @@ pub fn vrev64_p8(a: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))]
@@ -58746,7 +58746,7 @@ pub fn vrev64_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))]
@@ -58767,7 +58767,7 @@ pub fn vrev64_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))]
@@ -58788,7 +58788,7 @@ pub fn vrev64_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))]
@@ -58809,7 +58809,7 @@ pub fn vrev64_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))]
@@ -58830,7 +58830,7 @@ pub fn vrev64_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))]
@@ -58851,7 +58851,7 @@ pub fn vrev64_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))]
@@ -58872,7 +58872,7 @@ pub fn vrev64q_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))]
@@ -58893,7 +58893,7 @@ pub fn vrev64q_p16(a: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))]
@@ -58914,7 +58914,7 @@ pub fn vrev64q_p8(a: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))]
@@ -58935,7 +58935,7 @@ pub fn vrev64q_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))]
@@ -58956,7 +58956,7 @@ pub fn vrev64q_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))]
@@ -58977,7 +58977,7 @@ pub fn vrev64q_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))]
@@ -58998,7 +58998,7 @@ pub fn vrev64q_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))]
@@ -59019,7 +59019,7 @@ pub fn vrev64q_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Reversing vector elements (swap endianness)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))]
@@ -59040,7 +59040,7 @@ pub fn vrev64q_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Reverse elements in 64-bit doublewords"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrev64))]
 #[cfg_attr(
@@ -59062,7 +59062,7 @@ pub fn vrev64_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Reverse elements in 64-bit doublewords"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrev64q_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrev64))]
 #[cfg_attr(
@@ -59084,7 +59084,7 @@ pub fn vrev64q_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))]
@@ -59113,7 +59113,7 @@ pub fn vrhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))]
@@ -59142,7 +59142,7 @@ pub fn vrhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))]
@@ -59171,7 +59171,7 @@ pub fn vrhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))]
@@ -59200,7 +59200,7 @@ pub fn vrhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))]
@@ -59229,7 +59229,7 @@ pub fn vrhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))]
@@ -59258,7 +59258,7 @@ pub fn vrhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))]
@@ -59287,7 +59287,7 @@ pub fn vrhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))]
@@ -59316,7 +59316,7 @@ pub fn vrhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))]
@@ -59345,7 +59345,7 @@ pub fn vrhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))]
@@ -59374,7 +59374,7 @@ pub fn vrhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhadd_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))]
@@ -59403,7 +59403,7 @@ pub fn vrhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Rounding halving add"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrhaddq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))]
@@ -59432,7 +59432,7 @@ pub fn vrhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndn_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))]
 #[cfg_attr(
@@ -59461,7 +59461,7 @@ pub fn vrndn_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndnq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))]
 #[cfg_attr(
@@ -59490,7 +59490,7 @@ pub fn vrndnq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndn_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))]
@@ -59518,7 +59518,7 @@ pub fn vrndn_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point round to integral, to nearest with ties to even"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndnq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))]
@@ -59546,7 +59546,7 @@ pub fn vrndnq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59575,7 +59575,7 @@ pub fn vrshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59604,7 +59604,7 @@ pub fn vrshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59633,7 +59633,7 @@ pub fn vrshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59662,7 +59662,7 @@ pub fn vrshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59691,7 +59691,7 @@ pub fn vrshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59720,7 +59720,7 @@ pub fn vrshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59749,7 +59749,7 @@ pub fn vrshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59778,7 +59778,7 @@ pub fn vrshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59807,7 +59807,7 @@ pub fn vrshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59836,7 +59836,7 @@ pub fn vrshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59865,7 +59865,7 @@ pub fn vrshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59894,7 +59894,7 @@ pub fn vrshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59923,7 +59923,7 @@ pub fn vrshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59952,7 +59952,7 @@ pub fn vrshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshl_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -59981,7 +59981,7 @@ pub fn vrshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned rounding shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshlq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))]
@@ -60010,7 +60010,7 @@ pub fn vrshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60033,7 +60033,7 @@ pub fn vrshr_n_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60056,7 +60056,7 @@ pub fn vrshrq_n_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60079,7 +60079,7 @@ pub fn vrshr_n_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60102,7 +60102,7 @@ pub fn vrshrq_n_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60125,7 +60125,7 @@ pub fn vrshr_n_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60148,7 +60148,7 @@ pub fn vrshrq_n_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60171,7 +60171,7 @@ pub fn vrshr_n_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60194,7 +60194,7 @@ pub fn vrshrq_n_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60217,7 +60217,7 @@ pub fn vrshr_n_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60240,7 +60240,7 @@ pub fn vrshrq_n_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60263,7 +60263,7 @@ pub fn vrshr_n_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60286,7 +60286,7 @@ pub fn vrshrq_n_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60309,7 +60309,7 @@ pub fn vrshr_n_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60332,7 +60332,7 @@ pub fn vrshrq_n_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshr_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60355,7 +60355,7 @@ pub fn vrshr_n_u64(a: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned rounding shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))]
@@ -60378,7 +60378,7 @@ pub fn vrshrq_n_u64(a: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vrshrn, N = 2))]
@@ -60394,7 +60394,7 @@ pub fn vrshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vrshrn, N = 2))]
@@ -60410,7 +60410,7 @@ pub fn vrshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vrshrn, N = 2))]
@@ -60426,7 +60426,7 @@ pub fn vrshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(rshrn, N = 2))]
@@ -60445,7 +60445,7 @@ pub fn vrshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(rshrn, N = 2))]
@@ -60464,7 +60464,7 @@ pub fn vrshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(rshrn, N = 2))]
@@ -60483,7 +60483,7 @@ pub fn vrshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))]
@@ -60506,7 +60506,7 @@ pub fn vrshrn_n_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))]
@@ -60529,7 +60529,7 @@ pub fn vrshrn_n_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Rounding shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshrn_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))]
@@ -60552,7 +60552,7 @@ pub fn vrshrn_n_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))]
@@ -60582,7 +60582,7 @@ pub fn vrsqrte_f16(a: float16x4_t) -> float16x4_t {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))]
@@ -60612,7 +60612,7 @@ pub fn vrsqrteq_f16(a: float16x8_t) -> float16x8_t {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))]
@@ -60641,7 +60641,7 @@ pub fn vrsqrte_f32(a: float32x2_t) -> float32x2_t {
 }
 #[doc = "Reciprocal square-root estimate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))]
@@ -60670,7 +60670,7 @@ pub fn vrsqrteq_f32(a: float32x4_t) -> float32x4_t {
 }
 #[doc = "Unsigned reciprocal square root estimate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrte_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))]
@@ -60699,7 +60699,7 @@ pub fn vrsqrte_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned reciprocal square root estimate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrteq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))]
@@ -60728,7 +60728,7 @@ pub fn vrsqrteq_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrts_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrts))]
@@ -60758,7 +60758,7 @@ pub fn vrsqrts_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,fp16")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrts))]
@@ -60788,7 +60788,7 @@ pub fn vrsqrtsq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrts_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrts))]
@@ -60817,7 +60817,7 @@ pub fn vrsqrts_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Floating-point reciprocal square root step"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsqrtsq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrts))]
@@ -60846,7 +60846,7 @@ pub fn vrsqrtsq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -60869,7 +60869,7 @@ pub fn vrsra_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -60892,7 +60892,7 @@ pub fn vrsraq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -60915,7 +60915,7 @@ pub fn vrsra_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -60938,7 +60938,7 @@ pub fn vrsraq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -60961,7 +60961,7 @@ pub fn vrsra_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -60984,7 +60984,7 @@ pub fn vrsraq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61007,7 +61007,7 @@ pub fn vrsra_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61030,7 +61030,7 @@ pub fn vrsraq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61053,7 +61053,7 @@ pub fn vrsra_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61076,7 +61076,7 @@ pub fn vrsraq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61099,7 +61099,7 @@ pub fn vrsra_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61122,7 +61122,7 @@ pub fn vrsraq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61145,7 +61145,7 @@ pub fn vrsra_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61168,7 +61168,7 @@ pub fn vrsraq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsra_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61191,7 +61191,7 @@ pub fn vrsra_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned rounding shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsraq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))]
@@ -61214,7 +61214,7 @@ pub fn vrsraq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))]
@@ -61243,7 +61243,7 @@ pub fn vrsubhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))]
@@ -61272,7 +61272,7 @@ pub fn vrsubhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsubhn))]
@@ -61301,7 +61301,7 @@ pub fn vrsubhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -61323,7 +61323,7 @@ pub fn vrsubhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -61350,7 +61350,7 @@ pub fn vrsubhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -61372,7 +61372,7 @@ pub fn vrsubhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -61399,7 +61399,7 @@ pub fn vrsubhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -61421,7 +61421,7 @@ pub fn vrsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Rounding subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrsubhn_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -61448,7 +61448,7 @@ pub fn vrsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
 #[cfg_attr(
@@ -61465,7 +61465,7 @@ pub fn vset_lane_f16(a: f16, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
 #[cfg_attr(
@@ -61482,7 +61482,7 @@ pub fn vsetq_lane_f16(a: f16, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61505,7 +61505,7 @@ pub fn vset_lane_f32(a: f32, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61528,7 +61528,7 @@ pub fn vsetq_lane_f32(a: f32, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61551,7 +61551,7 @@ pub fn vset_lane_s8(a: i8, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61574,7 +61574,7 @@ pub fn vsetq_lane_s8(a: i8, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61597,7 +61597,7 @@ pub fn vset_lane_s16(a: i16, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61620,7 +61620,7 @@ pub fn vsetq_lane_s16(a: i16, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61643,7 +61643,7 @@ pub fn vset_lane_s32(a: i32, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61666,7 +61666,7 @@ pub fn vsetq_lane_s32(a: i32, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61689,7 +61689,7 @@ pub fn vsetq_lane_s64(a: i64, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61712,7 +61712,7 @@ pub fn vset_lane_u8(a: u8, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61735,7 +61735,7 @@ pub fn vsetq_lane_u8(a: u8, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61758,7 +61758,7 @@ pub fn vset_lane_u16(a: u16, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61781,7 +61781,7 @@ pub fn vsetq_lane_u16(a: u16, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61804,7 +61804,7 @@ pub fn vset_lane_u32(a: u32, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61827,7 +61827,7 @@ pub fn vsetq_lane_u32(a: u32, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61850,7 +61850,7 @@ pub fn vsetq_lane_u64(a: u64, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61873,7 +61873,7 @@ pub fn vset_lane_p8(a: p8, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61896,7 +61896,7 @@ pub fn vsetq_lane_p8(a: p8, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61919,7 +61919,7 @@ pub fn vset_lane_p16(a: p16, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61942,7 +61942,7 @@ pub fn vsetq_lane_p16(a: p16, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61965,7 +61965,7 @@ pub fn vset_lane_p64(a: p64, b: poly64x1_t) -> poly64x1_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -61988,7 +61988,7 @@ pub fn vset_lane_s64(a: i64, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vset_lane_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -62011,7 +62011,7 @@ pub fn vset_lane_u64(a: u64, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Insert vector element from another vector element"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsetq_lane_p64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -62034,7 +62034,7 @@ pub fn vsetq_lane_p64(a: p64, b: poly64x2_t) -> poly64x2_t {
 }
 #[doc = "SHA1 hash update accelerator, choose."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1cq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha1c))]
@@ -62059,7 +62059,7 @@ pub fn vsha1cq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32
 }
 #[doc = "SHA1 fixed rotate."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1h_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha1h))]
@@ -62084,7 +62084,7 @@ pub fn vsha1h_u32(hash_e: u32) -> u32 {
 }
 #[doc = "SHA1 hash update accelerator, majority"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1mq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha1m))]
@@ -62109,7 +62109,7 @@ pub fn vsha1mq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32
 }
 #[doc = "SHA1 hash update accelerator, parity"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1pq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha1p))]
@@ -62134,7 +62134,7 @@ pub fn vsha1pq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32
 }
 #[doc = "SHA1 schedule update accelerator, first part."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1su0q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha1su0))]
@@ -62159,7 +62159,7 @@ pub fn vsha1su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t, w8_11: uint32x4_t) -> u
 }
 #[doc = "SHA1 schedule update accelerator, second part."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha1su1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha1su1))]
@@ -62184,7 +62184,7 @@ pub fn vsha1su1q_u32(tw0_3: uint32x4_t, w12_15: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "SHA1 schedule update accelerator, upper part."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256h2q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha256h2))]
@@ -62213,7 +62213,7 @@ pub fn vsha256h2q_u32(hash_abcd: uint32x4_t, hash_efgh: uint32x4_t, wk: uint32x4
 }
 #[doc = "SHA1 schedule update accelerator, first part."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256hq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha256h))]
@@ -62242,7 +62242,7 @@ pub fn vsha256hq_u32(hash_abcd: uint32x4_t, hash_efgh: uint32x4_t, wk: uint32x4_
 }
 #[doc = "SHA256 schedule update accelerator, first part."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256su0q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha256su0))]
@@ -62267,7 +62267,7 @@ pub fn vsha256su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "SHA256 schedule update accelerator, second part."]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsha256su1q_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "sha2")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(test, assert_instr(sha256su1))]
@@ -62291,7 +62291,7 @@ pub fn vsha256su1q_u32(tw0_3: uint32x4_t, w8_11: uint32x4_t, w12_15: uint32x4_t)
     }
     unsafe { _vsha256su1q_u32(tw0_3, w8_11, w12_15) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62304,7 +62304,7 @@ fn vshiftlins_v16i8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
     }
     unsafe { _vshiftlins_v16i8(a, b, const { int8x16_t([N as i8; 16]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62317,7 +62317,7 @@ fn vshiftlins_v1i64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
     }
     unsafe { _vshiftlins_v1i64(a, b, const { int64x1_t([N as i64; 1]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62330,7 +62330,7 @@ fn vshiftlins_v2i32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
     }
     unsafe { _vshiftlins_v2i32(a, b, const { int32x2_t([N; 2]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62343,7 +62343,7 @@ fn vshiftlins_v2i64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
     }
     unsafe { _vshiftlins_v2i64(a, b, const { int64x2_t([N as i64; 2]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62356,7 +62356,7 @@ fn vshiftlins_v4i16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
     }
     unsafe { _vshiftlins_v4i16(a, b, const { int16x4_t([N as i16; 4]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62369,7 +62369,7 @@ fn vshiftlins_v4i32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
     }
     unsafe { _vshiftlins_v4i32(a, b, const { int32x4_t([N; 4]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62382,7 +62382,7 @@ fn vshiftlins_v8i16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
     }
     unsafe { _vshiftlins_v8i16(a, b, const { int16x8_t([N as i16; 8]) }) }
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62397,7 +62397,7 @@ fn vshiftlins_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v16i8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62412,7 +62412,7 @@ fn vshiftrins_v16i8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v1i64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62427,7 +62427,7 @@ fn vshiftrins_v1i64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v2i32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62442,7 +62442,7 @@ fn vshiftrins_v2i32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v2i64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62457,7 +62457,7 @@ fn vshiftrins_v2i64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v4i16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62472,7 +62472,7 @@ fn vshiftrins_v4i16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v4i32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62487,7 +62487,7 @@ fn vshiftrins_v4i32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v8i16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62502,7 +62502,7 @@ fn vshiftrins_v8i16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshiftrins_v8i8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -62517,7 +62517,7 @@ fn vshiftrins_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62540,7 +62540,7 @@ pub fn vshl_n_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62563,7 +62563,7 @@ pub fn vshlq_n_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62586,7 +62586,7 @@ pub fn vshl_n_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62609,7 +62609,7 @@ pub fn vshlq_n_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62632,7 +62632,7 @@ pub fn vshl_n_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62655,7 +62655,7 @@ pub fn vshlq_n_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62678,7 +62678,7 @@ pub fn vshl_n_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62701,7 +62701,7 @@ pub fn vshlq_n_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62724,7 +62724,7 @@ pub fn vshl_n_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62747,7 +62747,7 @@ pub fn vshlq_n_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62770,7 +62770,7 @@ pub fn vshl_n_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62793,7 +62793,7 @@ pub fn vshlq_n_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62816,7 +62816,7 @@ pub fn vshl_n_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62839,7 +62839,7 @@ pub fn vshlq_n_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62862,7 +62862,7 @@ pub fn vshl_n_u64(a: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))]
@@ -62885,7 +62885,7 @@ pub fn vshlq_n_u64(a: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -62914,7 +62914,7 @@ pub fn vshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -62943,7 +62943,7 @@ pub fn vshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -62972,7 +62972,7 @@ pub fn vshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63001,7 +63001,7 @@ pub fn vshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63030,7 +63030,7 @@ pub fn vshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63059,7 +63059,7 @@ pub fn vshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63088,7 +63088,7 @@ pub fn vshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63117,7 +63117,7 @@ pub fn vshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63146,7 +63146,7 @@ pub fn vshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63175,7 +63175,7 @@ pub fn vshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63204,7 +63204,7 @@ pub fn vshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63233,7 +63233,7 @@ pub fn vshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63262,7 +63262,7 @@ pub fn vshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63291,7 +63291,7 @@ pub fn vshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshl_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63320,7 +63320,7 @@ pub fn vshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned Shift left"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshlq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))]
@@ -63349,7 +63349,7 @@ pub fn vshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s16", N = 2))]
@@ -63372,7 +63372,7 @@ pub fn vshll_n_s16(a: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s32", N = 2))]
@@ -63395,7 +63395,7 @@ pub fn vshll_n_s32(a: int32x2_t) -> int64x2_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s8", N = 2))]
@@ -63418,7 +63418,7 @@ pub fn vshll_n_s8(a: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u16", N = 2))]
@@ -63441,7 +63441,7 @@ pub fn vshll_n_u16(a: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u32", N = 2))]
@@ -63464,7 +63464,7 @@ pub fn vshll_n_u32(a: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Signed shift left long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshll_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u8", N = 2))]
@@ -63487,7 +63487,7 @@ pub fn vshll_n_u8(a: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s8", N = 2))]
@@ -63511,7 +63511,7 @@ pub fn vshr_n_s8(a: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s8", N = 2))]
@@ -63535,7 +63535,7 @@ pub fn vshrq_n_s8(a: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s16", N = 2))]
@@ -63559,7 +63559,7 @@ pub fn vshr_n_s16(a: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s16", N = 2))]
@@ -63583,7 +63583,7 @@ pub fn vshrq_n_s16(a: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s32", N = 2))]
@@ -63607,7 +63607,7 @@ pub fn vshr_n_s32(a: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s32", N = 2))]
@@ -63631,7 +63631,7 @@ pub fn vshrq_n_s32(a: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s64", N = 2))]
@@ -63655,7 +63655,7 @@ pub fn vshr_n_s64(a: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s64", N = 2))]
@@ -63679,7 +63679,7 @@ pub fn vshrq_n_s64(a: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", N = 2))]
@@ -63707,7 +63707,7 @@ pub fn vshr_n_u8(a: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", N = 2))]
@@ -63735,7 +63735,7 @@ pub fn vshrq_n_u8(a: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u16", N = 2))]
@@ -63763,7 +63763,7 @@ pub fn vshr_n_u16(a: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u16", N = 2))]
@@ -63791,7 +63791,7 @@ pub fn vshrq_n_u16(a: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u32", N = 2))]
@@ -63819,7 +63819,7 @@ pub fn vshr_n_u32(a: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u32", N = 2))]
@@ -63847,7 +63847,7 @@ pub fn vshrq_n_u32(a: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshr_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u64", N = 2))]
@@ -63875,7 +63875,7 @@ pub fn vshr_n_u64(a: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Shift right"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u64", N = 2))]
@@ -63903,7 +63903,7 @@ pub fn vshrq_n_u64(a: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i16", N = 2))]
@@ -63926,7 +63926,7 @@ pub fn vshrn_n_s16(a: int16x8_t) -> int8x8_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i32", N = 2))]
@@ -63949,7 +63949,7 @@ pub fn vshrn_n_s32(a: int32x4_t) -> int16x4_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i64", N = 2))]
@@ -63972,7 +63972,7 @@ pub fn vshrn_n_s64(a: int64x2_t) -> int32x2_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i16", N = 2))]
@@ -63995,7 +63995,7 @@ pub fn vshrn_n_u16(a: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i32", N = 2))]
@@ -64018,7 +64018,7 @@ pub fn vshrn_n_u32(a: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Shift right narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vshrn_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i64", N = 2))]
@@ -64041,7 +64041,7 @@ pub fn vshrn_n_u64(a: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64053,7 +64053,7 @@ pub fn vsli_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64065,7 +64065,7 @@ pub fn vsliq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64077,7 +64077,7 @@ pub fn vsli_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64089,7 +64089,7 @@ pub fn vsliq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64101,7 +64101,7 @@ pub fn vsli_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64113,7 +64113,7 @@ pub fn vsliq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64125,7 +64125,7 @@ pub fn vsli_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64137,7 +64137,7 @@ pub fn vsliq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64149,7 +64149,7 @@ pub fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64161,7 +64161,7 @@ pub fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64173,7 +64173,7 @@ pub fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64185,7 +64185,7 @@ pub fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64197,7 +64197,7 @@ pub fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64209,7 +64209,7 @@ pub fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64221,7 +64221,7 @@ pub fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64233,7 +64233,7 @@ pub fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64245,7 +64245,7 @@ pub fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64257,7 +64257,7 @@ pub fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsli_n_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64269,7 +64269,7 @@ pub fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Shift Left and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsliq_n_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64281,7 +64281,7 @@ pub fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64304,7 +64304,7 @@ pub fn vsra_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64327,7 +64327,7 @@ pub fn vsraq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64350,7 +64350,7 @@ pub fn vsra_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64373,7 +64373,7 @@ pub fn vsraq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64396,7 +64396,7 @@ pub fn vsra_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64419,7 +64419,7 @@ pub fn vsraq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64442,7 +64442,7 @@ pub fn vsra_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Signed shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64465,7 +64465,7 @@ pub fn vsraq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64488,7 +64488,7 @@ pub fn vsra_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64511,7 +64511,7 @@ pub fn vsraq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64534,7 +64534,7 @@ pub fn vsra_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64557,7 +64557,7 @@ pub fn vsraq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64580,7 +64580,7 @@ pub fn vsra_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64603,7 +64603,7 @@ pub fn vsraq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsra_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64626,7 +64626,7 @@ pub fn vsra_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Unsigned shift right and accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsraq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))]
@@ -64649,7 +64649,7 @@ pub fn vsraq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64661,7 +64661,7 @@ pub fn vsri_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64673,7 +64673,7 @@ pub fn vsriq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64685,7 +64685,7 @@ pub fn vsri_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64697,7 +64697,7 @@ pub fn vsriq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64709,7 +64709,7 @@ pub fn vsri_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64721,7 +64721,7 @@ pub fn vsriq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64733,7 +64733,7 @@ pub fn vsri_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64745,7 +64745,7 @@ pub fn vsriq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64757,7 +64757,7 @@ pub fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64769,7 +64769,7 @@ pub fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64781,7 +64781,7 @@ pub fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64793,7 +64793,7 @@ pub fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64805,7 +64805,7 @@ pub fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64817,7 +64817,7 @@ pub fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64829,7 +64829,7 @@ pub fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_u64)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64841,7 +64841,7 @@ pub fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64853,7 +64853,7 @@ pub fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64865,7 +64865,7 @@ pub fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64877,7 +64877,7 @@ pub fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t {
 }
 #[doc = "Shift Right and Insert (immediate)"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsriq_n_p16)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -64891,7 +64891,7 @@ pub fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[target_feature(enable = "neon,fp16")]
@@ -64909,7 +64909,7 @@ pub unsafe fn vst1_f16(ptr: *mut f16, a: float16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[target_feature(enable = "neon,fp16")]
@@ -64927,7 +64927,7 @@ pub unsafe fn vst1q_f16(ptr: *mut f16, a: float16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(vst1))]
@@ -64945,7 +64945,7 @@ pub unsafe fn vst1_f16_x2(a: *mut f16, b: float16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(vst1))]
@@ -64963,7 +64963,7 @@ pub unsafe fn vst1q_f16_x2(a: *mut f16, b: float16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
 #[target_feature(enable = "neon,fp16")]
@@ -64983,7 +64983,7 @@ pub unsafe fn vst1_f16_x2(a: *mut f16, b: float16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
 #[target_feature(enable = "neon,fp16")]
@@ -65003,7 +65003,7 @@ pub unsafe fn vst1q_f16_x2(a: *mut f16, b: float16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(vst1))]
@@ -65021,7 +65021,7 @@ pub unsafe fn vst1_f16_x3(a: *mut f16, b: float16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(test, assert_instr(vst1))]
@@ -65039,7 +65039,7 @@ pub unsafe fn vst1q_f16_x3(a: *mut f16, b: float16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
 #[target_feature(enable = "neon,fp16")]
@@ -65059,7 +65059,7 @@ pub unsafe fn vst1_f16_x3(a: *mut f16, b: float16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
 #[target_feature(enable = "neon,fp16")]
@@ -65079,7 +65079,7 @@ pub unsafe fn vst1q_f16_x3(a: *mut f16, b: float16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65104,7 +65104,7 @@ pub unsafe fn vst1_f16_x4(a: *mut f16, b: float16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65129,7 +65129,7 @@ pub unsafe fn vst1q_f16_x4(a: *mut f16, b: float16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
 #[target_feature(enable = "neon,fp16")]
@@ -65155,7 +65155,7 @@ pub unsafe fn vst1_f16_x4(a: *mut f16, b: float16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
 #[target_feature(enable = "neon,fp16")]
@@ -65181,7 +65181,7 @@ pub unsafe fn vst1q_f16_x4(a: *mut f16, b: float16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65195,7 +65195,7 @@ pub unsafe fn vst1_f32(ptr: *mut f32, a: float32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65209,7 +65209,7 @@ pub unsafe fn vst1q_f32(ptr: *mut f32, a: float32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65223,7 +65223,7 @@ pub unsafe fn vst1_s8(ptr: *mut i8, a: int8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65237,7 +65237,7 @@ pub unsafe fn vst1q_s8(ptr: *mut i8, a: int8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65251,7 +65251,7 @@ pub unsafe fn vst1_s16(ptr: *mut i16, a: int16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65265,7 +65265,7 @@ pub unsafe fn vst1q_s16(ptr: *mut i16, a: int16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65279,7 +65279,7 @@ pub unsafe fn vst1_s32(ptr: *mut i32, a: int32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65293,7 +65293,7 @@ pub unsafe fn vst1q_s32(ptr: *mut i32, a: int32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65307,7 +65307,7 @@ pub unsafe fn vst1_s64(ptr: *mut i64, a: int64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65321,7 +65321,7 @@ pub unsafe fn vst1q_s64(ptr: *mut i64, a: int64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65335,7 +65335,7 @@ pub unsafe fn vst1_u8(ptr: *mut u8, a: uint8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65349,7 +65349,7 @@ pub unsafe fn vst1q_u8(ptr: *mut u8, a: uint8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65363,7 +65363,7 @@ pub unsafe fn vst1_u16(ptr: *mut u16, a: uint16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65377,7 +65377,7 @@ pub unsafe fn vst1q_u16(ptr: *mut u16, a: uint16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65391,7 +65391,7 @@ pub unsafe fn vst1_u32(ptr: *mut u32, a: uint32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65405,7 +65405,7 @@ pub unsafe fn vst1q_u32(ptr: *mut u32, a: uint32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65419,7 +65419,7 @@ pub unsafe fn vst1_u64(ptr: *mut u64, a: uint64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65433,7 +65433,7 @@ pub unsafe fn vst1q_u64(ptr: *mut u64, a: uint64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65447,7 +65447,7 @@ pub unsafe fn vst1_p8(ptr: *mut p8, a: poly8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65461,7 +65461,7 @@ pub unsafe fn vst1q_p8(ptr: *mut p8, a: poly8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65475,7 +65475,7 @@ pub unsafe fn vst1_p16(ptr: *mut p16, a: poly16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65489,7 +65489,7 @@ pub unsafe fn vst1q_p16(ptr: *mut p16, a: poly16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65503,7 +65503,7 @@ pub unsafe fn vst1_p64(ptr: *mut p64, a: poly64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -65517,7 +65517,7 @@ pub unsafe fn vst1q_p64(ptr: *mut p64, a: poly64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst1))]
@@ -65533,7 +65533,7 @@ pub unsafe fn vst1_f32_x2(a: *mut f32, b: float32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst1))]
@@ -65549,7 +65549,7 @@ pub unsafe fn vst1q_f32_x2(a: *mut f32, b: float32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
@@ -65568,7 +65568,7 @@ pub unsafe fn vst1_f32_x2(a: *mut f32, b: float32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
@@ -65587,7 +65587,7 @@ pub unsafe fn vst1q_f32_x2(a: *mut f32, b: float32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
@@ -65606,7 +65606,7 @@ pub unsafe fn vst1_f32_x3(a: *mut f32, b: float32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
@@ -65625,7 +65625,7 @@ pub unsafe fn vst1q_f32_x3(a: *mut f32, b: float32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -65647,7 +65647,7 @@ pub unsafe fn vst1_f32_x4(a: *mut f32, b: float32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -65669,7 +65669,7 @@ pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_f32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
@@ -65694,7 +65694,7 @@ pub unsafe fn vst1_f32_x4(a: *mut f32, b: float32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_f32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(test, assert_instr(st1))]
@@ -65719,7 +65719,7 @@ pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65739,7 +65739,7 @@ pub unsafe fn vst1_lane_f16(a: *mut f16, b: float16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65759,7 +65759,7 @@ pub unsafe fn vst1q_lane_f16(a: *mut f16, b: float16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65784,7 +65784,7 @@ pub unsafe fn vst1_lane_f32(a: *mut f32, b: float32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65809,7 +65809,7 @@ pub unsafe fn vst1q_lane_f32(a: *mut f32, b: float32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65834,7 +65834,7 @@ pub unsafe fn vst1_lane_s8(a: *mut i8, b: int8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65859,7 +65859,7 @@ pub unsafe fn vst1q_lane_s8(a: *mut i8, b: int8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65884,7 +65884,7 @@ pub unsafe fn vst1_lane_s16(a: *mut i16, b: int16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65909,7 +65909,7 @@ pub unsafe fn vst1q_lane_s16(a: *mut i16, b: int16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65934,7 +65934,7 @@ pub unsafe fn vst1_lane_s32(a: *mut i32, b: int32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65959,7 +65959,7 @@ pub unsafe fn vst1q_lane_s32(a: *mut i32, b: int32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -65984,7 +65984,7 @@ pub unsafe fn vst1q_lane_s64(a: *mut i64, b: int64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66009,7 +66009,7 @@ pub unsafe fn vst1_lane_u8(a: *mut u8, b: uint8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66034,7 +66034,7 @@ pub unsafe fn vst1q_lane_u8(a: *mut u8, b: uint8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66059,7 +66059,7 @@ pub unsafe fn vst1_lane_u16(a: *mut u16, b: uint16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66084,7 +66084,7 @@ pub unsafe fn vst1q_lane_u16(a: *mut u16, b: uint16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66109,7 +66109,7 @@ pub unsafe fn vst1_lane_u32(a: *mut u32, b: uint32x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66134,7 +66134,7 @@ pub unsafe fn vst1q_lane_u32(a: *mut u32, b: uint32x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66159,7 +66159,7 @@ pub unsafe fn vst1q_lane_u64(a: *mut u64, b: uint64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66184,7 +66184,7 @@ pub unsafe fn vst1_lane_p8(a: *mut p8, b: poly8x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66209,7 +66209,7 @@ pub unsafe fn vst1q_lane_p8(a: *mut p8, b: poly8x16_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66234,7 +66234,7 @@ pub unsafe fn vst1_lane_p16(a: *mut p16, b: poly16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66259,7 +66259,7 @@ pub unsafe fn vst1q_lane_p16(a: *mut p16, b: poly16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66284,7 +66284,7 @@ pub unsafe fn vst1_lane_p64(a: *mut p64, b: poly64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66309,7 +66309,7 @@ pub unsafe fn vst1_lane_s64(a: *mut i64, b: int64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -66334,7 +66334,7 @@ pub unsafe fn vst1_lane_u64(a: *mut u64, b: uint64x1_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -66357,7 +66357,7 @@ pub unsafe fn vst1_p64_x2(a: *mut p64, b: poly64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -66380,7 +66380,7 @@ pub unsafe fn vst1_p64_x3(a: *mut p64, b: poly64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -66403,7 +66403,7 @@ pub unsafe fn vst1_p64_x4(a: *mut p64, b: poly64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -66426,7 +66426,7 @@ pub unsafe fn vst1q_p64_x2(a: *mut p64, b: poly64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -66449,7 +66449,7 @@ pub unsafe fn vst1q_p64_x3(a: *mut p64, b: poly64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -66472,7 +66472,7 @@ pub unsafe fn vst1q_p64_x4(a: *mut p64, b: poly64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66491,7 +66491,7 @@ pub unsafe fn vst1_s8_x2(a: *mut i8, b: int8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66510,7 +66510,7 @@ pub unsafe fn vst1q_s8_x2(a: *mut i8, b: int8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66529,7 +66529,7 @@ pub unsafe fn vst1_s16_x2(a: *mut i16, b: int16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66548,7 +66548,7 @@ pub unsafe fn vst1q_s16_x2(a: *mut i16, b: int16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66567,7 +66567,7 @@ pub unsafe fn vst1_s32_x2(a: *mut i32, b: int32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66586,7 +66586,7 @@ pub unsafe fn vst1q_s32_x2(a: *mut i32, b: int32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66605,7 +66605,7 @@ pub unsafe fn vst1_s64_x2(a: *mut i64, b: int64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66624,7 +66624,7 @@ pub unsafe fn vst1q_s64_x2(a: *mut i64, b: int64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66640,7 +66640,7 @@ pub unsafe fn vst1_s8_x2(a: *mut i8, b: int8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66656,7 +66656,7 @@ pub unsafe fn vst1q_s8_x2(a: *mut i8, b: int8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66672,7 +66672,7 @@ pub unsafe fn vst1_s16_x2(a: *mut i16, b: int16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66688,7 +66688,7 @@ pub unsafe fn vst1q_s16_x2(a: *mut i16, b: int16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66704,7 +66704,7 @@ pub unsafe fn vst1_s32_x2(a: *mut i32, b: int32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66720,7 +66720,7 @@ pub unsafe fn vst1q_s32_x2(a: *mut i32, b: int32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66736,7 +66736,7 @@ pub unsafe fn vst1_s64_x2(a: *mut i64, b: int64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66752,7 +66752,7 @@ pub unsafe fn vst1q_s64_x2(a: *mut i64, b: int64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66771,7 +66771,7 @@ pub unsafe fn vst1_s8_x3(a: *mut i8, b: int8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66790,7 +66790,7 @@ pub unsafe fn vst1q_s8_x3(a: *mut i8, b: int8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66809,7 +66809,7 @@ pub unsafe fn vst1_s16_x3(a: *mut i16, b: int16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66828,7 +66828,7 @@ pub unsafe fn vst1q_s16_x3(a: *mut i16, b: int16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66847,7 +66847,7 @@ pub unsafe fn vst1_s32_x3(a: *mut i32, b: int32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66866,7 +66866,7 @@ pub unsafe fn vst1q_s32_x3(a: *mut i32, b: int32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66885,7 +66885,7 @@ pub unsafe fn vst1_s64_x3(a: *mut i64, b: int64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -66904,7 +66904,7 @@ pub unsafe fn vst1q_s64_x3(a: *mut i64, b: int64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66920,7 +66920,7 @@ pub unsafe fn vst1_s8_x3(a: *mut i8, b: int8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66936,7 +66936,7 @@ pub unsafe fn vst1q_s8_x3(a: *mut i8, b: int8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66952,7 +66952,7 @@ pub unsafe fn vst1_s16_x3(a: *mut i16, b: int16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66968,7 +66968,7 @@ pub unsafe fn vst1q_s16_x3(a: *mut i16, b: int16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -66984,7 +66984,7 @@ pub unsafe fn vst1_s32_x3(a: *mut i32, b: int32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67000,7 +67000,7 @@ pub unsafe fn vst1q_s32_x3(a: *mut i32, b: int32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67016,7 +67016,7 @@ pub unsafe fn vst1_s64_x3(a: *mut i64, b: int64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67032,7 +67032,7 @@ pub unsafe fn vst1q_s64_x3(a: *mut i64, b: int64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67051,7 +67051,7 @@ pub unsafe fn vst1_s8_x4(a: *mut i8, b: int8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67070,7 +67070,7 @@ pub unsafe fn vst1q_s8_x4(a: *mut i8, b: int8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67089,7 +67089,7 @@ pub unsafe fn vst1_s16_x4(a: *mut i16, b: int16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67108,7 +67108,7 @@ pub unsafe fn vst1q_s16_x4(a: *mut i16, b: int16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67127,7 +67127,7 @@ pub unsafe fn vst1_s32_x4(a: *mut i32, b: int32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67146,7 +67146,7 @@ pub unsafe fn vst1q_s32_x4(a: *mut i32, b: int32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67165,7 +67165,7 @@ pub unsafe fn vst1_s64_x4(a: *mut i64, b: int64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -67184,7 +67184,7 @@ pub unsafe fn vst1q_s64_x4(a: *mut i64, b: int64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67200,7 +67200,7 @@ pub unsafe fn vst1_s8_x4(a: *mut i8, b: int8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67216,7 +67216,7 @@ pub unsafe fn vst1q_s8_x4(a: *mut i8, b: int8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67232,7 +67232,7 @@ pub unsafe fn vst1_s16_x4(a: *mut i16, b: int16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67248,7 +67248,7 @@ pub unsafe fn vst1q_s16_x4(a: *mut i16, b: int16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67264,7 +67264,7 @@ pub unsafe fn vst1_s32_x4(a: *mut i32, b: int32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67280,7 +67280,7 @@ pub unsafe fn vst1q_s32_x4(a: *mut i32, b: int32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_s64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67296,7 +67296,7 @@ pub unsafe fn vst1_s64_x4(a: *mut i64, b: int64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_s64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -67312,7 +67312,7 @@ pub unsafe fn vst1q_s64_x4(a: *mut i64, b: int64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67335,7 +67335,7 @@ pub unsafe fn vst1_u8_x2(a: *mut u8, b: uint8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67358,7 +67358,7 @@ pub unsafe fn vst1_u8_x3(a: *mut u8, b: uint8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67381,7 +67381,7 @@ pub unsafe fn vst1_u8_x4(a: *mut u8, b: uint8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67404,7 +67404,7 @@ pub unsafe fn vst1q_u8_x2(a: *mut u8, b: uint8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67427,7 +67427,7 @@ pub unsafe fn vst1q_u8_x3(a: *mut u8, b: uint8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67450,7 +67450,7 @@ pub unsafe fn vst1q_u8_x4(a: *mut u8, b: uint8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67473,7 +67473,7 @@ pub unsafe fn vst1_u16_x2(a: *mut u16, b: uint16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67496,7 +67496,7 @@ pub unsafe fn vst1_u16_x3(a: *mut u16, b: uint16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67519,7 +67519,7 @@ pub unsafe fn vst1_u16_x4(a: *mut u16, b: uint16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67542,7 +67542,7 @@ pub unsafe fn vst1q_u16_x2(a: *mut u16, b: uint16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67565,7 +67565,7 @@ pub unsafe fn vst1q_u16_x3(a: *mut u16, b: uint16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67588,7 +67588,7 @@ pub unsafe fn vst1q_u16_x4(a: *mut u16, b: uint16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67611,7 +67611,7 @@ pub unsafe fn vst1_u32_x2(a: *mut u32, b: uint32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67634,7 +67634,7 @@ pub unsafe fn vst1_u32_x3(a: *mut u32, b: uint32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67657,7 +67657,7 @@ pub unsafe fn vst1_u32_x4(a: *mut u32, b: uint32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67680,7 +67680,7 @@ pub unsafe fn vst1q_u32_x2(a: *mut u32, b: uint32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67703,7 +67703,7 @@ pub unsafe fn vst1q_u32_x3(a: *mut u32, b: uint32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u32_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67726,7 +67726,7 @@ pub unsafe fn vst1q_u32_x4(a: *mut u32, b: uint32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67749,7 +67749,7 @@ pub unsafe fn vst1_u64_x2(a: *mut u64, b: uint64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67772,7 +67772,7 @@ pub unsafe fn vst1_u64_x3(a: *mut u64, b: uint64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_u64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67795,7 +67795,7 @@ pub unsafe fn vst1_u64_x4(a: *mut u64, b: uint64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67818,7 +67818,7 @@ pub unsafe fn vst1q_u64_x2(a: *mut u64, b: uint64x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67841,7 +67841,7 @@ pub unsafe fn vst1q_u64_x3(a: *mut u64, b: uint64x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_u64_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67864,7 +67864,7 @@ pub unsafe fn vst1q_u64_x4(a: *mut u64, b: uint64x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67887,7 +67887,7 @@ pub unsafe fn vst1_p8_x2(a: *mut p8, b: poly8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67910,7 +67910,7 @@ pub unsafe fn vst1_p8_x3(a: *mut p8, b: poly8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67933,7 +67933,7 @@ pub unsafe fn vst1_p8_x4(a: *mut p8, b: poly8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67956,7 +67956,7 @@ pub unsafe fn vst1q_p8_x2(a: *mut p8, b: poly8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -67979,7 +67979,7 @@ pub unsafe fn vst1q_p8_x3(a: *mut p8, b: poly8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p8_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68002,7 +68002,7 @@ pub unsafe fn vst1q_p8_x4(a: *mut p8, b: poly8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68025,7 +68025,7 @@ pub unsafe fn vst1_p16_x2(a: *mut p16, b: poly16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68048,7 +68048,7 @@ pub unsafe fn vst1_p16_x3(a: *mut p16, b: poly16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68071,7 +68071,7 @@ pub unsafe fn vst1_p16_x4(a: *mut p16, b: poly16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16_x2)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68094,7 +68094,7 @@ pub unsafe fn vst1q_p16_x2(a: *mut p16, b: poly16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16_x3)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68117,7 +68117,7 @@ pub unsafe fn vst1q_p16_x3(a: *mut p16, b: poly16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_p16_x4)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst1))]
@@ -68136,7 +68136,7 @@ pub unsafe fn vst1q_p16_x3(a: *mut p16, b: poly16x8x3_t) {
 pub unsafe fn vst1q_p16_x4(a: *mut p16, b: poly16x8x4_t) {
     vst1q_s16_x4(transmute(a), transmute(b))
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68150,7 +68150,7 @@ unsafe fn vst1_v1i64(addr: *const i8, val: int64x1_t) {
     }
     _vst1_v1i64(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68164,7 +68164,7 @@ unsafe fn vst1_v2f32(addr: *const i8, val: float32x2_t) {
     }
     _vst1_v2f32(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68178,7 +68178,7 @@ unsafe fn vst1_v2i32(addr: *const i8, val: int32x2_t) {
     }
     _vst1_v2i32(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68192,7 +68192,7 @@ unsafe fn vst1_v4i16(addr: *const i8, val: int16x4_t) {
     }
     _vst1_v4i16(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68206,7 +68206,7 @@ unsafe fn vst1_v8i8(addr: *const i8, val: int8x8_t) {
     }
     _vst1_v8i8(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68220,7 +68220,7 @@ unsafe fn vst1q_v16i8(addr: *const i8, val: int8x16_t) {
     }
     _vst1q_v16i8(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68234,7 +68234,7 @@ unsafe fn vst1q_v2i64(addr: *const i8, val: int64x2_t) {
     }
     _vst1q_v2i64(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68248,7 +68248,7 @@ unsafe fn vst1q_v4f32(addr: *const i8, val: float32x4_t) {
     }
     _vst1q_v4f32(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68262,7 +68262,7 @@ unsafe fn vst1q_v4i32(addr: *const i8, val: int32x4_t) {
     }
     _vst1q_v4i32(addr, val, ALIGN)
 }
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68280,7 +68280,7 @@ unsafe fn vst1q_v8i16(addr: *const i8, val: int16x8_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_v4f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[target_feature(enable = "neon,fp16")]
@@ -68298,7 +68298,7 @@ unsafe fn vst1_v4f16(addr: *const i8, val: float16x4_t, align: i32) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_v8f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[target_feature(enable = "neon,fp16")]
@@ -68316,7 +68316,7 @@ unsafe fn vst1q_v8f16(addr: *const i8, val: float16x8_t, align: i32) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))]
@@ -68341,7 +68341,7 @@ pub unsafe fn vst1q_lane_p64(a: *mut p64, b: poly64x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -68362,7 +68362,7 @@ pub unsafe fn vst2_f16(a: *mut f16, b: float16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -68383,7 +68383,7 @@ pub unsafe fn vst2q_f16(a: *mut f16, b: float16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68402,7 +68402,7 @@ pub unsafe fn vst2_f16(a: *mut f16, b: float16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68421,7 +68421,7 @@ pub unsafe fn vst2q_f16(a: *mut f16, b: float16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68440,7 +68440,7 @@ pub unsafe fn vst2_f32(a: *mut f32, b: float32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68459,7 +68459,7 @@ pub unsafe fn vst2q_f32(a: *mut f32, b: float32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68478,7 +68478,7 @@ pub unsafe fn vst2_s8(a: *mut i8, b: int8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68497,7 +68497,7 @@ pub unsafe fn vst2q_s8(a: *mut i8, b: int8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68516,7 +68516,7 @@ pub unsafe fn vst2_s16(a: *mut i16, b: int16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68535,7 +68535,7 @@ pub unsafe fn vst2q_s16(a: *mut i16, b: int16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68554,7 +68554,7 @@ pub unsafe fn vst2_s32(a: *mut i32, b: int32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -68573,7 +68573,7 @@ pub unsafe fn vst2q_s32(a: *mut i32, b: int32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68589,7 +68589,7 @@ pub unsafe fn vst2_f32(a: *mut f32, b: float32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68605,7 +68605,7 @@ pub unsafe fn vst2q_f32(a: *mut f32, b: float32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68621,7 +68621,7 @@ pub unsafe fn vst2_s8(a: *mut i8, b: int8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68637,7 +68637,7 @@ pub unsafe fn vst2q_s8(a: *mut i8, b: int8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68653,7 +68653,7 @@ pub unsafe fn vst2_s16(a: *mut i16, b: int16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68669,7 +68669,7 @@ pub unsafe fn vst2q_s16(a: *mut i16, b: int16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68685,7 +68685,7 @@ pub unsafe fn vst2_s32(a: *mut i32, b: int32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -68701,7 +68701,7 @@ pub unsafe fn vst2q_s32(a: *mut i32, b: int32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68724,7 +68724,7 @@ pub unsafe fn vst2_lane_f16(a: *mut f16, b: float16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68747,7 +68747,7 @@ pub unsafe fn vst2q_lane_f16(a: *mut f16, b: float16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68768,7 +68768,7 @@ pub unsafe fn vst2_lane_f16(a: *mut f16, b: float16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -68789,7 +68789,7 @@ pub unsafe fn vst2q_lane_f16(a: *mut f16, b: float16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68810,7 +68810,7 @@ pub unsafe fn vst2_lane_f32(a: *mut f32, b: float32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68831,7 +68831,7 @@ pub unsafe fn vst2q_lane_f32(a: *mut f32, b: float32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68852,7 +68852,7 @@ pub unsafe fn vst2_lane_s8(a: *mut i8, b: int8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68873,7 +68873,7 @@ pub unsafe fn vst2_lane_s16(a: *mut i16, b: int16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68894,7 +68894,7 @@ pub unsafe fn vst2q_lane_s16(a: *mut i16, b: int16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68915,7 +68915,7 @@ pub unsafe fn vst2_lane_s32(a: *mut i32, b: int32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -68936,7 +68936,7 @@ pub unsafe fn vst2q_lane_s32(a: *mut i32, b: int32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -68954,7 +68954,7 @@ pub unsafe fn vst2_lane_f32(a: *mut f32, b: float32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -68972,7 +68972,7 @@ pub unsafe fn vst2q_lane_f32(a: *mut f32, b: float32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -68990,7 +68990,7 @@ pub unsafe fn vst2_lane_s8(a: *mut i8, b: int8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -69008,7 +69008,7 @@ pub unsafe fn vst2_lane_s16(a: *mut i16, b: int16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -69026,7 +69026,7 @@ pub unsafe fn vst2q_lane_s16(a: *mut i16, b: int16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -69044,7 +69044,7 @@ pub unsafe fn vst2_lane_s32(a: *mut i32, b: int32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst2, LANE = 0))]
@@ -69062,7 +69062,7 @@ pub unsafe fn vst2q_lane_s32(a: *mut i32, b: int32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69087,7 +69087,7 @@ pub unsafe fn vst2_lane_u8(a: *mut u8, b: uint8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69112,7 +69112,7 @@ pub unsafe fn vst2_lane_u16(a: *mut u16, b: uint16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69137,7 +69137,7 @@ pub unsafe fn vst2q_lane_u16(a: *mut u16, b: uint16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69162,7 +69162,7 @@ pub unsafe fn vst2_lane_u32(a: *mut u32, b: uint32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69187,7 +69187,7 @@ pub unsafe fn vst2q_lane_u32(a: *mut u32, b: uint32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69212,7 +69212,7 @@ pub unsafe fn vst2_lane_p8(a: *mut p8, b: poly8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69237,7 +69237,7 @@ pub unsafe fn vst2_lane_p16(a: *mut p16, b: poly16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2, LANE = 0))]
@@ -69262,7 +69262,7 @@ pub unsafe fn vst2q_lane_p16(a: *mut p16, b: poly16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -69285,7 +69285,7 @@ pub unsafe fn vst2_p64(a: *mut p64, b: poly64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69301,7 +69301,7 @@ pub unsafe fn vst2_s64(a: *mut i64, b: int64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69320,7 +69320,7 @@ pub unsafe fn vst2_s64(a: *mut i64, b: int64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -69343,7 +69343,7 @@ pub unsafe fn vst2_u64(a: *mut u64, b: uint64x1x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69366,7 +69366,7 @@ pub unsafe fn vst2_u8(a: *mut u8, b: uint8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69389,7 +69389,7 @@ pub unsafe fn vst2q_u8(a: *mut u8, b: uint8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69412,7 +69412,7 @@ pub unsafe fn vst2_u16(a: *mut u16, b: uint16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69435,7 +69435,7 @@ pub unsafe fn vst2q_u16(a: *mut u16, b: uint16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69458,7 +69458,7 @@ pub unsafe fn vst2_u32(a: *mut u32, b: uint32x2x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69481,7 +69481,7 @@ pub unsafe fn vst2q_u32(a: *mut u32, b: uint32x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69504,7 +69504,7 @@ pub unsafe fn vst2_p8(a: *mut p8, b: poly8x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69527,7 +69527,7 @@ pub unsafe fn vst2q_p8(a: *mut p8, b: poly8x16x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69550,7 +69550,7 @@ pub unsafe fn vst2_p16(a: *mut p16, b: poly16x4x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst2))]
@@ -69573,7 +69573,7 @@ pub unsafe fn vst2q_p16(a: *mut p16, b: poly16x8x2_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -69592,7 +69592,7 @@ pub unsafe fn vst3_f16(a: *mut f16, b: float16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -69611,7 +69611,7 @@ pub unsafe fn vst3q_f16(a: *mut f16, b: float16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -69632,7 +69632,7 @@ pub unsafe fn vst3_f16(a: *mut f16, b: float16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -69653,7 +69653,7 @@ pub unsafe fn vst3q_f16(a: *mut f16, b: float16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69669,7 +69669,7 @@ pub unsafe fn vst3_f32(a: *mut f32, b: float32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69685,7 +69685,7 @@ pub unsafe fn vst3q_f32(a: *mut f32, b: float32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69701,7 +69701,7 @@ pub unsafe fn vst3_s8(a: *mut i8, b: int8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69717,7 +69717,7 @@ pub unsafe fn vst3q_s8(a: *mut i8, b: int8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69733,7 +69733,7 @@ pub unsafe fn vst3_s16(a: *mut i16, b: int16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69749,7 +69749,7 @@ pub unsafe fn vst3q_s16(a: *mut i16, b: int16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69765,7 +69765,7 @@ pub unsafe fn vst3_s32(a: *mut i32, b: int32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -69781,7 +69781,7 @@ pub unsafe fn vst3q_s32(a: *mut i32, b: int32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69800,7 +69800,7 @@ pub unsafe fn vst3_f32(a: *mut f32, b: float32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69819,7 +69819,7 @@ pub unsafe fn vst3q_f32(a: *mut f32, b: float32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69838,7 +69838,7 @@ pub unsafe fn vst3_s8(a: *mut i8, b: int8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69857,7 +69857,7 @@ pub unsafe fn vst3q_s8(a: *mut i8, b: int8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69876,7 +69876,7 @@ pub unsafe fn vst3_s16(a: *mut i16, b: int16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69895,7 +69895,7 @@ pub unsafe fn vst3q_s16(a: *mut i16, b: int16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69914,7 +69914,7 @@ pub unsafe fn vst3_s32(a: *mut i32, b: int32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -69933,7 +69933,7 @@ pub unsafe fn vst3q_s32(a: *mut i32, b: int32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -69961,7 +69961,7 @@ pub unsafe fn vst3_lane_f16(a: *mut f16, b: float16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -69989,7 +69989,7 @@ pub unsafe fn vst3q_lane_f16(a: *mut f16, b: float16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70012,7 +70012,7 @@ pub unsafe fn vst3_lane_f16(a: *mut f16, b: float16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70035,7 +70035,7 @@ pub unsafe fn vst3q_lane_f16(a: *mut f16, b: float16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70060,7 +70060,7 @@ pub unsafe fn vst3_lane_f32(a: *mut f32, b: float32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70085,7 +70085,7 @@ pub unsafe fn vst3q_lane_f32(a: *mut f32, b: float32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70103,7 +70103,7 @@ pub unsafe fn vst3_lane_s8(a: *mut i8, b: int8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70128,7 +70128,7 @@ pub unsafe fn vst3_lane_s16(a: *mut i16, b: int16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70153,7 +70153,7 @@ pub unsafe fn vst3q_lane_s16(a: *mut i16, b: int16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70178,7 +70178,7 @@ pub unsafe fn vst3_lane_s32(a: *mut i32, b: int32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst3, LANE = 0))]
@@ -70203,7 +70203,7 @@ pub unsafe fn vst3q_lane_s32(a: *mut i32, b: int32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70224,7 +70224,7 @@ pub unsafe fn vst3_lane_f32(a: *mut f32, b: float32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70245,7 +70245,7 @@ pub unsafe fn vst3q_lane_f32(a: *mut f32, b: float32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70266,7 +70266,7 @@ pub unsafe fn vst3_lane_s8(a: *mut i8, b: int8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70287,7 +70287,7 @@ pub unsafe fn vst3_lane_s16(a: *mut i16, b: int16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70308,7 +70308,7 @@ pub unsafe fn vst3q_lane_s16(a: *mut i16, b: int16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70329,7 +70329,7 @@ pub unsafe fn vst3_lane_s32(a: *mut i32, b: int32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -70350,7 +70350,7 @@ pub unsafe fn vst3q_lane_s32(a: *mut i32, b: int32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70375,7 +70375,7 @@ pub unsafe fn vst3_lane_u8(a: *mut u8, b: uint8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70400,7 +70400,7 @@ pub unsafe fn vst3_lane_u16(a: *mut u16, b: uint16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70425,7 +70425,7 @@ pub unsafe fn vst3q_lane_u16(a: *mut u16, b: uint16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70450,7 +70450,7 @@ pub unsafe fn vst3_lane_u32(a: *mut u32, b: uint32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70475,7 +70475,7 @@ pub unsafe fn vst3q_lane_u32(a: *mut u32, b: uint32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70500,7 +70500,7 @@ pub unsafe fn vst3_lane_p8(a: *mut p8, b: poly8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70525,7 +70525,7 @@ pub unsafe fn vst3_lane_p16(a: *mut p16, b: poly16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3, LANE = 0))]
@@ -70550,7 +70550,7 @@ pub unsafe fn vst3q_lane_p16(a: *mut p16, b: poly16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -70573,7 +70573,7 @@ pub unsafe fn vst3_p64(a: *mut p64, b: poly64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -70592,7 +70592,7 @@ pub unsafe fn vst3_s64(a: *mut i64, b: int64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -70608,7 +70608,7 @@ pub unsafe fn vst3_s64(a: *mut i64, b: int64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -70631,7 +70631,7 @@ pub unsafe fn vst3_u64(a: *mut u64, b: uint64x1x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70654,7 +70654,7 @@ pub unsafe fn vst3_u8(a: *mut u8, b: uint8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70677,7 +70677,7 @@ pub unsafe fn vst3q_u8(a: *mut u8, b: uint8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70700,7 +70700,7 @@ pub unsafe fn vst3_u16(a: *mut u16, b: uint16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70723,7 +70723,7 @@ pub unsafe fn vst3q_u16(a: *mut u16, b: uint16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70746,7 +70746,7 @@ pub unsafe fn vst3_u32(a: *mut u32, b: uint32x2x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70769,7 +70769,7 @@ pub unsafe fn vst3q_u32(a: *mut u32, b: uint32x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70792,7 +70792,7 @@ pub unsafe fn vst3_p8(a: *mut p8, b: poly8x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70815,7 +70815,7 @@ pub unsafe fn vst3q_p8(a: *mut p8, b: poly8x16x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70838,7 +70838,7 @@ pub unsafe fn vst3_p16(a: *mut p16, b: poly16x4x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst3q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst3))]
@@ -70861,7 +70861,7 @@ pub unsafe fn vst3q_p16(a: *mut p16, b: poly16x8x3_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -70887,7 +70887,7 @@ pub unsafe fn vst4_f16(a: *mut f16, b: float16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -70913,7 +70913,7 @@ pub unsafe fn vst4q_f16(a: *mut f16, b: float16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -70934,7 +70934,7 @@ pub unsafe fn vst4_f16(a: *mut f16, b: float16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "fp16"))]
@@ -70955,7 +70955,7 @@ pub unsafe fn vst4q_f16(a: *mut f16, b: float16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -70978,7 +70978,7 @@ pub unsafe fn vst4_f32(a: *mut f32, b: float32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71001,7 +71001,7 @@ pub unsafe fn vst4q_f32(a: *mut f32, b: float32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71017,7 +71017,7 @@ pub unsafe fn vst4_s8(a: *mut i8, b: int8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71040,7 +71040,7 @@ pub unsafe fn vst4q_s8(a: *mut i8, b: int8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71063,7 +71063,7 @@ pub unsafe fn vst4_s16(a: *mut i16, b: int16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71086,7 +71086,7 @@ pub unsafe fn vst4q_s16(a: *mut i16, b: int16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71109,7 +71109,7 @@ pub unsafe fn vst4_s32(a: *mut i32, b: int32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -71132,7 +71132,7 @@ pub unsafe fn vst4q_s32(a: *mut i32, b: int32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71151,7 +71151,7 @@ pub unsafe fn vst4_f32(a: *mut f32, b: float32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71170,7 +71170,7 @@ pub unsafe fn vst4q_f32(a: *mut f32, b: float32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71189,7 +71189,7 @@ pub unsafe fn vst4_s8(a: *mut i8, b: int8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71208,7 +71208,7 @@ pub unsafe fn vst4q_s8(a: *mut i8, b: int8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71227,7 +71227,7 @@ pub unsafe fn vst4_s16(a: *mut i16, b: int16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71246,7 +71246,7 @@ pub unsafe fn vst4q_s16(a: *mut i16, b: int16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71265,7 +71265,7 @@ pub unsafe fn vst4_s32(a: *mut i32, b: int32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -71284,7 +71284,7 @@ pub unsafe fn vst4q_s32(a: *mut i32, b: int32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -71313,7 +71313,7 @@ pub unsafe fn vst4_lane_f16(a: *mut f16, b: float16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -71342,7 +71342,7 @@ pub unsafe fn vst4q_lane_f16(a: *mut f16, b: float16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71372,7 +71372,7 @@ pub unsafe fn vst4_lane_f16(a: *mut f16, b: float16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71402,7 +71402,7 @@ pub unsafe fn vst4q_lane_f16(a: *mut f16, b: float16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71428,7 +71428,7 @@ pub unsafe fn vst4_lane_f32(a: *mut f32, b: float32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71454,7 +71454,7 @@ pub unsafe fn vst4q_lane_f32(a: *mut f32, b: float32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71480,7 +71480,7 @@ pub unsafe fn vst4_lane_s8(a: *mut i8, b: int8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71506,7 +71506,7 @@ pub unsafe fn vst4_lane_s16(a: *mut i16, b: int16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71532,7 +71532,7 @@ pub unsafe fn vst4q_lane_s16(a: *mut i16, b: int16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71558,7 +71558,7 @@ pub unsafe fn vst4_lane_s32(a: *mut i32, b: int32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[cfg_attr(test, assert_instr(vst4, LANE = 0))]
@@ -71584,7 +71584,7 @@ pub unsafe fn vst4q_lane_s32(a: *mut i32, b: int32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71612,7 +71612,7 @@ pub unsafe fn vst4_lane_f32(a: *mut f32, b: float32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_f32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71640,7 +71640,7 @@ pub unsafe fn vst4q_lane_f32(a: *mut f32, b: float32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71661,7 +71661,7 @@ pub unsafe fn vst4_lane_s8(a: *mut i8, b: int8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71689,7 +71689,7 @@ pub unsafe fn vst4_lane_s16(a: *mut i16, b: int16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71717,7 +71717,7 @@ pub unsafe fn vst4q_lane_s16(a: *mut i16, b: int16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71745,7 +71745,7 @@ pub unsafe fn vst4_lane_s32(a: *mut i32, b: int32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_s32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[rustc_legacy_const_generics(2)]
@@ -71773,7 +71773,7 @@ pub unsafe fn vst4q_lane_s32(a: *mut i32, b: int32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71798,7 +71798,7 @@ pub unsafe fn vst4_lane_u8(a: *mut u8, b: uint8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71823,7 +71823,7 @@ pub unsafe fn vst4_lane_u16(a: *mut u16, b: uint16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71848,7 +71848,7 @@ pub unsafe fn vst4q_lane_u16(a: *mut u16, b: uint16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71873,7 +71873,7 @@ pub unsafe fn vst4_lane_u32(a: *mut u32, b: uint32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71898,7 +71898,7 @@ pub unsafe fn vst4q_lane_u32(a: *mut u32, b: uint32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71923,7 +71923,7 @@ pub unsafe fn vst4_lane_p8(a: *mut p8, b: poly8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71948,7 +71948,7 @@ pub unsafe fn vst4_lane_p16(a: *mut p16, b: poly16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_lane_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4, LANE = 0))]
@@ -71973,7 +71973,7 @@ pub unsafe fn vst4q_lane_p16(a: *mut p16, b: poly16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_p64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[target_feature(enable = "neon,aes")]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -71996,7 +71996,7 @@ pub unsafe fn vst4_p64(a: *mut p64, b: poly64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[cfg(target_arch = "arm")]
 #[target_feature(enable = "neon,v7")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -72019,7 +72019,7 @@ pub unsafe fn vst4_s64(a: *mut i64, b: int64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_s64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(not(target_arch = "arm"))]
 #[stable(feature = "neon_intrinsics", since = "1.59.0")]
@@ -72038,7 +72038,7 @@ pub unsafe fn vst4_s64(a: *mut i64, b: int64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u64)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -72061,7 +72061,7 @@ pub unsafe fn vst4_u64(a: *mut u64, b: uint64x1x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72084,7 +72084,7 @@ pub unsafe fn vst4_u8(a: *mut u8, b: uint8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72107,7 +72107,7 @@ pub unsafe fn vst4q_u8(a: *mut u8, b: uint8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72130,7 +72130,7 @@ pub unsafe fn vst4_u16(a: *mut u16, b: uint16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72153,7 +72153,7 @@ pub unsafe fn vst4q_u16(a: *mut u16, b: uint16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72176,7 +72176,7 @@ pub unsafe fn vst4_u32(a: *mut u32, b: uint32x2x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_u32)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72199,7 +72199,7 @@ pub unsafe fn vst4q_u32(a: *mut u32, b: uint32x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72222,7 +72222,7 @@ pub unsafe fn vst4_p8(a: *mut p8, b: poly8x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_p8)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72245,7 +72245,7 @@ pub unsafe fn vst4q_p8(a: *mut p8, b: poly8x16x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72268,7 +72268,7 @@ pub unsafe fn vst4_p16(a: *mut p16, b: poly16x4x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst4q_p16)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vst4))]
@@ -72291,7 +72291,7 @@ pub unsafe fn vst4q_p16(a: *mut p16, b: poly16x8x4_t) {
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vstrq_p128)"]
 #[doc = "## Safety"]
 #[doc = "  * Neon instrinsic unsafe"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -72312,7 +72312,7 @@ pub unsafe fn vstrq_p128(a: *mut p128, b: p128) {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f16"))]
 #[cfg_attr(
@@ -72334,7 +72334,7 @@ pub fn vsub_f16(a: float16x4_t, b: float16x4_t) -> float16x4_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f16"))]
 #[cfg_attr(
@@ -72356,7 +72356,7 @@ pub fn vsubq_f16(a: float16x8_t, b: float16x8_t) -> float16x8_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))]
@@ -72377,7 +72377,7 @@ pub fn vsub_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))]
@@ -72398,7 +72398,7 @@ pub fn vsubq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))]
@@ -72419,7 +72419,7 @@ pub fn vsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))]
@@ -72440,7 +72440,7 @@ pub fn vsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))]
@@ -72461,7 +72461,7 @@ pub fn vsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))]
@@ -72482,7 +72482,7 @@ pub fn vsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))]
@@ -72503,7 +72503,7 @@ pub fn vsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))]
@@ -72524,7 +72524,7 @@ pub fn vsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))]
@@ -72545,7 +72545,7 @@ pub fn vsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))]
@@ -72566,7 +72566,7 @@ pub fn vsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))]
@@ -72587,7 +72587,7 @@ pub fn vsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))]
@@ -72608,7 +72608,7 @@ pub fn vsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))]
@@ -72629,7 +72629,7 @@ pub fn vsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))]
@@ -72650,7 +72650,7 @@ pub fn vsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))]
@@ -72671,7 +72671,7 @@ pub fn vsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))]
@@ -72692,7 +72692,7 @@ pub fn vsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsub_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))]
@@ -72713,7 +72713,7 @@ pub fn vsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Subtract"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))]
@@ -72734,7 +72734,7 @@ pub fn vsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72756,7 +72756,7 @@ pub fn vsubhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72778,7 +72778,7 @@ pub fn vsubhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72800,7 +72800,7 @@ pub fn vsubhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72822,7 +72822,7 @@ pub fn vsubhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_t
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72844,7 +72844,7 @@ pub fn vsubhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8_
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_high_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72866,7 +72866,7 @@ pub fn vsubhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4_
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72888,7 +72888,7 @@ pub fn vsubhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72910,7 +72910,7 @@ pub fn vsubhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_s64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72932,7 +72932,7 @@ pub fn vsubhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72954,7 +72954,7 @@ pub fn vsubhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72976,7 +72976,7 @@ pub fn vsubhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t {
 }
 #[doc = "Subtract returning high narrow"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubhn_u64)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))]
@@ -72998,7 +72998,7 @@ pub fn vsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t {
 }
 #[doc = "Signed Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))]
@@ -73023,7 +73023,7 @@ pub fn vsubl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))]
@@ -73048,7 +73048,7 @@ pub fn vsubl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))]
@@ -73073,7 +73073,7 @@ pub fn vsubl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))]
@@ -73098,7 +73098,7 @@ pub fn vsubl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))]
@@ -73123,7 +73123,7 @@ pub fn vsubl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Subtract Long"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubl_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))]
@@ -73148,7 +73148,7 @@ pub fn vsubl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Signed Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))]
@@ -73169,7 +73169,7 @@ pub fn vsubw_s8(a: int16x8_t, b: int8x8_t) -> int16x8_t {
 }
 #[doc = "Signed Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))]
@@ -73190,7 +73190,7 @@ pub fn vsubw_s16(a: int32x4_t, b: int16x4_t) -> int32x4_t {
 }
 #[doc = "Signed Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))]
@@ -73211,7 +73211,7 @@ pub fn vsubw_s32(a: int64x2_t, b: int32x2_t) -> int64x2_t {
 }
 #[doc = "Unsigned Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))]
@@ -73232,7 +73232,7 @@ pub fn vsubw_u8(a: uint16x8_t, b: uint8x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))]
@@ -73253,7 +73253,7 @@ pub fn vsubw_u16(a: uint32x4_t, b: uint16x4_t) -> uint32x4_t {
 }
 #[doc = "Unsigned Subtract Wide"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsubw_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))]
@@ -73274,7 +73274,7 @@ pub fn vsubw_u32(a: uint64x2_t, b: uint32x2_t) -> uint64x2_t {
 }
 #[doc = "Dot product index form with signed and unsigned integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsudot_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -73302,7 +73302,7 @@ pub fn vsudot_lane_s32(a: int32x2_t, b: int8x8_t, c: uint8x8_t)
 }
 #[doc = "Dot product index form with signed and unsigned integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsudot_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -73334,7 +73334,7 @@ pub fn vsudot_lane_s32(a: int32x2_t, b: int8x8_t, c: uint8x8_t)
 }
 #[doc = "Dot product index form with signed and unsigned integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsudotq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -73363,7 +73363,7 @@ pub fn vsudotq_lane_s32(a: int32x4_t, b: int8x16_t, c: uint8x8_
 }
 #[doc = "Dot product index form with signed and unsigned integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsudotq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -73397,7 +73397,7 @@ pub fn vsudotq_lane_s32(a: int32x4_t, b: int8x16_t, c: uint8x8_
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73412,7 +73412,7 @@ fn vtbl1(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73423,7 +73423,7 @@ pub fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73435,7 +73435,7 @@ pub fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73452,7 +73452,7 @@ pub fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73464,7 +73464,7 @@ pub fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl1_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73481,7 +73481,7 @@ pub fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73496,7 +73496,7 @@ fn vtbl2(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73507,7 +73507,7 @@ pub fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73519,7 +73519,7 @@ pub fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73538,7 +73538,7 @@ pub fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73550,7 +73550,7 @@ pub fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73569,7 +73569,7 @@ pub fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73584,7 +73584,7 @@ fn vtbl3(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73595,7 +73595,7 @@ pub fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73614,7 +73614,7 @@ pub fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73639,7 +73639,7 @@ pub fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73658,7 +73658,7 @@ pub fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73683,7 +73683,7 @@ pub fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73698,7 +73698,7 @@ fn vtbl4(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, e: int8x8_t) -> int
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
@@ -73709,7 +73709,7 @@ pub fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73729,7 +73729,7 @@ pub fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73756,7 +73756,7 @@ pub fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73776,7 +73776,7 @@ pub fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbl4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon")]
 #[cfg(target_arch = "arm")]
@@ -73803,7 +73803,7 @@ pub fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -73817,7 +73817,7 @@ fn vtbx1(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -73827,7 +73827,7 @@ pub fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73838,7 +73838,7 @@ pub fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73855,7 +73855,7 @@ pub fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73866,7 +73866,7 @@ pub fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx1_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73883,7 +73883,7 @@ pub fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -73897,7 +73897,7 @@ fn vtbx2(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -73907,7 +73907,7 @@ pub fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73925,7 +73925,7 @@ pub fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73949,7 +73949,7 @@ pub fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73967,7 +73967,7 @@ pub fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx2_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -73991,7 +73991,7 @@ pub fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -74005,7 +74005,7 @@ fn vtbx3(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, e: int8x8_t) -> int
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -74015,7 +74015,7 @@ pub fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74034,7 +74034,7 @@ pub fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74060,7 +74060,7 @@ pub fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74079,7 +74079,7 @@ pub fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx3_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74105,7 +74105,7 @@ pub fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
 #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")]
@@ -74126,7 +74126,7 @@ fn vtbx4(a: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t, e: int8x8_t, f: int
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74146,7 +74146,7 @@ pub fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_s8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74174,7 +74174,7 @@ pub fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74194,7 +74194,7 @@ pub fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_u8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74222,7 +74222,7 @@ pub fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74242,7 +74242,7 @@ pub fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Extended table look-up"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtbx4_p8)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,v7")]
 #[cfg(target_arch = "arm")]
@@ -74270,7 +74270,7 @@ pub fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
 #[cfg_attr(
@@ -74300,7 +74300,7 @@ pub fn vtrn_f16(a: float16x4_t, b: float16x4_t) -> float16x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
 #[cfg_attr(
@@ -74330,7 +74330,7 @@ pub fn vtrnq_f16(a: float16x8_t, b: float16x8_t) -> float16x8x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74359,7 +74359,7 @@ pub fn vtrn_f32(a: float32x2_t, b: float32x2_t) -> float32x2x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74388,7 +74388,7 @@ pub fn vtrn_s32(a: int32x2_t, b: int32x2_t) -> int32x2x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74417,7 +74417,7 @@ pub fn vtrn_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74446,7 +74446,7 @@ pub fn vtrnq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74475,7 +74475,7 @@ pub fn vtrn_s8(a: int8x8_t, b: int8x8_t) -> int8x8x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74512,7 +74512,7 @@ pub fn vtrnq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74541,7 +74541,7 @@ pub fn vtrn_s16(a: int16x4_t, b: int16x4_t) -> int16x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74570,7 +74570,7 @@ pub fn vtrnq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74599,7 +74599,7 @@ pub fn vtrnq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74628,7 +74628,7 @@ pub fn vtrn_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74665,7 +74665,7 @@ pub fn vtrnq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74694,7 +74694,7 @@ pub fn vtrn_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74723,7 +74723,7 @@ pub fn vtrnq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74752,7 +74752,7 @@ pub fn vtrnq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74781,7 +74781,7 @@ pub fn vtrn_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74818,7 +74818,7 @@ pub fn vtrnq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrn_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74847,7 +74847,7 @@ pub fn vtrn_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t {
 }
 #[doc = "Transpose elements"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtrnq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -74876,7 +74876,7 @@ pub fn vtrnq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8x2_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -74901,7 +74901,7 @@ pub fn vtst_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -74926,7 +74926,7 @@ pub fn vtstq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -74951,7 +74951,7 @@ pub fn vtst_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -74976,7 +74976,7 @@ pub fn vtstq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75001,7 +75001,7 @@ pub fn vtst_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75026,7 +75026,7 @@ pub fn vtstq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75051,7 +75051,7 @@ pub fn vtst_p8(a: poly8x8_t, b: poly8x8_t) -> uint8x8_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75076,7 +75076,7 @@ pub fn vtstq_p8(a: poly8x16_t, b: poly8x16_t) -> uint8x16_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75101,7 +75101,7 @@ pub fn vtst_p16(a: poly16x4_t, b: poly16x4_t) -> uint16x4_t {
 }
 #[doc = "Signed compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75126,7 +75126,7 @@ pub fn vtstq_p16(a: poly16x8_t, b: poly16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75151,7 +75151,7 @@ pub fn vtst_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75176,7 +75176,7 @@ pub fn vtstq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75201,7 +75201,7 @@ pub fn vtst_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75226,7 +75226,7 @@ pub fn vtstq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtst_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75251,7 +75251,7 @@ pub fn vtst_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t {
 }
 #[doc = "Unsigned compare bitwise Test bits nonzero"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vtstq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))]
@@ -75276,7 +75276,7 @@ pub fn vtstq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
 }
 #[doc = "Dot product index form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdot_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -75304,7 +75304,7 @@ pub fn vusdot_lane_s32(a: int32x2_t, b: uint8x8_t, c: int8x8_t)
 }
 #[doc = "Dot product index form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdot_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -75336,7 +75336,7 @@ pub fn vusdot_lane_s32(a: int32x2_t, b: uint8x8_t, c: int8x8_t)
 }
 #[doc = "Dot product index form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdotq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "little")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -75365,7 +75365,7 @@ pub fn vusdotq_lane_s32(a: int32x4_t, b: uint8x16_t, c: int8x8_
 }
 #[doc = "Dot product index form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdotq_lane_s32)"]
-#[inline]
+#[inline(always)]
 #[cfg(target_endian = "big")]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
@@ -75399,7 +75399,7 @@ pub fn vusdotq_lane_s32(a: int32x4_t, b: uint8x16_t, c: int8x8_
 }
 #[doc = "Dot product vector form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdot_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vusdot))]
@@ -75428,7 +75428,7 @@ pub fn vusdot_s32(a: int32x2_t, b: uint8x8_t, c: int8x8_t) -> int32x2_t {
 }
 #[doc = "Dot product vector form with unsigned and signed integers"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusdotq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vusdot))]
@@ -75457,7 +75457,7 @@ pub fn vusdotq_s32(a: int32x4_t, b: uint8x16_t, c: int8x16_t) -> int32x4_t {
 }
 #[doc = "Unsigned and signed 8-bit integer matrix multiply-accumulate"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vusmmlaq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon,i8mm")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))]
@@ -75486,7 +75486,7 @@ pub fn vusmmlaq_s32(a: int32x4_t, b: uint8x16_t, c: int8x16_t) -> int32x4_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
 #[cfg_attr(
@@ -75516,7 +75516,7 @@ pub fn vuzp_f16(a: float16x4_t, b: float16x4_t) -> float16x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
 #[cfg_attr(
@@ -75546,7 +75546,7 @@ pub fn vuzpq_f16(a: float16x8_t, b: float16x8_t) -> float16x8x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -75575,7 +75575,7 @@ pub fn vuzp_f32(a: float32x2_t, b: float32x2_t) -> float32x2x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -75604,7 +75604,7 @@ pub fn vuzp_s32(a: int32x2_t, b: int32x2_t) -> int32x2x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -75633,7 +75633,7 @@ pub fn vuzp_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75662,7 +75662,7 @@ pub fn vuzpq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75691,7 +75691,7 @@ pub fn vuzp_s8(a: int8x8_t, b: int8x8_t) -> int8x8x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75728,7 +75728,7 @@ pub fn vuzpq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75757,7 +75757,7 @@ pub fn vuzp_s16(a: int16x4_t, b: int16x4_t) -> int16x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75786,7 +75786,7 @@ pub fn vuzpq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75815,7 +75815,7 @@ pub fn vuzpq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75844,7 +75844,7 @@ pub fn vuzp_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75881,7 +75881,7 @@ pub fn vuzpq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75910,7 +75910,7 @@ pub fn vuzp_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75939,7 +75939,7 @@ pub fn vuzpq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75968,7 +75968,7 @@ pub fn vuzpq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -75997,7 +75997,7 @@ pub fn vuzp_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -76034,7 +76034,7 @@ pub fn vuzpq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzp_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -76063,7 +76063,7 @@ pub fn vuzp_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t {
 }
 #[doc = "Unzip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vuzpq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vuzp))]
@@ -76092,7 +76092,7 @@ pub fn vuzpq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vzip.16"))]
 #[cfg_attr(
@@ -76122,7 +76122,7 @@ pub fn vzip_f16(a: float16x4_t, b: float16x4_t) -> float16x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_f16)"]
-#[inline]
+#[inline(always)]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vzip.16"))]
 #[cfg_attr(
@@ -76152,7 +76152,7 @@ pub fn vzipq_f16(a: float16x8_t, b: float16x8_t) -> float16x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -76181,7 +76181,7 @@ pub fn vzip_f32(a: float32x2_t, b: float32x2_t) -> float32x2x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -76210,7 +76210,7 @@ pub fn vzip_s32(a: int32x2_t, b: int32x2_t) -> int32x2x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtrn))]
@@ -76239,7 +76239,7 @@ pub fn vzip_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))]
@@ -76268,7 +76268,7 @@ pub fn vzip_s8(a: int8x8_t, b: int8x8_t) -> int8x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))]
@@ -76297,7 +76297,7 @@ pub fn vzip_s16(a: int16x4_t, b: int16x4_t) -> int16x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))]
@@ -76326,7 +76326,7 @@ pub fn vzip_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))]
@@ -76355,7 +76355,7 @@ pub fn vzip_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))]
@@ -76384,7 +76384,7 @@ pub fn vzip_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzip_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))]
@@ -76413,7 +76413,7 @@ pub fn vzip_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_f32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76442,7 +76442,7 @@ pub fn vzipq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_s8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76479,7 +76479,7 @@ pub fn vzipq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_s16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76508,7 +76508,7 @@ pub fn vzipq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_s32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76537,7 +76537,7 @@ pub fn vzipq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_u8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76574,7 +76574,7 @@ pub fn vzipq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_u16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76603,7 +76603,7 @@ pub fn vzipq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_u32)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76632,7 +76632,7 @@ pub fn vzipq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_p8)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
@@ -76669,7 +76669,7 @@ pub fn vzipq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t {
 }
 #[doc = "Zip vectors"]
 #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vzipq_p16)"]
-#[inline]
+#[inline(always)]
 #[target_feature(enable = "neon")]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))]
 #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))]
diff --git a/library/stdarch/crates/core_arch/src/lib.rs b/library/stdarch/crates/core_arch/src/lib.rs
index d69089b95b0c..3992eaee30fd 100644
--- a/library/stdarch/crates/core_arch/src/lib.rs
+++ b/library/stdarch/crates/core_arch/src/lib.rs
@@ -32,6 +32,7 @@
     x86_amx_intrinsics,
     f16,
     aarch64_unstable_target_feature,
+    target_feature_inline_always,
     bigint_helper_methods,
     funnel_shifts,
     avx10_target_feature,
diff --git a/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs b/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs
index 0cdff6ff6c39..71301c5ba6ce 100644
--- a/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs
+++ b/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs
@@ -1736,7 +1736,7 @@ fn create_tokens(intrinsic: &Intrinsic, endianness: Endianness, tokens: &mut Tok
         );
     }
 
-    tokens.append_all(quote! { #[inline] });
+    tokens.append_all(quote! { #[inline(always)] });
 
     match endianness {
         Endianness::Little => tokens.append_all(quote! { #[cfg(target_endian = "little")] }),

From 9a5aa90516602b288de34dd65161d90f2db202a6 Mon Sep 17 00:00:00 2001
From: Eric Huss 
Date: Mon, 12 Jan 2026 09:34:11 -0800
Subject: [PATCH 0218/1843] Add some clarifications and fixes for fmt syntax

This tries to clarify a few things regarding fmt syntax:

- The comment on `Parser::word` seems to be wrong, as that
  underscore-prefixed words are just fine. This was changed in
  https://github.com/rust-lang/rust/pull/66847.
- I struggled to follow the description of the width argument. It
  referred to a "second argument", but I don't know what second argument
  it is referring to (which is the first?). Either way, I rewrote the
  paragraph to try to be a little more explicit, and to use shorter
  sentences.
- The description of the precision argument wasn't really clear about
  the distinction of an Nth argument and a named argument. I added
  a sentence to try to emphasize the difference.
- `IDENTIFIER_OR_KEYWORD` was changed recently in
  https://github.com/rust-lang/reference/pull/2049 to include bare `_`.
  But fmt named arguments are not allowed to be a bare `_`.
---
 compiler/rustc_parse_format/src/lib.rs |  2 +-
 library/alloc/src/fmt.rs               | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs
index 86326fc6536c..a687a45480dc 100644
--- a/compiler/rustc_parse_format/src/lib.rs
+++ b/compiler/rustc_parse_format/src/lib.rs
@@ -753,7 +753,7 @@ impl<'input> Parser<'input> {
     }
 
     /// Parses a word starting at the current position. A word is the same as a
-    /// Rust identifier, except that it can't start with `_` character.
+    /// Rust identifier or keyword, except that it can't be a bare `_` character.
     fn word(&mut self) -> &'input str {
         let index = self.input_vec_index;
         match self.peek() {
diff --git a/library/alloc/src/fmt.rs b/library/alloc/src/fmt.rs
index 3d7c580be8c9..e3ff2ba51aba 100644
--- a/library/alloc/src/fmt.rs
+++ b/library/alloc/src/fmt.rs
@@ -136,9 +136,10 @@
 //! padding specified by fill/alignment will be used to take up the required
 //! space (see below).
 //!
-//! The value for the width can also be provided as a [`usize`] in the list of
-//! parameters by adding a postfix `$`, indicating that the second argument is
-//! a [`usize`] specifying the width.
+//! The width can also be provided dynamically by referencing another argument
+//! with a `$` suffix. Use `{:N$}` to reference the Nth positional argument
+//! (where N is an integer), or `{:name$}` to reference a named argument. The
+//! referenced argument must be of type [`usize`].
 //!
 //! Referring to an argument with the dollar syntax does not affect the "next
 //! argument" counter, so it's usually a good idea to refer to arguments by
@@ -236,7 +237,8 @@
 //!
 //! 2. An integer or name followed by dollar sign `.N$`:
 //!
-//!    use format *argument* `N` (which must be a `usize`) as the precision.
+//!    use the value of format *argument* `N` (which must be a `usize`) as the precision.
+//!    An integer refers to a positional argument, and a name refers to a named argument.
 //!
 //! 3. An asterisk `.*`:
 //!
@@ -363,7 +365,10 @@
 //! - `ws` is any character for which [`char::is_whitespace`] returns `true`, has no semantic
 //!   meaning and is completely optional,
 //! - `integer` is a decimal integer that may contain leading zeroes and must fit into an `usize` and
-//! - `identifier` is an `IDENTIFIER_OR_KEYWORD` (not an `IDENTIFIER`) as defined by the [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html).
+//! - `identifier` is an `IDENTIFIER_OR_KEYWORD` (not an `IDENTIFIER`) as
+//!   defined by the [Rust language
+//!   reference](https://doc.rust-lang.org/reference/identifiers.html), except
+//!   for a bare `_`.
 //!
 //! # Formatting traits
 //!

From 815407acb45eb42366995bf31b6ecfa9960af28f Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Mon, 12 Jan 2026 15:41:02 +0100
Subject: [PATCH 0219/1843] clean-up

early-return earlier, and create suggestion-related things later
---
 clippy_lints/src/strlen_on_c_strings.rs | 32 ++++++++++++-------------
 tests/ui/strlen_on_c_strings.fixed      |  3 +--
 tests/ui/strlen_on_c_strings.rs         |  3 +--
 tests/ui/strlen_on_c_strings.stderr     | 28 +++++++++++-----------
 4 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/clippy_lints/src/strlen_on_c_strings.rs b/clippy_lints/src/strlen_on_c_strings.rs
index 0d50bd547652..2b3e0ce611ff 100644
--- a/clippy_lints/src/strlen_on_c_strings.rs
+++ b/clippy_lints/src/strlen_on_c_strings.rs
@@ -1,4 +1,4 @@
-use clippy_utils::diagnostics::span_lint_and_sugg;
+use clippy_utils::diagnostics::span_lint_and_then;
 use clippy_utils::res::MaybeDef;
 use clippy_utils::source::snippet_with_context;
 use clippy_utils::visitors::is_expr_unsafe;
@@ -48,6 +48,15 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
             && !recv.span.from_expansion()
             && path.ident.name == sym::as_ptr
         {
+            let ty = cx.typeck_results().expr_ty(self_arg).peel_refs();
+            let method_name = if ty.is_diag_item(cx, sym::cstring_type) {
+                "as_bytes"
+            } else if ty.is_lang_item(cx, LangItem::CStr) {
+                "to_bytes"
+            } else {
+                return;
+            };
+
             let ctxt = expr.span.ctxt();
             let span = match cx.tcx.parent_hir_node(expr.hir_id) {
                 Node::Block(&Block {
@@ -58,25 +67,16 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
                 _ => expr.span,
             };
 
-            let ty = cx.typeck_results().expr_ty(self_arg).peel_refs();
-            let mut app = Applicability::MachineApplicable;
-            let val_name = snippet_with_context(cx, self_arg.span, ctxt, "..", &mut app).0;
-            let method_name = if ty.is_diag_item(cx, sym::cstring_type) {
-                "as_bytes"
-            } else if ty.is_lang_item(cx, LangItem::CStr) {
-                "to_bytes"
-            } else {
-                return;
-            };
-
-            span_lint_and_sugg(
+            span_lint_and_then(
                 cx,
                 STRLEN_ON_C_STRINGS,
                 span,
                 "using `libc::strlen` on a `CString` or `CStr` value",
-                "try",
-                format!("{val_name}.{method_name}().len()"),
-                app,
+                |diag| {
+                    let mut app = Applicability::MachineApplicable;
+                    let val_name = snippet_with_context(cx, self_arg.span, ctxt, "_", &mut app).0;
+                    diag.span_suggestion(span, "use", format!("{val_name}.{method_name}().len()"), app);
+                },
             );
         }
     }
diff --git a/tests/ui/strlen_on_c_strings.fixed b/tests/ui/strlen_on_c_strings.fixed
index 17c1b541f77c..f52c571c1088 100644
--- a/tests/ui/strlen_on_c_strings.fixed
+++ b/tests/ui/strlen_on_c_strings.fixed
@@ -1,7 +1,6 @@
 #![warn(clippy::strlen_on_c_strings)]
-#![allow(dead_code, clippy::manual_c_str_literals)]
+#![allow(clippy::manual_c_str_literals)]
 
-#[allow(unused)]
 use libc::strlen;
 use std::ffi::{CStr, CString};
 
diff --git a/tests/ui/strlen_on_c_strings.rs b/tests/ui/strlen_on_c_strings.rs
index c641422f5df4..39366d08c1a2 100644
--- a/tests/ui/strlen_on_c_strings.rs
+++ b/tests/ui/strlen_on_c_strings.rs
@@ -1,7 +1,6 @@
 #![warn(clippy::strlen_on_c_strings)]
-#![allow(dead_code, clippy::manual_c_str_literals)]
+#![allow(clippy::manual_c_str_literals)]
 
-#[allow(unused)]
 use libc::strlen;
 use std::ffi::{CStr, CString};
 
diff --git a/tests/ui/strlen_on_c_strings.stderr b/tests/ui/strlen_on_c_strings.stderr
index 84a93b99ee33..eecce8d97865 100644
--- a/tests/ui/strlen_on_c_strings.stderr
+++ b/tests/ui/strlen_on_c_strings.stderr
@@ -1,47 +1,47 @@
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:11:13
+  --> tests/ui/strlen_on_c_strings.rs:10:13
    |
 LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `cstring.as_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.as_bytes().len()`
    |
    = note: `-D clippy::strlen-on-c-strings` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::strlen_on_c_strings)]`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:16:13
+  --> tests/ui/strlen_on_c_strings.rs:15:13
    |
 LL |     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `cstr.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:19:13
+  --> tests/ui/strlen_on_c_strings.rs:18:13
    |
 LL |     let _ = unsafe { strlen(cstr.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `cstr.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:23:22
+  --> tests/ui/strlen_on_c_strings.rs:22:22
    |
 LL |     let _ = unsafe { strlen((*pcstr).as_ptr()) };
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `(*pcstr).to_bytes().len()`
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `(*pcstr).to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:29:22
+  --> tests/ui/strlen_on_c_strings.rs:28:22
    |
 LL |     let _ = unsafe { strlen(unsafe_identity(cstr).as_ptr()) };
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `unsafe_identity(cstr).to_bytes().len()`
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe_identity(cstr).to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:31:13
+  --> tests/ui/strlen_on_c_strings.rs:30:13
    |
 LL |     let _ = unsafe { strlen(unsafe { unsafe_identity(cstr) }.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `unsafe { unsafe_identity(cstr) }.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe { unsafe_identity(cstr) }.to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:35:22
+  --> tests/ui/strlen_on_c_strings.rs:34:22
    |
 LL |     let _ = unsafe { strlen(f(cstr).as_ptr()) };
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `f(cstr).to_bytes().len()`
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `f(cstr).to_bytes().len()`
 
 error: aborting due to 7 previous errors
 

From 7cdfee9de8d7d9c627c4b5be3c5a6e4a7c192d13 Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Mon, 12 Jan 2026 15:44:20 +0100
Subject: [PATCH 0220/1843] feat: specify the type

---
 clippy_lints/src/strlen_on_c_strings.rs |  8 ++++----
 tests/ui/strlen_on_c_strings.fixed      | 14 +++++++-------
 tests/ui/strlen_on_c_strings.rs         | 14 +++++++-------
 tests/ui/strlen_on_c_strings.stderr     | 14 +++++++-------
 4 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/clippy_lints/src/strlen_on_c_strings.rs b/clippy_lints/src/strlen_on_c_strings.rs
index 2b3e0ce611ff..ec0fc08ea552 100644
--- a/clippy_lints/src/strlen_on_c_strings.rs
+++ b/clippy_lints/src/strlen_on_c_strings.rs
@@ -49,10 +49,10 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
             && path.ident.name == sym::as_ptr
         {
             let ty = cx.typeck_results().expr_ty(self_arg).peel_refs();
-            let method_name = if ty.is_diag_item(cx, sym::cstring_type) {
-                "as_bytes"
+            let (ty_name, method_name) = if ty.is_diag_item(cx, sym::cstring_type) {
+                ("CString", "as_bytes")
             } else if ty.is_lang_item(cx, LangItem::CStr) {
-                "to_bytes"
+                ("CStr", "to_bytes")
             } else {
                 return;
             };
@@ -71,7 +71,7 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
                 cx,
                 STRLEN_ON_C_STRINGS,
                 span,
-                "using `libc::strlen` on a `CString` or `CStr` value",
+                format!("using `libc::strlen` on a `{ty_name}` value"),
                 |diag| {
                     let mut app = Applicability::MachineApplicable;
                     let val_name = snippet_with_context(cx, self_arg.span, ctxt, "_", &mut app).0;
diff --git a/tests/ui/strlen_on_c_strings.fixed b/tests/ui/strlen_on_c_strings.fixed
index f52c571c1088..68cf1ba2edc5 100644
--- a/tests/ui/strlen_on_c_strings.fixed
+++ b/tests/ui/strlen_on_c_strings.fixed
@@ -8,29 +8,29 @@ fn main() {
     // CString
     let cstring = CString::new("foo").expect("CString::new failed");
     let _ = cstring.as_bytes().len();
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CString` value
 
     // CStr
     let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
     let _ = cstr.to_bytes().len();
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let _ = cstr.to_bytes().len();
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let pcstr: *const &CStr = &cstr;
     let _ = unsafe { (*pcstr).to_bytes().len() };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     unsafe fn unsafe_identity(x: T) -> T {
         x
     }
     let _ = unsafe { unsafe_identity(cstr).to_bytes().len() };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
     let _ = unsafe { unsafe_identity(cstr) }.to_bytes().len();
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let f: unsafe fn(_) -> _ = unsafe_identity;
     let _ = unsafe { f(cstr).to_bytes().len() };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 }
diff --git a/tests/ui/strlen_on_c_strings.rs b/tests/ui/strlen_on_c_strings.rs
index 39366d08c1a2..6d6eb0b5c152 100644
--- a/tests/ui/strlen_on_c_strings.rs
+++ b/tests/ui/strlen_on_c_strings.rs
@@ -8,29 +8,29 @@ fn main() {
     // CString
     let cstring = CString::new("foo").expect("CString::new failed");
     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CString` value
 
     // CStr
     let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let _ = unsafe { strlen(cstr.as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let pcstr: *const &CStr = &cstr;
     let _ = unsafe { strlen((*pcstr).as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     unsafe fn unsafe_identity(x: T) -> T {
         x
     }
     let _ = unsafe { strlen(unsafe_identity(cstr).as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
     let _ = unsafe { strlen(unsafe { unsafe_identity(cstr) }.as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let f: unsafe fn(_) -> _ = unsafe_identity;
     let _ = unsafe { strlen(f(cstr).as_ptr()) };
-    //~^ strlen_on_c_strings
+    //~^ ERROR: using `libc::strlen` on a `CStr` value
 }
diff --git a/tests/ui/strlen_on_c_strings.stderr b/tests/ui/strlen_on_c_strings.stderr
index eecce8d97865..f2ccc91aabf1 100644
--- a/tests/ui/strlen_on_c_strings.stderr
+++ b/tests/ui/strlen_on_c_strings.stderr
@@ -1,4 +1,4 @@
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CString` value
   --> tests/ui/strlen_on_c_strings.rs:10:13
    |
 LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
@@ -7,37 +7,37 @@ LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
    = note: `-D clippy::strlen-on-c-strings` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::strlen_on_c_strings)]`
 
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:15:13
    |
 LL |     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
 
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:18:13
    |
 LL |     let _ = unsafe { strlen(cstr.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
 
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:22:22
    |
 LL |     let _ = unsafe { strlen((*pcstr).as_ptr()) };
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `(*pcstr).to_bytes().len()`
 
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:28:22
    |
 LL |     let _ = unsafe { strlen(unsafe_identity(cstr).as_ptr()) };
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe_identity(cstr).to_bytes().len()`
 
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:30:13
    |
 LL |     let _ = unsafe { strlen(unsafe { unsafe_identity(cstr) }.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe { unsafe_identity(cstr) }.to_bytes().len()`
 
-error: using `libc::strlen` on a `CString` or `CStr` value
+error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:34:22
    |
 LL |     let _ = unsafe { strlen(f(cstr).as_ptr()) };

From 49c86140fb2326ea58655c0fc3e46d52bcacce89 Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Mon, 12 Jan 2026 17:54:50 +0100
Subject: [PATCH 0221/1843] feat: also lint types that dereference to `CStr`

For simplicity's sake, this also changes the lint to always suggest
`to_bytes`, as it is somewhat hard to find out whether a type could
dereference to `CString` (which is what `as_bytes` would require)
---
 clippy_lints/src/strlen_on_c_strings.rs | 19 ++++++++++++-------
 tests/ui/strlen_on_c_strings.fixed      | 14 ++++++++++++--
 tests/ui/strlen_on_c_strings.rs         | 12 +++++++++++-
 tests/ui/strlen_on_c_strings.stderr     | 22 ++++++++++++++++++++--
 4 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/clippy_lints/src/strlen_on_c_strings.rs b/clippy_lints/src/strlen_on_c_strings.rs
index ec0fc08ea552..5eb160720c52 100644
--- a/clippy_lints/src/strlen_on_c_strings.rs
+++ b/clippy_lints/src/strlen_on_c_strings.rs
@@ -47,14 +47,19 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
             && let ExprKind::MethodCall(path, self_arg, [], _) = recv.kind
             && !recv.span.from_expansion()
             && path.ident.name == sym::as_ptr
+            && let typeck = cx.typeck_results()
+            && typeck
+                .expr_ty_adjusted(self_arg)
+                .peel_refs()
+                .is_lang_item(cx, LangItem::CStr)
         {
-            let ty = cx.typeck_results().expr_ty(self_arg).peel_refs();
-            let (ty_name, method_name) = if ty.is_diag_item(cx, sym::cstring_type) {
-                ("CString", "as_bytes")
+            let ty = typeck.expr_ty(self_arg).peel_refs();
+            let ty_kind = if ty.is_diag_item(cx, sym::cstring_type) {
+                "`CString` value"
             } else if ty.is_lang_item(cx, LangItem::CStr) {
-                ("CStr", "to_bytes")
+                "`CStr` value"
             } else {
-                return;
+                "type that dereferences to `CStr`"
             };
 
             let ctxt = expr.span.ctxt();
@@ -71,11 +76,11 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
                 cx,
                 STRLEN_ON_C_STRINGS,
                 span,
-                format!("using `libc::strlen` on a `{ty_name}` value"),
+                format!("using `libc::strlen` on a {ty_kind}"),
                 |diag| {
                     let mut app = Applicability::MachineApplicable;
                     let val_name = snippet_with_context(cx, self_arg.span, ctxt, "_", &mut app).0;
-                    diag.span_suggestion(span, "use", format!("{val_name}.{method_name}().len()"), app);
+                    diag.span_suggestion(span, "use", format!("{val_name}.to_bytes().len()"), app);
                 },
             );
         }
diff --git a/tests/ui/strlen_on_c_strings.fixed b/tests/ui/strlen_on_c_strings.fixed
index 68cf1ba2edc5..33a328af6df4 100644
--- a/tests/ui/strlen_on_c_strings.fixed
+++ b/tests/ui/strlen_on_c_strings.fixed
@@ -1,5 +1,5 @@
 #![warn(clippy::strlen_on_c_strings)]
-#![allow(clippy::manual_c_str_literals)]
+#![allow(clippy::manual_c_str_literals, clippy::boxed_local)]
 
 use libc::strlen;
 use std::ffi::{CStr, CString};
@@ -7,7 +7,7 @@ use std::ffi::{CStr, CString};
 fn main() {
     // CString
     let cstring = CString::new("foo").expect("CString::new failed");
-    let _ = cstring.as_bytes().len();
+    let _ = cstring.to_bytes().len();
     //~^ ERROR: using `libc::strlen` on a `CString` value
 
     // CStr
@@ -34,3 +34,13 @@ fn main() {
     let _ = unsafe { f(cstr).to_bytes().len() };
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 }
+
+// make sure we lint types that _adjust_ to `CStr`
+fn adjusted(box_cstring: Box, box_cstr: Box, arc_cstring: std::sync::Arc) {
+    let _ = box_cstring.to_bytes().len();
+    //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
+    let _ = box_cstr.to_bytes().len();
+    //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
+    let _ = arc_cstring.to_bytes().len();
+    //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
+}
diff --git a/tests/ui/strlen_on_c_strings.rs b/tests/ui/strlen_on_c_strings.rs
index 6d6eb0b5c152..3c11c3a05269 100644
--- a/tests/ui/strlen_on_c_strings.rs
+++ b/tests/ui/strlen_on_c_strings.rs
@@ -1,5 +1,5 @@
 #![warn(clippy::strlen_on_c_strings)]
-#![allow(clippy::manual_c_str_literals)]
+#![allow(clippy::manual_c_str_literals, clippy::boxed_local)]
 
 use libc::strlen;
 use std::ffi::{CStr, CString};
@@ -34,3 +34,13 @@ fn main() {
     let _ = unsafe { strlen(f(cstr).as_ptr()) };
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 }
+
+// make sure we lint types that _adjust_ to `CStr`
+fn adjusted(box_cstring: Box, box_cstr: Box, arc_cstring: std::sync::Arc) {
+    let _ = unsafe { libc::strlen(box_cstring.as_ptr()) };
+    //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
+    let _ = unsafe { libc::strlen(box_cstr.as_ptr()) };
+    //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
+    let _ = unsafe { libc::strlen(arc_cstring.as_ptr()) };
+    //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
+}
diff --git a/tests/ui/strlen_on_c_strings.stderr b/tests/ui/strlen_on_c_strings.stderr
index f2ccc91aabf1..2b059872a2da 100644
--- a/tests/ui/strlen_on_c_strings.stderr
+++ b/tests/ui/strlen_on_c_strings.stderr
@@ -2,7 +2,7 @@ error: using `libc::strlen` on a `CString` value
   --> tests/ui/strlen_on_c_strings.rs:10:13
    |
 LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.as_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.to_bytes().len()`
    |
    = note: `-D clippy::strlen-on-c-strings` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::strlen_on_c_strings)]`
@@ -43,5 +43,23 @@ error: using `libc::strlen` on a `CStr` value
 LL |     let _ = unsafe { strlen(f(cstr).as_ptr()) };
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `f(cstr).to_bytes().len()`
 
-error: aborting due to 7 previous errors
+error: using `libc::strlen` on a type that dereferences to `CStr`
+  --> tests/ui/strlen_on_c_strings.rs:40:13
+   |
+LL |     let _ = unsafe { libc::strlen(box_cstring.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `box_cstring.to_bytes().len()`
+
+error: using `libc::strlen` on a type that dereferences to `CStr`
+  --> tests/ui/strlen_on_c_strings.rs:42:13
+   |
+LL |     let _ = unsafe { libc::strlen(box_cstr.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `box_cstr.to_bytes().len()`
+
+error: using `libc::strlen` on a type that dereferences to `CStr`
+  --> tests/ui/strlen_on_c_strings.rs:44:13
+   |
+LL |     let _ = unsafe { libc::strlen(arc_cstring.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `arc_cstring.to_bytes().len()`
+
+error: aborting due to 10 previous errors
 

From 97a06feab0869c9a57a3a63c840d9e3e6664aea8 Mon Sep 17 00:00:00 2001
From: Tshepang Mbambo 
Date: Mon, 12 Jan 2026 22:33:08 +0200
Subject: [PATCH 0222/1843] add range-diff triagebot option

---
 src/doc/rustc-dev-guide/triagebot.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/doc/rustc-dev-guide/triagebot.toml b/src/doc/rustc-dev-guide/triagebot.toml
index 974f4cd3dd96..f894456d0de3 100644
--- a/src/doc/rustc-dev-guide/triagebot.toml
+++ b/src/doc/rustc-dev-guide/triagebot.toml
@@ -86,3 +86,6 @@ rustc-dev-guide = [
     "@jyn514",
     "@tshepang",
 ]
+
+# Make rebases more easy to read: https://forge.rust-lang.org/triagebot/range-diff.html
+[range-diff]

From 3777ebc6ca156092e47318a20c9c2ae40d9813e2 Mon Sep 17 00:00:00 2001
From: Moulins 
Date: Thu, 18 Dec 2025 19:23:37 +0100
Subject: [PATCH 0223/1843] Don't expose redundant information in
 `rustc_public`'s `LayoutShape`

Enum variant layouts don't need to store a full `LayoutShape`; just storing
the fields offsets is enough and all other information can be inferred from
the parent layout:
- size, align and ABI don't make much sense for individual variants and
  should generally be taken from the parent layout instead;
- variants always have `fields: FieldsShape::Arbitrary { .. }` and
  `variant: VariantShape::Single { .. }`.
---
 compiler/rustc_public/src/abi.rs              | 19 ++++++++++++++++++-
 .../src/unstable/convert/stable/abi.rs        | 12 ++++++++++--
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_public/src/abi.rs b/compiler/rustc_public/src/abi.rs
index aced39059f6b..413bc835e50a 100644
--- a/compiler/rustc_public/src/abi.rs
+++ b/compiler/rustc_public/src/abi.rs
@@ -188,10 +188,27 @@ pub enum VariantsShape {
         tag: Scalar,
         tag_encoding: TagEncoding,
         tag_field: usize,
-        variants: Vec,
+        variants: Vec,
     },
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
+pub struct VariantFields {
+    /// Offsets for the first byte of each field,
+    /// ordered to match the source definition order.
+    /// I.e.: It follows the same order as [super::ty::VariantDef::fields()].
+    /// This vector does not go in increasing order.
+    pub offsets: Vec,
+}
+
+impl VariantFields {
+    pub fn fields_by_offset_order(&self) -> Vec {
+        let mut indices = (0..self.offsets.len()).collect::>();
+        indices.sort_by_key(|idx| self.offsets[*idx]);
+        indices
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
 pub enum TagEncoding {
     /// The tag directly stores the discriminant, but possibly with a smaller layout
diff --git a/compiler/rustc_public/src/unstable/convert/stable/abi.rs b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
index 03328d084ee9..f6b750f75aea 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/abi.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
@@ -11,7 +11,7 @@ use rustc_target::callconv;
 use crate::abi::{
     AddressSpace, ArgAbi, CallConvention, FieldsShape, FloatLength, FnAbi, IntegerLength,
     IntegerType, Layout, LayoutShape, PassMode, Primitive, ReprFlags, ReprOptions, Scalar,
-    TagEncoding, TyAndLayout, ValueAbi, VariantsShape, WrappingRange,
+    TagEncoding, TyAndLayout, ValueAbi, VariantFields, VariantsShape, WrappingRange,
 };
 use crate::compiler_interface::BridgeTys;
 use crate::target::MachineSize as Size;
@@ -212,7 +212,15 @@ impl<'tcx> Stable<'tcx> for rustc_abi::Variants VariantFields {
+                                offsets: offsets.iter().as_slice().stable(tables, cx),
+                            },
+                            _ => panic!("variant layout should be Arbitrary"),
+                        })
+                        .collect(),
                 }
             }
         }

From 02333ad8a2805b333d2d0a944fc04a2ff3b37b90 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Tue, 13 Jan 2026 14:10:53 +1100
Subject: [PATCH 0224/1843] Simplify some literal-value negations with
 `u128::wrapping_neg`

---
 compiler/rustc_mir_build/src/builder/expr/as_constant.rs | 4 +++-
 compiler/rustc_mir_build/src/thir/constant.rs            | 8 ++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
index 1772d66f5285..8a242445f175 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
@@ -157,7 +157,9 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
         }
         (ast::LitKind::Int(n, _), ty::Uint(_)) if !neg => trunc(n.get()),
         (ast::LitKind::Int(n, _), ty::Int(_)) => {
-            trunc(if neg { (n.get() as i128).overflowing_neg().0 as u128 } else { n.get() })
+            // Unsigned "negation" has the same bitwise effect as signed negation,
+            // which gets the result we want without additional casts.
+            trunc(if neg { u128::wrapping_neg(n.get()) } else { n.get() })
         }
         (ast::LitKind::Float(n, _), ty::Float(fty)) => {
             parse_float_into_constval(n, *fty, neg).unwrap()
diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs
index 7964a58a7ab0..96248499044a 100644
--- a/compiler/rustc_mir_build/src/thir/constant.rs
+++ b/compiler/rustc_mir_build/src/thir/constant.rs
@@ -72,10 +72,10 @@ pub(crate) fn lit_to_const<'tcx>(
             ty::ValTree::from_scalar_int(tcx, scalar_int)
         }
         (ast::LitKind::Int(n, _), ty::Int(i)) => {
-            let scalar_int = trunc(
-                if neg { (n.get() as i128).overflowing_neg().0 as u128 } else { n.get() },
-                i.to_unsigned(),
-            );
+            // Unsigned "negation" has the same bitwise effect as signed negation,
+            // which gets the result we want without additional casts.
+            let scalar_int =
+                trunc(if neg { u128::wrapping_neg(n.get()) } else { n.get() }, i.to_unsigned());
             ty::ValTree::from_scalar_int(tcx, scalar_int)
         }
         (ast::LitKind::Bool(b), ty::Bool) => ty::ValTree::from_scalar_int(tcx, b.into()),

From 959be82effe087543befbd1aeb0920171ae2fc3f Mon Sep 17 00:00:00 2001
From: joboet 
Date: Mon, 12 Jan 2026 13:58:29 +0100
Subject: [PATCH 0225/1843] std: implement `sleep_until` on Apple platforms

---
 library/std/src/lib.rs               |  1 +
 library/std/src/sys/pal/unix/time.rs | 52 ++++++++++++++++++++++------
 library/std/src/sys/thread/mod.rs    |  2 ++
 library/std/src/sys/thread/unix.rs   | 41 ++++++++++++++++++++++
 library/std/src/thread/functions.rs  |  1 +
 5 files changed, 87 insertions(+), 10 deletions(-)

diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs
index f5b9f69a5f9f..93f2c519dd1b 100644
--- a/library/std/src/lib.rs
+++ b/library/std/src/lib.rs
@@ -326,6 +326,7 @@
 #![feature(const_convert)]
 #![feature(core_intrinsics)]
 #![feature(core_io_borrowed_buf)]
+#![feature(cstr_display)]
 #![feature(drop_guard)]
 #![feature(duration_constants)]
 #![feature(error_generic_member_access)]
diff --git a/library/std/src/sys/pal/unix/time.rs b/library/std/src/sys/pal/unix/time.rs
index 50b690256a20..a56034f3778c 100644
--- a/library/std/src/sys/pal/unix/time.rs
+++ b/library/std/src/sys/pal/unix/time.rs
@@ -270,22 +270,25 @@ pub struct Instant {
 }
 
 impl Instant {
+    // CLOCK_UPTIME_RAW   clock that increments monotonically, in the same man-
+    //                    ner as CLOCK_MONOTONIC_RAW, but that does not incre-
+    //                    ment while the system is asleep.  The returned value
+    //                    is identical to the result of mach_absolute_time()
+    //                    after the appropriate mach_timebase conversion is
+    //                    applied.
+    //
+    // We use `CLOCK_UPTIME_RAW` instead of `CLOCK_MONOTONIC` since
+    // `CLOCK_UPTIME_RAW` is based on `mach_absolute_time`, which is the
+    // clock that all timeouts and deadlines are measured against inside
+    // the kernel.
     #[cfg(target_vendor = "apple")]
     pub(crate) const CLOCK_ID: libc::clockid_t = libc::CLOCK_UPTIME_RAW;
+
     #[cfg(not(target_vendor = "apple"))]
     pub(crate) const CLOCK_ID: libc::clockid_t = libc::CLOCK_MONOTONIC;
+
     pub fn now() -> Instant {
         // https://pubs.opengroup.org/onlinepubs/9799919799/functions/clock_getres.html
-        //
-        // CLOCK_UPTIME_RAW   clock that increments monotonically, in the same man-
-        //                    ner as CLOCK_MONOTONIC_RAW, but that does not incre-
-        //                    ment while the system is asleep.  The returned value
-        //                    is identical to the result of mach_absolute_time()
-        //                    after the appropriate mach_timebase conversion is
-        //                    applied.
-        //
-        // Instant on macos was historically implemented using mach_absolute_time;
-        // we preserve this value domain out of an abundance of caution.
         Instant { t: Timespec::now(Self::CLOCK_ID) }
     }
 
@@ -308,6 +311,35 @@ impl Instant {
     pub(crate) fn into_timespec(self) -> Timespec {
         self.t
     }
+
+    /// Returns `self` converted into units of `mach_absolute_time`, or `None`
+    /// if `self` is before the system boot time.
+    #[cfg(target_vendor = "apple")]
+    pub fn into_mach_absolute_time(self) -> Option {
+        #[repr(C)]
+        struct mach_timebase_info {
+            numer: u32,
+            denom: u32,
+        }
+
+        unsafe extern "C" {
+            unsafe fn mach_timebase_info(info: *mut mach_timebase_info) -> libc::kern_return_t;
+        }
+
+        let secs = u64::try_from(self.t.tv_sec).ok()?;
+
+        let mut timebase = mach_timebase_info { numer: 0, denom: 0 };
+        assert_eq!(unsafe { mach_timebase_info(&mut timebase) }, libc::KERN_SUCCESS);
+
+        // Since `tv_sec` is 64-bit and `tv_nsec` is smaller than 1 billion,
+        // this cannot overflow. The resulting number needs at most 94 bits.
+        let nanos =
+            u128::from(secs) * u128::from(NSEC_PER_SEC) + u128::from(self.t.tv_nsec.as_inner());
+        // This multiplication cannot overflow since multiplying a 94-bit
+        // number by a 32-bit number yields a number that needs at most
+        // 126 bits.
+        Some(nanos * u128::from(timebase.denom) / u128::from(timebase.numer))
+    }
 }
 
 impl AsInner for Instant {
diff --git a/library/std/src/sys/thread/mod.rs b/library/std/src/sys/thread/mod.rs
index cb6bf6518f81..3460270b15fc 100644
--- a/library/std/src/sys/thread/mod.rs
+++ b/library/std/src/sys/thread/mod.rs
@@ -73,6 +73,7 @@ cfg_select! {
             target_os = "fuchsia",
             target_os = "vxworks",
             target_os = "wasi",
+            target_vendor = "apple",
         ))]
         pub use unix::sleep_until;
         #[expect(dead_code)]
@@ -133,6 +134,7 @@ cfg_select! {
     target_os = "fuchsia",
     target_os = "vxworks",
     target_os = "wasi",
+    target_vendor = "apple",
 )))]
 pub fn sleep_until(deadline: crate::time::Instant) {
     use crate::time::Instant;
diff --git a/library/std/src/sys/thread/unix.rs b/library/std/src/sys/thread/unix.rs
index d0396ed71300..e52bc63ea5b1 100644
--- a/library/std/src/sys/thread/unix.rs
+++ b/library/std/src/sys/thread/unix.rs
@@ -636,6 +636,47 @@ pub fn sleep_until(deadline: crate::time::Instant) {
     }
 }
 
+#[cfg(target_vendor = "apple")]
+pub fn sleep_until(deadline: crate::time::Instant) {
+    unsafe extern "C" {
+        // This is defined in the public header mach/mach_time.h alongside
+        // `mach_absolute_time`, and like it has been available since the very
+        // beginning.
+        //
+        // There isn't really any documentation on this function, except for a
+        // short reference in technical note 2169:
+        // https://developer.apple.com/library/archive/technotes/tn2169/_index.html
+        safe fn mach_wait_until(deadline: u64) -> libc::kern_return_t;
+    }
+
+    let Some(deadline) = deadline.into_inner().into_mach_absolute_time() else {
+        // Since the deadline is before the system boot time, it has already
+        // passed, so we can return immediately.
+        return;
+    };
+
+    // If the deadline is not representable, then sleep for the maximum duration
+    // possible and worry about the potential clock issues later (in ca. 600 years).
+    let deadline = deadline.try_into().unwrap_or(u64::MAX);
+    loop {
+        match mach_wait_until(deadline) {
+            // Success! The deadline has passed.
+            libc::KERN_SUCCESS => break,
+            // If the sleep gets interrupted by a signal, `mach_wait_until`
+            // returns KERN_ABORTED, so we need to restart the syscall.
+            // Also see Apple's implementation of the POSIX `nanosleep`, which
+            // converts this error to the POSIX equivalent EINTR:
+            // https://github.com/apple-oss-distributions/Libc/blob/55b54c0a0c37b3b24393b42b90a4c561d6c606b1/gen/nanosleep.c#L281-L306
+            libc::KERN_ABORTED => continue,
+            // All other errors indicate that something has gone wrong...
+            error => {
+                let description = unsafe { CStr::from_ptr(libc::mach_error_string(error)) };
+                panic!("mach_wait_until failed: {} (code {error})", description.display())
+            }
+        }
+    }
+}
+
 pub fn yield_now() {
     let ret = unsafe { libc::sched_yield() };
     debug_assert_eq!(ret, 0);
diff --git a/library/std/src/thread/functions.rs b/library/std/src/thread/functions.rs
index a25bae1aae31..73d727878570 100644
--- a/library/std/src/thread/functions.rs
+++ b/library/std/src/thread/functions.rs
@@ -318,6 +318,7 @@ pub fn sleep(dur: Duration) {
 /// | Hurd      | [clock_nanosleep] (Monotonic Clock)]                                 |
 /// | Fuchsia   | [clock_nanosleep] (Monotonic Clock)]                                 |
 /// | Vxworks   | [clock_nanosleep] (Monotonic Clock)]                                 |
+/// | Apple     | `mach_wait_until`                                                    |
 /// | Other     | `sleep_until` uses [`sleep`] and does not issue a syscall itself     |
 ///
 /// [currently]: crate::io#platform-specific-behavior

From f09b0bc2cb7b7f43ecd6e0adee5e7b3dbc2c1290 Mon Sep 17 00:00:00 2001
From: reucru01 
Date: Mon, 22 Dec 2025 10:31:05 +0000
Subject: [PATCH 0226/1843] Creates README for stdarch-gen-arm

---
 .../stdarch/crates/stdarch-gen-arm/README.md  | 300 ++++++++++++++++++
 1 file changed, 300 insertions(+)
 create mode 100644 library/stdarch/crates/stdarch-gen-arm/README.md

diff --git a/library/stdarch/crates/stdarch-gen-arm/README.md b/library/stdarch/crates/stdarch-gen-arm/README.md
new file mode 100644
index 000000000000..4da14bcbb6c9
--- /dev/null
+++ b/library/stdarch/crates/stdarch-gen-arm/README.md
@@ -0,0 +1,300 @@
+# stdarch-gen-arm generator guide
+## Running the generator
+- Run: `cargo run --bin=stdarch-gen-arm -- crates/stdarch-gen-arm/spec`
+```
+$ cargo run --bin=stdarch-gen-arm -- crates/stdarch-gen-arm/spec
+    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.18s
+     Running `target/debug/stdarch-gen-arm crates/stdarch-gen-arm/spec`
+```
+## Input/Output
+### Input files (intrinsic YAML definitions)
+ - `crates/stdarch-gen-arm/spec//*.spec.yml`
+### Output files
+ - Generated intrinsics:
+	 - `crates/core_arch/src///generated.rs`
+ - Generated load/store tests:
+	 - `crates/core_arch/src///ld_st_tests_.rs`
+	 - Only generated when `test: { load:  }` or `test: { store:  }` is set for SVE/SVE2 intrinsics.
+## `.spec.yml` file anatomy
+```
+---
+Configs
+---
+Variable definitions
+---
+
+Intrinsic definitions
+
+---
+```
+- If you're new to YAML syntax, consider [reviewing](https://quickref.me/yaml.html) some of the less obvious syntax and features.
+- For example, mapping an attribute to a sequence can be done in two different ways:
+```yaml
+attribute: [item_a, item_b, item_c]
+```
+or
+```yaml
+attribute:
+    - item_a
+    - item_b
+    - item_c
+``` 
+## Configs
+- Mappings defining top-level settings applied to all intrinsics:
+- `arch_cfgs`
+    - Sequence of mappings specifying `arch_name`, `target_feature` (sequence), and `llvm_prefix`.
+- `uses_neon_types`(_Optional_)
+    - A boolean specifying whether to emit NEON type imports in generated code.
+- `auto_big_endian`(_Optional_)
+    - A boolean specifying whether to auto-generate big-endian shuffles when possible.
+- `auto_llvm_sign_conversion`(_Optional_)
+    - A boolean specifying whether to auto-convert LLVM wrapper args to signed types.
+## Variable definitions
+- Defines YAML anchors/variables to avoid repetition.
+- Commonly used for stability attributes, cfgs and target features.
+## Intrinsic definitions
+### Example
+```yaml
+    - name: "vtst{neon_type[0].no}"
+      doc: "Signed compare bitwise Test bits nonzero"
+      arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"]
+      return_type: "{neon_type[1]}"
+      attr:
+        - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmtst]]}]]
+        - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]
+      safety: safe
+      types:
+        - [int64x1_t, uint64x1_t, 'i64x1', 'i64x1::new(0)']
+        - [int64x2_t, uint64x2_t, 'i64x2', 'i64x2::new(0, 0)']
+        - [poly64x1_t, uint64x1_t, 'i64x1', 'i64x1::new(0)']
+        - [poly64x2_t, uint64x2_t, 'i64x2', 'i64x2::new(0, 0)']
+      compose:
+        - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}]
+        - Let: [d, "{type[2]}", "{type[3]}"]
+        - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]]
+```
+
+### Explanation of fields
+- `name`
+    - The name of the intrinsic
+    - Often built from a base name followed by a type suffix
+- `doc` (_Optional_)
+    - A string explaining the purpose of the intrinsic
+- `static_defs` (_Optional_)
+    - A sequence of const generics of the format `"const : "`
+- `arguments`
+    - A sequence of strings in the format `": "`
+- `return_type` (_Optional_)
+    - A string specifying the return type. If omitted, the intrinsic returns `()`.
+- `attr` (_Optional_)
+    - A sequence of items defining the attributes to be applied to the intrinsic. Often stability attributes, target features, or `assert_instr` tests. At least one of `attr` or `assert_instr` must be set.
+- `target_features` (_Optional_)
+    - A sequence of target features to enable for this intrinsic (merged with any global `arch_cfgs` settings).
+- `assert_instr` (_Optional_)
+    - A sequence of strings expected to be found in the assembly. Required if `attr` is not set.
+- `safety` (_Optional_)
+    - Use `safe`, or map `unsafe:` to a sequence of unsafety comments:
+        - `custom: ""`
+        - `uninitialized`
+        - `pointer_offset`, `pointer_offset_vnum`, or `dereference` (optionally qualified with `predicated`, `predicated_non_faulting`, or `predicated_first_faulting`)
+        - `unpredictable_on_fault`
+        - `non_temporal`
+        - `neon`
+        - `no_provenance: ""`
+- `substitutions` (_Optional_)
+    - Mappings of custom wildcard names to either `MatchSize` or `MatchKind` expressions
+- `types`
+    - A sequence or sequence of sequences specifying the types to use when producing each intrinsic variant. These sequences can then be indexed by wildcards.
+- `constraints` (_Optional_)
+    - A sequence of mappings. Each specifies a variable and a constraint. The available mappings are:
+    - Assert a variable's value exists in a sequence of i32's
+        - Usage: `{ variable: , any_values: [,...] }`
+    - Assert a variable's value exists in a range (inclusive)
+        - Usage: `{ variable: , range: [, ] }`
+    - Assert a variable's value exists in a range via a match (inclusive)
+        - Usage: `{ variable: , range:  }`
+    - Assert a variable's value does not exceed the number of elements in a SVE type ``.
+        - Usage: `{ variable: , sve_max_elems_type:  }`
+    - Assert a variable's value does not exceed the number of elements in a vector type ``.
+        - Usage: `{ variable: , vec_max_elems_type:  }`
+- `predication_methods` (_Optional_)
+    - Configuration for predicate-form variants. Only used when the intrinsic name includes an `_m*_` wildcard (e.g., `{_mx}`, `{_mxz}`).
+    - `zeroing_method`: Required when requesting `_z`; either `{ drop:  }` to remove an argument and replace it with a zero initialiser, or `{ select:  }` to select zeros into a predicate.
+    - `dont_care_method`: How `_x` should be implemented (`inferred`, `as_zeroing`, or `as_merging`).
+- `compose`
+    - A sequence of expressions that make up the body of the intrinsic
+- `big_endian_inverse` (_Optional_)
+    - A boolean, default false. If true, generates two implementations of each intrinsic variant, one for each endianness, and attempts to automatically generate the required bit swizzles
+- `visibility` (_Optional_)
+    - Function visibility. One of `public` (default) or `private`.
+- `n_variant_op` (_Optional_)
+    - Enables generation of an `_n` variant when the intrinsic name includes the `{_n}` wildcard. Set to the operand name that should be splattered for the `_n` form.
+- `test` (_Optional_)
+	- When set, load/store tests are automatically generated.
+    - A mapping of either `load` or `store` to a number that indexes `types` to specify the type that the test should be addressing in memory. 
+### Expressions
+#### Common
+- `Let`
+    - Defines a variable
+    - Usage: `Let: [, , ]`
+- `Const`
+    - Defines a const
+    - Usage: `Const: [, , ]`
+- `Assign`
+    - Performs variable assignment
+    - Usage: `Assign: [, ]`
+- `FnCall`
+    - Performs a function call
+    - Usage: `FnCall: [, [, ... ], [, ...](optional), ]`
+- `MacroCall`
+    - Performs a macro call
+    - Usage: `MacroCall: [, ]`
+- `MethodCall`
+    - Performs a method call
+    - Usage: `MethodCall: [, , [, ... ]]`
+- `LLVMLink`
+    - Creates an LLVM link and stores the function's name in the wildcard `{llvm_link}` for later use in subsequent expressions.
+    - If left unset, the arguments and return type inherit from the intrinsic's signature by default. The links will also be set automatically if unset.
+    - Usage:
+```yaml
+LLVMLink:
+    name: 
+    arguments: [, ... ](optional)
+    return_type: (optional)
+    links: (optional)
+        - link: 
+          arch: 
+        - ...
+```
+- `Identifier`
+    - Emits a symbol. Prepend with a `$` to treat it as a scope variable, which engages variable tracking and enables inference. For example, `my_function_name` for a generic symbol or `$my_variable` for a variable.
+    - Usage `Identifier: [, ]`
+- `CastAs`
+    - Casts an expression to an unchecked type
+    - Usage: `CastAs: [, ]`
+- `MatchSize`
+    - Allows for conditional generation depending on the size of a specified type
+    - Usage:
+```yaml
+MatchSize:
+    - 
+    - default: 
+      byte(optional): 
+      halfword(optional): 
+      doubleword(optional): 
+```
+- `MatchKind`
+    - Allows for conditional generation depending on the kind of a specified type
+```yaml
+MatchKind:
+    - 
+    - default: 
+      float(optional): 
+      unsigned(optional): 
+```
+#### Rarely Used
+- `IntConstant`
+    - Constant signed integer expression
+    - Usage: `IntConstant: `
+- `FloatConstant`
+    - Constant floating-point expression
+    - Usage: `FloatConstant: `
+- `BoolConstant`
+    - Constant boolean expression
+    - Usage: `BoolConstant: `
+- `Array`
+    - An array of expressions
+    - Usage: `Array: [, ...]`
+- `SvUndef`
+    - Returns the LLVM `undef` symbol
+    - Usage: `SvUndef`
+- `Multiply`
+    - Simply `*`
+    - Usage: `Multiply: [, ]`
+- `Xor`
+    - Simply `^`
+    - Usage: `Xor: [, ]`
+- `ConvertConst`
+    - Converts the specified constant to the specified type's kind
+    - Usage: `ConvertConst: [, ]`
+- `Type`
+    - Yields the given type in the Rust representation
+    - Usage: `Type: []`
+
+### Wildstrings
+- Wildstrings let you take advantage of wildcards.
+- For example, they are often used in intrinsic names `name: "vtst{neon_type[0].no}"`
+- As shown above, wildcards are identified by the surrounding curly brackets.
+- Double curly brackets can be used to escape wildcard functionality if you need literal curly brackets in the generated intrinsic.
+### Wildcards
+Wildcards are heavily used in the spec. They let you write generalised definitions for a group of intrinsics that generate multiple variants. The wildcard itself is replaced with the relevant string in each variant.
+Ignoring endianness, for each row in the `types` field of an intrinsic in the spec, a variant of the intrinsic will be generated. That row's contents can be indexed by the wildcards. Below is the behaviour of each wildcard.
+- `type[]`
+    - Replaced in each variant with the value in the indexed position in the relevant row of the `types` field.
+    - For unnested sequences of `types` (i.e., `types` is a sequence where each element is a single item, not another sequence), the square brackets can be omitted. Simply: `type`
+- `neon_type[]`
+    - Extends the behaviour of `type` with some NEON-specific features and inference.
+    - Tuples: This wildcard can also be written as `neon_type_x` where `n` is in the set `{2,3,4}`. This generates the `n`-tuple variant of the (inferred) NEON type.
+    - Suffixes: These modify the behaviour of the wildcard from simple substitution.
+	    - `no` -  normal behaviour. Tries to do as much work as it can for you, inferring when to emit:
+            - Regular type-size suffixes: `_s8`, `_u16`, `_f32`, ...
+            - `q` variants for double-width (128b) vector types: `q_s8`, `q_u16`, `q_f32`, ...
+            - `_x` variants for tuple vector types: `_s8_x2`, `_u32_x3`, `_f64_x4`, ...
+            - As well as any combination of the above: `q_s16_x16` ...
+    - Most of the other suffixes modify the normal behaviour by disabling features or adding new ones. (See table below)
+- `sve_type[]`
+    - Similar to `neon_type`, but without the suffixes.
+- `size[]`
+    - The size (in bits) of the indexed type.
+- `size_minus_one[]`
+    - Emits the size (in bits) of the indexed type minus one.
+- `size_literal[]`
+    - The literal representation of the indexed type.
+    - `b`: byte, `h`: halfword, `w`: word, or `d`: double.
+- `type_kind[]`
+    - The literal representation of the indexed type's kind.
+    - `f`: float, `s`: signed, `u`: unsigned, `p`: polynomial, `b`: boolean.
+- `size_in_bytes_log2[]`
+    - Log2 of the size of the indexed type in *bytes*.
+- `predicate[]`
+    - SVE predicate vector type inferred from the indexed type.
+- `max_predicate`
+    - The same as predicate, but uses the largest type in the relevant `types` sequence/row.
+- `_n`
+    - Emits the current N-variant suffix when `n_variant_op` is configured.
+- ` as `
+    - If `` evaluates to a vector, it produces a vector of the same shape, but with `` as the base type.
+- `llvm_link`
+    - If the `LLVMLink` mapping has been set for an intrinsic, this will give the name of the link.
+- `_m*`
+    - Predicate form masks. Use wildcards such as `{_mx}` or `{_mxz}` to expand merging/don't-care/zeroing variants according to the mask.
+- ``
+    - You may simply call upon wildcards defined under `substitutions`.
+### neon_type suffixes
+
+| suffix            | implication                                   |
+| ----------------- | --------------------------------------------- |
+| `.no`             | Normal                                        |
+| `.noq`            | Never include `q`s                            |
+| `.nox`            | Never include `_x`s                        |
+| `.N`              | Include `_n_`, e.g., `_n_s8`                  |
+| `.noq_N`          | Include `_n_`, but never `q`s                 |
+| `.dup`            | Include `_dup_`, e.g., `_dup_s8`              |
+| `.dup_nox`        | Include `_dup_` but never `_x`s            |
+| `.lane`           | Include `_lane_`, e.g., `_lane_s8`            |
+| `.lane_nox`       | Include `_lane_`, but never `_x`s          |
+| `.rot90`          | Include `_rot90_`, e.g., `_rot90_s8`          |
+| `.rot180`         | Include `_rot180_`, e.g., `_rot180_s8`        |
+| `.rot270`         | Include `_rot270_`, e.g., `_rot270_s8`        |
+| `.rot90_lane`     | Include `_rot90_lane_`                        |
+| `.rot180_lane`    | Include `_rot180_lane_`                       |
+| `.rot270_lane`    | Include `_rot270_lane_`                       |
+| `.rot90_laneq`    | Include `_rot90_laneq_`                       |
+| `.rot180_laneq`   | Include `_rot180_laneq_`                      |
+| `.rot270_laneq`   | Include `_rot270_laneq_`                      |
+| `.base`           | Produce only the size, e.g., `8`, `16`        |
+| `.u`              | Produce the type's unsigned equivalent        |
+| `.laneq_nox`      | Include `_laneq_`, but never `_x`s         |
+| `.tuple`          | Produce only the size of the tuple, e.g., `3` |
+| `.base_byte_size` | Produce only the size in bytes.               |
+ 

From 5bf9ca7b00c5074d984e836dcd9bef68fc9284cb Mon Sep 17 00:00:00 2001
From: Folkert de Vries 
Date: Tue, 13 Jan 2026 12:11:18 +0100
Subject: [PATCH 0227/1843] stabilize `__jscvt`

---
 .../stdarch/crates/core_arch/src/aarch64/neon/generated.rs  | 2 +-
 .../crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml       | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
index 7743e52f4fe1..90dba4409ed4 100644
--- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
+++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
@@ -49,7 +49,7 @@ pub fn __crc32d(crc: u32, data: u64) -> u32 {
 #[inline]
 #[target_feature(enable = "jsconv")]
 #[cfg_attr(test, assert_instr(fjcvtzs))]
-#[unstable(feature = "stdarch_aarch64_jscvt", issue = "147555")]
+#[stable(feature = "stdarch_aarch64_jscvt", since = "CURRENT_RUSTC_VERSION")]
 pub fn __jcvt(a: f64) -> i32 {
     unsafe extern "unadjusted" {
         #[cfg_attr(
diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml
index 8f36c525fae9..a951343e0183 100644
--- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml
+++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml
@@ -63,8 +63,8 @@ neon-unstable-f16: &neon-unstable-f16
 neon-unstable-feat-lut: &neon-unstable-feat-lut
   FnCall: [unstable, ['feature = "stdarch_neon_feat_lut"', 'issue = "138050"']]
 
-aarch64-unstable-jscvt: &aarch64-unstable-jscvt
-  FnCall: [unstable, ['feature = "stdarch_aarch64_jscvt"', 'issue = "147555"']]
+aarch64-stable-jscvt: &aarch64-stable-jscvt
+  FnCall: [stable, ['feature = "stdarch_aarch64_jscvt"', 'since = "CURRENT_RUSTC_VERSION"']]
 
 # #[cfg(target_endian = "little")]
 little-endian: &little-endian
@@ -14275,7 +14275,7 @@ intrinsics:
     attr:
       - FnCall: [target_feature, ['enable = "jsconv"']]
       - FnCall: [cfg_attr, [test, { FnCall: [assert_instr, ["fjcvtzs"]] }]]
-      - *aarch64-unstable-jscvt
+      - *aarch64-stable-jscvt
     safety: safe
     types:
       - f64

From 2c32c0e8047eb5535f96c3b7e468cfe023568b31 Mon Sep 17 00:00:00 2001
From: Chayim Refael Friedman 
Date: Tue, 13 Jan 2026 14:56:51 +0200
Subject: [PATCH 0228/1843] Make `naked_asm!()` always return `!`

As it should.
---
 .../rust-analyzer/crates/hir-ty/src/infer/expr.rs  |  8 ++++++--
 .../crates/hir-ty/src/tests/simple.rs              | 14 ++++++++++++++
 .../crates/test-utils/src/minicore.rs              |  4 ++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs
index 62339779a562..c57d41cc5f73 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer/expr.rs
@@ -8,7 +8,7 @@ use hir_def::{
     expr_store::path::{GenericArgs as HirGenericArgs, Path},
     hir::{
         Array, AsmOperand, AsmOptions, BinaryOp, BindingAnnotation, Expr, ExprId, ExprOrPatId,
-        LabelId, Literal, Pat, PatId, Statement, UnaryOp,
+        InlineAsmKind, LabelId, Literal, Pat, PatId, Statement, UnaryOp,
     },
     resolver::ValueNs,
 };
@@ -1037,7 +1037,11 @@ impl<'db> InferenceContext<'_, 'db> {
                     // FIXME: `sym` should report for things that are not functions or statics.
                     AsmOperand::Sym(_) => (),
                 });
-                if diverge { self.types.types.never } else { self.types.types.unit }
+                if diverge || asm.kind == InlineAsmKind::NakedAsm {
+                    self.types.types.never
+                } else {
+                    self.types.types.unit
+                }
             }
         };
         // use a new type variable if we got unknown here
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
index 6367521841ab..d02e455fc3dc 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
@@ -3983,3 +3983,17 @@ fn foo() {
         "#]],
     );
 }
+
+#[test]
+fn naked_asm_returns_never() {
+    check_no_mismatches(
+        r#"
+//- minicore: asm
+
+#[unsafe(naked)]
+extern "C" fn foo() -> ! {
+    core::arch::naked_asm!("");
+}
+    "#,
+    );
+}
diff --git a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
index 01274a9835f4..580a619cf108 100644
--- a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
+++ b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
@@ -1880,6 +1880,10 @@ mod arch {
     pub macro global_asm("assembly template", $(operands,)* $(options($(option),*))?) {
         /* compiler built-in */
     }
+    #[rustc_builtin_macro]
+    pub macro naked_asm("assembly template", $(operands,)* $(options($(option),*))?) {
+        /* compiler built-in */
+    }
 }
 // endregion:asm
 

From a2154802699c6de58702bcf4e3211ccb977cd3be Mon Sep 17 00:00:00 2001
From: A4-Tacks 
Date: Tue, 13 Jan 2026 19:00:03 +0800
Subject: [PATCH 0229/1843] Migrate `unwrap_block` assist to use SyntaxEditor

- Fix invalid match in let-stmt
- Fix multiple statements loses indent

Example
---
```rust
fn main() {
    let value = match rel_path {
        Ok(rel_path) => {$0
            let rel_path = RelativePathBuf::from_path(rel_path).ok()?;
            Some((*id, rel_path))
        }
        Err(_) => None,
    };
}
```

**Before this PR**

```rust
fn main() {
    let value = let rel_path = RelativePathBuf::from_path(rel_path).ok()?;
    let value = Some((*id, rel_path));
}
```

**After this PR**

```rust
fn main() {
    let rel_path = RelativePathBuf::from_path(rel_path).ok()?;
    let value = Some((*id, rel_path));
}
```

---

```rust
fn main() {
    let mut a = {$0
        1;
        2;
        3
    };
}
```

**Before this PR**

```rust
fn main() {
    1;
2;
    let mut a = 3;
}
```

**After this PR**

```rust
fn main() -> i32 {
    1;
    2;
    let mut a = 3;
}
```
---
 .../ide-assists/src/handlers/unwrap_block.rs  | 257 +++++++++---------
 1 file changed, 136 insertions(+), 121 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/unwrap_block.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/unwrap_block.rs
index a83f6835ca61..e4f5e3523bd2 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/unwrap_block.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/unwrap_block.rs
@@ -1,10 +1,12 @@
 use syntax::{
-    AstNode, SyntaxKind, T, TextRange,
+    AstNode, SyntaxElement, SyntaxKind, SyntaxNode, T,
     ast::{
         self,
         edit::{AstNodeEdit, IndentLevel},
         make,
     },
+    match_ast,
+    syntax_editor::{Element, Position, SyntaxEditor},
 };
 
 use crate::{AssistContext, AssistId, Assists};
@@ -27,123 +29,108 @@ use crate::{AssistContext, AssistId, Assists};
 // }
 // ```
 pub(crate) fn unwrap_block(acc: &mut Assists, ctx: &AssistContext<'_>) -> Option<()> {
-    let assist_id = AssistId::refactor_rewrite("unwrap_block");
-    let assist_label = "Unwrap block";
     let l_curly_token = ctx.find_token_syntax_at_offset(T!['{'])?;
-    let mut block = ast::BlockExpr::cast(l_curly_token.parent_ancestors().nth(1)?)?;
+    let block = l_curly_token.parent_ancestors().nth(1).and_then(ast::BlockExpr::cast)?;
     let target = block.syntax().text_range();
-    let mut parent = block.syntax().parent()?;
-    if ast::MatchArm::can_cast(parent.kind()) {
-        parent = parent.ancestors().find(|it| ast::MatchExpr::can_cast(it.kind()))?
-    }
+    let mut container = block.syntax().clone();
+    let mut replacement = block.clone();
+    let mut prefer_container = None;
 
-    let kind = parent.kind();
-    if matches!(kind, SyntaxKind::STMT_LIST | SyntaxKind::EXPR_STMT) {
-        acc.add(assist_id, assist_label, target, |builder| {
-            builder.replace(block.syntax().text_range(), update_expr_string(block.to_string()));
-        })
-    } else if matches!(kind, SyntaxKind::LET_STMT) {
-        let parent = ast::LetStmt::cast(parent)?;
-        let pattern = ast::Pat::cast(parent.syntax().first_child()?)?;
-        let ty = parent.ty();
-        let list = block.stmt_list()?;
-        let replaced = match list.syntax().last_child() {
-            Some(last) => {
-                let stmts: Vec = list.statements().collect();
-                let initializer = ast::Expr::cast(last)?;
-                let let_stmt = make::let_stmt(pattern, ty, Some(initializer));
-                if !stmts.is_empty() {
-                    let block = make::block_expr(stmts, None);
-                    format!("{}\n    {}", update_expr_string(block.to_string()), let_stmt)
-                } else {
-                    let_stmt.to_string()
-                }
-            }
-            None => {
-                let empty_tuple = make::ext::expr_unit();
-                make::let_stmt(pattern, ty, Some(empty_tuple)).to_string()
-            }
-        };
-        acc.add(assist_id, assist_label, target, |builder| {
-            builder.replace(parent.syntax().text_range(), replaced);
-        })
-    } else {
-        let parent = ast::Expr::cast(parent)?;
-        match parent.clone() {
-            ast::Expr::ForExpr(_) | ast::Expr::WhileExpr(_) | ast::Expr::LoopExpr(_) => (),
-            ast::Expr::MatchExpr(_) => block = block.dedent(IndentLevel(1)),
-            ast::Expr::IfExpr(if_expr) => {
-                let then_branch = if_expr.then_branch()?;
-                if then_branch == block {
-                    if let Some(ancestor) = if_expr.syntax().parent().and_then(ast::IfExpr::cast) {
-                        // For `else if` blocks
-                        let ancestor_then_branch = ancestor.then_branch()?;
-
-                        return acc.add(assist_id, assist_label, target, |edit| {
-                            let range_to_del_else_if = TextRange::new(
-                                ancestor_then_branch.syntax().text_range().end(),
-                                l_curly_token.text_range().start(),
-                            );
-                            let range_to_del_rest = TextRange::new(
-                                then_branch.syntax().text_range().end(),
-                                if_expr.syntax().text_range().end(),
-                            );
-
-                            edit.delete(range_to_del_rest);
-                            edit.delete(range_to_del_else_if);
-                            edit.replace(
-                                target,
-                                update_expr_string_without_newline(then_branch.to_string()),
-                            );
-                        });
-                    }
-                } else {
-                    return acc.add(assist_id, assist_label, target, |edit| {
-                        let range_to_del = TextRange::new(
-                            then_branch.syntax().text_range().end(),
-                            l_curly_token.text_range().start(),
-                        );
-
-                        edit.delete(range_to_del);
-                        edit.replace(target, update_expr_string_without_newline(block.to_string()));
+    let from_indent = block.indent_level();
+    let into_indent = loop {
+        let parent = container.parent()?;
+        container = match_ast! {
+            match parent {
+                ast::ForExpr(it) => it.syntax().clone(),
+                ast::LoopExpr(it) => it.syntax().clone(),
+                ast::WhileExpr(it) => it.syntax().clone(),
+                ast::MatchArm(it) => it.parent_match().syntax().clone(),
+                ast::LetStmt(it) => {
+                    replacement = wrap_let(&it, replacement);
+                    prefer_container = Some(it.syntax().clone());
+                    it.syntax().clone()
+                },
+                ast::IfExpr(it) => {
+                    prefer_container.get_or_insert_with(|| {
+                        if let Some(else_branch) = it.else_branch()
+                            && *else_branch.syntax() == container
+                        {
+                            else_branch.syntax().clone()
+                        } else {
+                            it.syntax().clone()
+                        }
                     });
-                }
+                    it.syntax().clone()
+                },
+                ast::ExprStmt(it) => it.syntax().clone(),
+                ast::StmtList(it) => break it.indent_level(),
+                _ => return None,
             }
-            _ => return None,
         };
+    };
+    let replacement = replacement.stmt_list()?;
 
-        acc.add(assist_id, assist_label, target, |builder| {
-            builder.replace(parent.syntax().text_range(), update_expr_string(block.to_string()));
-        })
-    }
+    acc.add(AssistId::refactor_rewrite("unwrap_block"), "Unwrap block", target, |builder| {
+        let mut edit = builder.make_editor(block.syntax());
+        let replacement = replacement.dedent(from_indent).indent(into_indent);
+        let container = prefer_container.unwrap_or(container);
+
+        edit.replace_with_many(&container, extract_statements(replacement));
+        delete_else_before(container, &mut edit);
+
+        builder.add_file_edits(ctx.vfs_file_id(), edit);
+    })
 }
 
-fn update_expr_string(expr_string: String) -> String {
-    update_expr_string_with_pat(expr_string, &[' ', '\n'])
+fn delete_else_before(container: SyntaxNode, edit: &mut SyntaxEditor) {
+    let Some(else_token) = container
+        .siblings_with_tokens(syntax::Direction::Prev)
+        .skip(1)
+        .map_while(|it| it.into_token())
+        .find(|it| it.kind() == T![else])
+    else {
+        return;
+    };
+    itertools::chain(else_token.prev_token(), else_token.next_token())
+        .filter(|it| it.kind() == SyntaxKind::WHITESPACE)
+        .for_each(|it| edit.delete(it));
+    let indent = IndentLevel::from_node(&container);
+    let newline = make::tokens::whitespace(&format!("\n{indent}"));
+    edit.replace(else_token, newline);
 }
 
-fn update_expr_string_without_newline(expr_string: String) -> String {
-    update_expr_string_with_pat(expr_string, &[' '])
+fn wrap_let(assign: &ast::LetStmt, replacement: ast::BlockExpr) -> ast::BlockExpr {
+    let try_wrap_assign = || {
+        let initializer = assign.initializer()?.syntax().syntax_element();
+        let replacement = replacement.clone_subtree();
+        let assign = assign.clone_for_update();
+        let tail_expr = replacement.tail_expr()?;
+        let before =
+            assign.syntax().children_with_tokens().take_while(|it| *it != initializer).collect();
+        let after = assign
+            .syntax()
+            .children_with_tokens()
+            .skip_while(|it| *it != initializer)
+            .skip(1)
+            .collect();
+
+        let mut edit = SyntaxEditor::new(replacement.syntax().clone());
+        edit.insert_all(Position::before(tail_expr.syntax()), before);
+        edit.insert_all(Position::after(tail_expr.syntax()), after);
+        ast::BlockExpr::cast(edit.finish().new_root().clone())
+    };
+    try_wrap_assign().unwrap_or(replacement)
 }
 
-fn update_expr_string_with_pat(expr_str: String, whitespace_pat: &[char]) -> String {
-    // Remove leading whitespace, index to remove the leading '{',
-    // then continue to remove leading whitespace.
-    // We cannot assume the `{` is the first character because there are block modifiers
-    // (`unsafe`, `async` etc.).
-    let after_open_brace_index = expr_str.find('{').map_or(0, |it| it + 1);
-    let expr_str = expr_str[after_open_brace_index..].trim_start_matches(whitespace_pat);
-
-    // Remove trailing whitespace, index [..expr_str.len() - 1] to remove the trailing '}',
-    // then continue to remove trailing whitespace.
-    let expr_str = expr_str.trim_end_matches(whitespace_pat);
-    let expr_str = expr_str[..expr_str.len() - 1].trim_end_matches(whitespace_pat);
-
-    expr_str
-        .lines()
-        .map(|line| line.replacen("    ", "", 1)) // Delete indentation
-        .collect::>()
-        .join("\n")
+fn extract_statements(stmt_list: ast::StmtList) -> Vec {
+    let mut elements = stmt_list
+        .syntax()
+        .children_with_tokens()
+        .filter(|it| !matches!(it.kind(), T!['{'] | T!['}']))
+        .skip_while(|it| it.kind() == SyntaxKind::WHITESPACE)
+        .collect::>();
+    while elements.pop_if(|it| it.kind() == SyntaxKind::WHITESPACE).is_some() {}
+    elements
 }
 
 #[cfg(test)]
@@ -593,6 +580,30 @@ fn main() {
         );
     }
 
+    #[test]
+    fn unwrap_match_arm_in_let() {
+        check_assist(
+            unwrap_block,
+            r#"
+fn main() {
+    let value = match rel_path {
+        Ok(rel_path) => {$0
+            let rel_path = RelativePathBuf::from_path(rel_path).ok()?;
+            Some((*id, rel_path))
+        }
+        Err(_) => None,
+    };
+}
+"#,
+            r#"
+fn main() {
+    let rel_path = RelativePathBuf::from_path(rel_path).ok()?;
+    let value = Some((*id, rel_path));
+}
+"#,
+        );
+    }
+
     #[test]
     fn simple_if_in_while_bad_cursor_position() {
         check_assist_not_applicable(
@@ -750,19 +761,6 @@ fn main() -> i32 {
         check_assist(
             unwrap_block,
             r#"
-fn main() {
-    let x = {$0};
-}
-"#,
-            r#"
-fn main() {
-    let x = ();
-}
-"#,
-        );
-        check_assist(
-            unwrap_block,
-            r#"
 fn main() {
     let x = {$0
         bar
@@ -784,8 +782,7 @@ fn main() -> i32 {
 "#,
             r#"
 fn main() -> i32 {
-    1;
-    let _ = 2;
+    1; let _ = 2;
 }
 "#,
         );
@@ -795,11 +792,29 @@ fn main() -> i32 {
 fn main() -> i32 {
     let mut a = {$01; 2};
 }
+"#,
+            r#"
+fn main() -> i32 {
+    1; let mut a = 2;
+}
+"#,
+        );
+        check_assist(
+            unwrap_block,
+            r#"
+fn main() -> i32 {
+    let mut a = {$0
+        1;
+        2;
+        3
+    };
+}
 "#,
             r#"
 fn main() -> i32 {
     1;
-    let mut a = 2;
+    2;
+    let mut a = 3;
 }
 "#,
         );

From 513b6ad231859f0b3c7953c454f9b209d58c25a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Tue, 13 Jan 2026 17:41:55 +0100
Subject: [PATCH 0230/1843] Remove references to homu and fix bors delegation
 command

---
 src/doc/rustc-dev-guide/src/about-this-guide.md |  4 ++--
 src/doc/rustc-dev-guide/src/compiler-team.md    |  4 ++--
 src/doc/rustc-dev-guide/src/contributing.md     |  2 +-
 src/doc/rustc-dev-guide/src/tests/ci.md         | 11 +++--------
 4 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/src/doc/rustc-dev-guide/src/about-this-guide.md b/src/doc/rustc-dev-guide/src/about-this-guide.md
index 2082481a200e..9d493e0cb065 100644
--- a/src/doc/rustc-dev-guide/src/about-this-guide.md
+++ b/src/doc/rustc-dev-guide/src/about-this-guide.md
@@ -103,9 +103,9 @@ You might also find the following sites useful:
 [tlgba]: https://tomlee.co/2014/04/a-more-detailed-tour-of-the-rust-compiler/
 [ro]: https://www.rustaceans.org/
 [rctd]: tests/intro.md
-[cheatsheet]: https://bors.rust-lang.org/
+[cheatsheet]: https://bors.rust-lang.org/help
 [Miri]: https://github.com/rust-lang/miri
-[@bors]: https://github.com/bors
+[@bors]: https://github.com/rust-lang/bors
 [a GitHub repository]: https://github.com/rust-lang/rustc-dev-guide/
 [rustc API docs]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle
 [Forge]: https://forge.rust-lang.org/
diff --git a/src/doc/rustc-dev-guide/src/compiler-team.md b/src/doc/rustc-dev-guide/src/compiler-team.md
index dfc1e7eedf62..495bd22da4d8 100644
--- a/src/doc/rustc-dev-guide/src/compiler-team.md
+++ b/src/doc/rustc-dev-guide/src/compiler-team.md
@@ -96,9 +96,9 @@ Once you have made a number of individual PRs to rustc, we will often
 offer r+ privileges. This means that you have the right to instruct
 "bors" (the robot that manages which PRs get landed into rustc) to
 merge a PR
-([here are some instructions for how to talk to bors][homu-guide]).
+([here are some instructions for how to talk to bors][bors-guide]).
 
-[homu-guide]: https://bors.rust-lang.org/
+[bors-guide]: https://bors.rust-lang.org/
 
 The guidelines for reviewers are as follows:
 
diff --git a/src/doc/rustc-dev-guide/src/contributing.md b/src/doc/rustc-dev-guide/src/contributing.md
index 40ad58e025c1..46d0dc23394a 100644
--- a/src/doc/rustc-dev-guide/src/contributing.md
+++ b/src/doc/rustc-dev-guide/src/contributing.md
@@ -283,7 +283,7 @@ this can take a while and the queue can sometimes be long.
 Also, note that PRs are never merged by hand.
 
 [@rustbot]: https://github.com/rustbot
-[@bors]: https://github.com/bors
+[@bors]: https://github.com/rust-lang/bors
 
 ### Opening a PR
 
diff --git a/src/doc/rustc-dev-guide/src/tests/ci.md b/src/doc/rustc-dev-guide/src/tests/ci.md
index e132946ae83e..ce80b07fe08d 100644
--- a/src/doc/rustc-dev-guide/src/tests/ci.md
+++ b/src/doc/rustc-dev-guide/src/tests/ci.md
@@ -198,7 +198,7 @@ to help make the perf comparison as fair as possible.
 >
 > 3. Run the prescribed try jobs with `@bors try`. As aforementioned, this
 >    requires the user to either (1) have `try` permissions or (2) be delegated
->    with `try` permissions by `@bors delegate` by someone who has `try`
+>    with `try` permissions by `@bors delegate=try` by someone who has `try`
 >    permissions.
 >
 > Note that this is usually easier to do than manually edit [`jobs.yml`].
@@ -213,10 +213,7 @@ the corresponding PR.
 Multiple try builds can execute concurrently across different PRs, but there can be at most
 a single try build running on a single PR at any given time.
 
-Note that try builds are handled using the [new bors] implementation.
-
 [rustc-perf]: https://github.com/rust-lang/rustc-perf
-[new bors]: https://github.com/rust-lang/bors
 
 ### Modifying CI jobs
 
@@ -281,8 +278,7 @@ Breakages like these usually happen when another, incompatible PR is merged
 after the build happened.
 
 To ensure a `main` branch that works all the time, we forbid manual merges.
-Instead, all PRs have to be approved through our bot, [bors] (the software
-behind it is called [homu]).
+Instead, all PRs have to be approved through our bot, [bors].
 All the approved PRs are put in a [merge queue]
 (sorted by priority and creation date) and are automatically tested one at the time.
 If all the builders are green, the PR is merged, otherwise the failure is
@@ -465,8 +461,7 @@ To do this:
 [`jobs.yml`]: https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml
 [`.github/workflows/ci.yml`]: https://github.com/rust-lang/rust/blob/HEAD/.github/workflows/ci.yml
 [`src/ci/citool`]: https://github.com/rust-lang/rust/blob/HEAD/src/ci/citool
-[bors]: https://github.com/bors
-[homu]: https://github.com/rust-lang/homu
+[bors]: https://github.com/rust-lang/bors
 [merge queue]: https://bors.rust-lang.org/queue/rust
 [dist-x86_64-linux]: https://github.com/rust-lang/rust/blob/HEAD/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
 [the GitHub Actions workflows page]: https://github.com/rust-lang/rust/actions

From 8fb704c17510f22700d4a84fc7151f67d9d20a27 Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Tue, 13 Jan 2026 19:57:37 +0100
Subject: [PATCH 0231/1843] fix: Hide renamed imports from macros in symbol
 index

---
 src/tools/rust-analyzer/crates/hir/src/symbols.rs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir/src/symbols.rs b/src/tools/rust-analyzer/crates/hir/src/symbols.rs
index 4461659f5c4e..c088f3aa0cc0 100644
--- a/src/tools/rust-analyzer/crates/hir/src/symbols.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/symbols.rs
@@ -5,8 +5,8 @@ use std::marker::PhantomData;
 use base_db::FxIndexSet;
 use either::Either;
 use hir_def::{
-    AdtId, AssocItemId, Complete, DefWithBodyId, ExternCrateId, HasModule, ImplId, Lookup, MacroId,
-    ModuleDefId, ModuleId, TraitId,
+    AdtId, AssocItemId, AstIdLoc, Complete, DefWithBodyId, ExternCrateId, HasModule, ImplId,
+    Lookup, MacroId, ModuleDefId, ModuleId, TraitId,
     db::DefDatabase,
     item_scope::{ImportId, ImportOrExternCrate, ImportOrGlob},
     nameres::crate_def_map,
@@ -169,6 +169,7 @@ impl<'a> SymbolCollector<'a> {
 
     fn collect_from_module(&mut self, module_id: ModuleId) {
         let collect_pub_only = self.collect_pub_only;
+        let is_block_module = module_id.is_block_module(self.db);
         let push_decl = |this: &mut Self, def: ModuleDefId, name, vis| {
             if collect_pub_only && vis != Visibility::Public {
                 return;
@@ -240,6 +241,10 @@ impl<'a> SymbolCollector<'a> {
             let source = import_child_source_cache
                 .entry(i.use_)
                 .or_insert_with(|| i.use_.child_source(this.db));
+            if is_block_module && source.file_id.is_macro() {
+                // Macros tend to generate a lot of imports, the user really won't care about them
+                return;
+            }
             let Some(use_tree_src) = source.value.get(i.idx) else { return };
             let rename = use_tree_src.rename().and_then(|rename| rename.name());
             let name_syntax = match rename {
@@ -276,6 +281,12 @@ impl<'a> SymbolCollector<'a> {
                     return;
                 }
                 let loc = i.lookup(this.db);
+                if is_block_module && loc.ast_id().file_id.is_macro() {
+                    // Macros (especially derivves) tend to generate renamed extern crate items,
+                    // the user really won't care about them
+                    return;
+                }
+
                 let source = loc.source(this.db);
                 let rename = source.value.rename().and_then(|rename| rename.name());
 

From 348bfe3e355a4479287d3940ca6b87f9be967040 Mon Sep 17 00:00:00 2001
From: Josh Stone 
Date: Tue, 13 Jan 2026 10:54:33 -0800
Subject: [PATCH 0232/1843] compiler: upgrade to hashbrown 0.16.1

See also #135634, #149159, and rust-lang/hashbrown#662.

This includes an in-tree upgrade of `indexmap` as well, which uses the
new `HashTable` buckets API internally, hopefully impacting performance
for the better!
---
 Cargo.lock                                 | 19 +++++++++++--------
 compiler/rustc_data_structures/Cargo.toml  |  4 ++--
 compiler/rustc_mir_transform/Cargo.toml    |  2 +-
 compiler/rustc_query_system/Cargo.toml     |  2 +-
 src/bootstrap/src/utils/proc_macro_deps.rs |  1 +
 5 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index bf28939ac87b..45d01c486faf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1670,9 +1670,12 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.16.0"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heck"
@@ -1950,12 +1953,12 @@ checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5"
 
 [[package]]
 name = "indexmap"
-version = "2.12.0"
+version = "2.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f"
+checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2"
 dependencies = [
  "equivalent",
- "hashbrown 0.16.0",
+ "hashbrown 0.16.1",
  "serde",
  "serde_core",
 ]
@@ -3730,7 +3733,7 @@ dependencies = [
  "either",
  "elsa",
  "ena",
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
  "indexmap",
  "jobserver",
  "libc",
@@ -4346,7 +4349,7 @@ name = "rustc_mir_transform"
 version = "0.0.0"
 dependencies = [
  "either",
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
  "itertools",
  "rustc_abi",
  "rustc_arena",
@@ -4557,7 +4560,7 @@ dependencies = [
 name = "rustc_query_system"
 version = "0.0.0"
 dependencies = [
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
  "parking_lot",
  "rustc_abi",
  "rustc_ast",
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index c8296e05f6bd..f358ffffb47d 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -10,7 +10,7 @@ bitflags = "2.4.1"
 either = "1.0"
 elsa = "1.11.0"
 ena = "0.14.3"
-indexmap = "2.4.0"
+indexmap = "2.12.1"
 jobserver_crate = { version = "0.1.28", package = "jobserver" }
 measureme = "12.0.1"
 parking_lot = "0.12"
@@ -31,7 +31,7 @@ tracing = "0.1"
 # tidy-alphabetical-end
 
 [dependencies.hashbrown]
-version = "0.15.2"
+version = "0.16.1"
 default-features = false
 features = ["nightly"] # for may_dangle
 
diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml
index 6df7a869ca28..22de197d374a 100644
--- a/compiler/rustc_mir_transform/Cargo.toml
+++ b/compiler/rustc_mir_transform/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
 either = "1"
-hashbrown = "0.15"
+hashbrown = { version = "0.16.1", default-features = false }
 itertools = "0.12"
 rustc_abi = { path = "../rustc_abi" }
 rustc_arena = { path = "../rustc_arena" }
diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml
index 7480ba03474f..73ab03576dec 100644
--- a/compiler/rustc_query_system/Cargo.toml
+++ b/compiler/rustc_query_system/Cargo.toml
@@ -25,6 +25,6 @@ tracing = "0.1"
 # tidy-alphabetical-end
 
 [dependencies.hashbrown]
-version = "0.15.2"
+version = "0.16.1"
 default-features = false
 features = ["nightly"] # for may_dangle
diff --git a/src/bootstrap/src/utils/proc_macro_deps.rs b/src/bootstrap/src/utils/proc_macro_deps.rs
index 71b6aacdebed..f1bf6e399fb1 100644
--- a/src/bootstrap/src/utils/proc_macro_deps.rs
+++ b/src/bootstrap/src/utils/proc_macro_deps.rs
@@ -22,6 +22,7 @@ pub static CRATES: &[&str] = &[
     "fluent-langneg",
     "fluent-syntax",
     "fnv",
+    "foldhash",
     "generic-array",
     "hashbrown",
     "heck",

From afa2260a9cea4592606353895e0f8eee88356198 Mon Sep 17 00:00:00 2001
From: Josh Stone 
Date: Tue, 13 Jan 2026 10:54:39 -0800
Subject: [PATCH 0233/1843] library: finish unspecializing `Copy`

---
 library/core/src/marker.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/library/core/src/marker.rs b/library/core/src/marker.rs
index 68f5649210de..da7ba167ef74 100644
--- a/library/core/src/marker.rs
+++ b/library/core/src/marker.rs
@@ -455,9 +455,6 @@ marker_impls! {
 /// [impls]: #implementors
 #[stable(feature = "rust1", since = "1.0.0")]
 #[lang = "copy"]
-// This is unsound, but required by `hashbrown`
-// FIXME(joboet): change `hashbrown` to use `TrivialClone`
-#[rustc_unsafe_specialization_marker]
 #[rustc_diagnostic_item = "Copy"]
 pub trait Copy: Clone {
     // Empty.

From 8f953dea3aad273bc5e1450be7a15449ad07d7bb Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Tue, 13 Jan 2026 20:30:38 +0100
Subject: [PATCH 0234/1843] feat: Implement support for `feature(new_range)`

---
 .../crates/hir-def/src/lang_item.rs           |   5 +
 .../crates/hir-expand/src/mod_path.rs         |   4 +
 .../rust-analyzer/crates/hir-ty/src/infer.rs  |  28 ++++-
 .../crates/hir-ty/src/tests/patterns.rs       | 118 +++++++++---------
 .../crates/hir-ty/src/tests/simple.rs         |  41 ++++--
 .../crates/hir/src/source_analyzer.rs         |  80 ++++++++----
 .../crates/intern/src/symbol/symbols.rs       |   6 +
 .../crates/test-utils/src/minicore.rs         |  92 +++++++++-----
 8 files changed, 246 insertions(+), 128 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs b/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs
index 092ff6e48671..51dd55301f44 100644
--- a/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs
+++ b/src/tools/rust-analyzer/crates/hir-def/src/lang_item.rs
@@ -499,6 +499,11 @@ language_item_table! { LangItems =>
     RangeToInclusive,        sym::RangeToInclusive,    StructId;
     RangeTo,                 sym::RangeTo,             StructId;
 
+    RangeFromCopy,           sym::RangeFromCopy,           StructId;
+    RangeInclusiveCopy,      sym::RangeInclusiveCopy,      StructId;
+    RangeCopy,               sym::RangeCopy,               StructId;
+    RangeToInclusiveCopy,    sym::RangeToInclusiveCopy,    StructId;
+
     String,                  sym::String,              StructId;
     CStr,                    sym::CStr,                StructId;
     Ordering,                sym::Ordering,            EnumId;
diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs b/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs
index 1712c28aa8ab..78228cf82e67 100644
--- a/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs
+++ b/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs
@@ -423,6 +423,10 @@ macro_rules! __known_path {
     (core::ops::RangeTo) => {};
     (core::ops::RangeToInclusive) => {};
     (core::ops::RangeInclusive) => {};
+    (core::range::Range) => {};
+    (core::range::RangeFrom) => {};
+    (core::range::RangeInclusive) => {};
+    (core::range::RangeToInclusive) => {};
     (core::future::Future) => {};
     (core::future::IntoFuture) => {};
     (core::fmt::Debug) => {};
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer.rs
index d527a4ae29c2..35d744e7d16b 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/infer.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer.rs
@@ -1815,18 +1815,34 @@ impl<'body, 'db> InferenceContext<'body, 'db> {
         Some(struct_.into())
     }
 
+    fn has_new_range_feature(&self) -> bool {
+        self.resolver.top_level_def_map().is_unstable_feature_enabled(&sym::new_range)
+    }
+
     fn resolve_range(&self) -> Option {
-        let struct_ = self.lang_items.Range?;
+        let struct_ = if self.has_new_range_feature() {
+            self.lang_items.RangeCopy?
+        } else {
+            self.lang_items.Range?
+        };
         Some(struct_.into())
     }
 
     fn resolve_range_inclusive(&self) -> Option {
-        let struct_ = self.lang_items.RangeInclusiveStruct?;
+        let struct_ = if self.has_new_range_feature() {
+            self.lang_items.RangeInclusiveCopy?
+        } else {
+            self.lang_items.RangeInclusiveStruct?
+        };
         Some(struct_.into())
     }
 
     fn resolve_range_from(&self) -> Option {
-        let struct_ = self.lang_items.RangeFrom?;
+        let struct_ = if self.has_new_range_feature() {
+            self.lang_items.RangeFromCopy?
+        } else {
+            self.lang_items.RangeFrom?
+        };
         Some(struct_.into())
     }
 
@@ -1836,7 +1852,11 @@ impl<'body, 'db> InferenceContext<'body, 'db> {
     }
 
     fn resolve_range_to_inclusive(&self) -> Option {
-        let struct_ = self.lang_items.RangeToInclusive?;
+        let struct_ = if self.has_new_range_feature() {
+            self.lang_items.RangeToInclusiveCopy?
+        } else {
+            self.lang_items.RangeToInclusive?
+        };
         Some(struct_.into())
     }
 
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/patterns.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/patterns.rs
index 0b776938c5b3..8c7d29f99371 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/patterns.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/patterns.rs
@@ -13,11 +13,11 @@ fn infer_pattern() {
             let a = z;
             let (c, d) = (1, "hello");
 
-            for (e, f) in some_iter {
+            for (e, f) in [(0, 1)] {
                 let g = e;
             }
 
-            if let [val] = opt {
+            if let [val] = [y] {
                 let h = val;
             }
 
@@ -33,7 +33,7 @@ fn infer_pattern() {
         "#,
         expect![[r#"
             8..9 'x': &'? i32
-            17..400 '{     ...o_x; }': ()
+            17..399 '{     ...o_x; }': ()
             27..28 'y': &'? i32
             31..32 'x': &'? i32
             42..44 '&z': &'? i32
@@ -47,58 +47,62 @@ fn infer_pattern() {
             82..94 '(1, "hello")': (i32, &'? str)
             83..84 '1': i32
             86..93 '"hello"': &'static str
-            101..151 'for (e...     }': fn into_iter<{unknown}>({unknown}) -> <{unknown} as IntoIterator>::IntoIter
-            101..151 'for (e...     }': <{unknown} as IntoIterator>::IntoIter
-            101..151 'for (e...     }': !
-            101..151 'for (e...     }': {unknown}
-            101..151 'for (e...     }': &'? mut {unknown}
-            101..151 'for (e...     }': fn next<{unknown}>(&'? mut {unknown}) -> Option<<{unknown} as Iterator>::Item>
-            101..151 'for (e...     }': Option<<{unknown} as Iterator>::Item>
-            101..151 'for (e...     }': ()
-            101..151 'for (e...     }': ()
-            101..151 'for (e...     }': ()
-            101..151 'for (e...     }': ()
-            105..111 '(e, f)': ({unknown}, {unknown})
-            106..107 'e': {unknown}
-            109..110 'f': {unknown}
-            115..124 'some_iter': {unknown}
-            125..151 '{     ...     }': ()
-            139..140 'g': {unknown}
-            143..144 'e': {unknown}
-            157..204 'if let...     }': ()
-            160..175 'let [val] = opt': bool
-            164..169 '[val]': [{unknown}]
-            165..168 'val': {unknown}
-            172..175 'opt': [{unknown}]
-            176..204 '{     ...     }': ()
-            190..191 'h': {unknown}
-            194..197 'val': {unknown}
-            210..236 'if let...rue {}': ()
-            213..233 'let x ... &true': bool
-            217..225 'x @ true': &'? bool
-            221..225 'true': bool
-            221..225 'true': bool
-            228..233 '&true': &'? bool
-            229..233 'true': bool
-            234..236 '{}': ()
-            246..252 'lambda': impl Fn(u64, u64, i32) -> i32
-            255..287 '|a: u6...b; c }': impl Fn(u64, u64, i32) -> i32
-            256..257 'a': u64
-            264..265 'b': u64
-            267..268 'c': i32
-            275..287 '{ a + b; c }': i32
-            277..278 'a': u64
-            277..282 'a + b': u64
-            281..282 'b': u64
-            284..285 'c': i32
-            298..310 'ref ref_to_x': &'? &'? i32
-            313..314 'x': &'? i32
-            324..333 'mut mut_x': &'? i32
-            336..337 'x': &'? i32
-            347..367 'ref mu...f_to_x': &'? mut &'? i32
-            370..371 'x': &'? i32
-            381..382 'k': &'? mut &'? i32
-            385..397 'mut_ref_to_x': &'? mut &'? i32
+            101..150 'for (e...     }': fn into_iter<[(i32, i32); 1]>([(i32, i32); 1]) -> <[(i32, i32); 1] as IntoIterator>::IntoIter
+            101..150 'for (e...     }': IntoIter<(i32, i32), 1>
+            101..150 'for (e...     }': !
+            101..150 'for (e...     }': IntoIter<(i32, i32), 1>
+            101..150 'for (e...     }': &'? mut IntoIter<(i32, i32), 1>
+            101..150 'for (e...     }': fn next>(&'? mut IntoIter<(i32, i32), 1>) -> Option< as Iterator>::Item>
+            101..150 'for (e...     }': Option<(i32, i32)>
+            101..150 'for (e...     }': ()
+            101..150 'for (e...     }': ()
+            101..150 'for (e...     }': ()
+            101..150 'for (e...     }': ()
+            105..111 '(e, f)': (i32, i32)
+            106..107 'e': i32
+            109..110 'f': i32
+            115..123 '[(0, 1)]': [(i32, i32); 1]
+            116..122 '(0, 1)': (i32, i32)
+            117..118 '0': i32
+            120..121 '1': i32
+            124..150 '{     ...     }': ()
+            138..139 'g': i32
+            142..143 'e': i32
+            156..203 'if let...     }': ()
+            159..174 'let [val] = [y]': bool
+            163..168 '[val]': [&'? i32; 1]
+            164..167 'val': &'? i32
+            171..174 '[y]': [&'? i32; 1]
+            172..173 'y': &'? i32
+            175..203 '{     ...     }': ()
+            189..190 'h': &'? i32
+            193..196 'val': &'? i32
+            209..235 'if let...rue {}': ()
+            212..232 'let x ... &true': bool
+            216..224 'x @ true': &'? bool
+            220..224 'true': bool
+            220..224 'true': bool
+            227..232 '&true': &'? bool
+            228..232 'true': bool
+            233..235 '{}': ()
+            245..251 'lambda': impl Fn(u64, u64, i32) -> i32
+            254..286 '|a: u6...b; c }': impl Fn(u64, u64, i32) -> i32
+            255..256 'a': u64
+            263..264 'b': u64
+            266..267 'c': i32
+            274..286 '{ a + b; c }': i32
+            276..277 'a': u64
+            276..281 'a + b': u64
+            280..281 'b': u64
+            283..284 'c': i32
+            297..309 'ref ref_to_x': &'? &'? i32
+            312..313 'x': &'? i32
+            323..332 'mut mut_x': &'? i32
+            335..336 'x': &'? i32
+            346..366 'ref mu...f_to_x': &'? mut &'? i32
+            369..370 'x': &'? i32
+            380..381 'k': &'? mut &'? i32
+            384..396 'mut_ref_to_x': &'? mut &'? i32
         "#]],
     );
 }
@@ -380,7 +384,7 @@ fn infer_pattern_match_string_literal() {
 fn infer_pattern_match_byte_string_literal() {
     check_infer_with_mismatches(
         r#"
-        //- minicore: index
+        //- minicore: index, range
         struct S;
         impl core::ops::Index for [T; N] {
             type Output = [u8];
@@ -395,7 +399,7 @@ fn infer_pattern_match_byte_string_literal() {
         "#,
         expect![[r#"
             105..109 'self': &'? [T; N]
-            111..116 'index': {unknown}
+            111..116 'index': RangeFull
             157..180 '{     ...     }': &'? [u8]
             167..174 'loop {}': !
             172..174 '{}': ()
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
index a9a5e96f75cc..988d1dc7f242 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
@@ -64,20 +64,37 @@ fn type_alias_in_struct_lit() {
 
 #[test]
 fn infer_ranges() {
-    check_types(
+    check_no_mismatches(
         r#"
-//- minicore: range
-fn test() {
-    let a = ..;
-    let b = 1..;
-    let c = ..2u32;
-    let d = 1..2usize;
-    let e = ..=10;
-    let f = 'a'..='z';
+//- minicore: range, new_range
 
-    let t = (a, b, c, d, e, f);
-    t;
-} //^ (RangeFull, RangeFrom, RangeTo, Range, RangeToInclusive, RangeInclusive)
+fn test() {
+    let _: core::ops::RangeFull = ..;
+    let _: core::ops::RangeFrom = 1..;
+    let _: core::ops::RangeTo = ..2u32;
+    let _: core::ops::Range = 1..2usize;
+    let _: core::ops::RangeToInclusive = ..=10;
+    let _: core::ops::RangeInclusive = 'a'..='z';
+}
+"#,
+    );
+}
+
+#[test]
+fn infer_ranges_new_range() {
+    check_no_mismatches(
+        r#"
+//- minicore: range, new_range
+#![feature(new_range)]
+
+fn test() {
+    let _: core::ops::RangeFull = ..;
+    let _: core::range::RangeFrom = 1..;
+    let _: core::ops::RangeTo = ..2u32;
+    let _: core::range::Range = 1..2usize;
+    let _: core::range::RangeToInclusive = ..=10;
+    let _: core::range::RangeInclusive = 'a'..='z';
+}
 "#,
     );
 }
diff --git a/src/tools/rust-analyzer/crates/hir/src/source_analyzer.rs b/src/tools/rust-analyzer/crates/hir/src/source_analyzer.rs
index 6ba7a42c1946..38e9c5b3f714 100644
--- a/src/tools/rust-analyzer/crates/hir/src/source_analyzer.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/source_analyzer.rs
@@ -531,18 +531,12 @@ impl<'db> SourceAnalyzer<'db> {
         db: &'db dyn HirDatabase,
         range_pat: &ast::RangePat,
     ) -> Option {
-        let path: ModPath = match (range_pat.op_kind()?, range_pat.start(), range_pat.end()) {
-            (RangeOp::Exclusive, None, Some(_)) => path![core::ops::RangeTo],
-            (RangeOp::Exclusive, Some(_), None) => path![core::ops::RangeFrom],
-            (RangeOp::Exclusive, Some(_), Some(_)) => path![core::ops::Range],
-            (RangeOp::Inclusive, None, Some(_)) => path![core::ops::RangeToInclusive],
-            (RangeOp::Inclusive, Some(_), Some(_)) => path![core::ops::RangeInclusive],
-
-            (RangeOp::Exclusive, None, None) => return None,
-            (RangeOp::Inclusive, None, None) => return None,
-            (RangeOp::Inclusive, Some(_), None) => return None,
-        };
-        self.resolver.resolve_known_struct(db, &path)
+        self.resolve_range_struct(
+            db,
+            range_pat.op_kind()?,
+            range_pat.start().is_some(),
+            range_pat.end().is_some(),
+        )
     }
 
     pub(crate) fn resolve_range_expr(
@@ -550,19 +544,59 @@ impl<'db> SourceAnalyzer<'db> {
         db: &'db dyn HirDatabase,
         range_expr: &ast::RangeExpr,
     ) -> Option {
-        let path: ModPath = match (range_expr.op_kind()?, range_expr.start(), range_expr.end()) {
-            (RangeOp::Exclusive, None, None) => path![core::ops::RangeFull],
-            (RangeOp::Exclusive, None, Some(_)) => path![core::ops::RangeTo],
-            (RangeOp::Exclusive, Some(_), None) => path![core::ops::RangeFrom],
-            (RangeOp::Exclusive, Some(_), Some(_)) => path![core::ops::Range],
-            (RangeOp::Inclusive, None, Some(_)) => path![core::ops::RangeToInclusive],
-            (RangeOp::Inclusive, Some(_), Some(_)) => path![core::ops::RangeInclusive],
+        self.resolve_range_struct(
+            db,
+            range_expr.op_kind()?,
+            range_expr.start().is_some(),
+            range_expr.end().is_some(),
+        )
+    }
 
+    fn resolve_range_struct(
+        &self,
+        db: &'db dyn HirDatabase,
+        op_kind: RangeOp,
+        has_start: bool,
+        has_end: bool,
+    ) -> Option {
+        let has_new_range =
+            self.resolver.top_level_def_map().is_unstable_feature_enabled(&sym::new_range);
+        let lang_items = self.lang_items(db);
+        match (op_kind, has_start, has_end) {
+            (RangeOp::Exclusive, false, false) => lang_items.RangeFull,
+            (RangeOp::Exclusive, false, true) => lang_items.RangeTo,
+            (RangeOp::Exclusive, true, false) => {
+                if has_new_range {
+                    lang_items.RangeFromCopy
+                } else {
+                    lang_items.RangeFrom
+                }
+            }
+            (RangeOp::Exclusive, true, true) => {
+                if has_new_range {
+                    lang_items.RangeCopy
+                } else {
+                    lang_items.Range
+                }
+            }
+            (RangeOp::Inclusive, false, true) => {
+                if has_new_range {
+                    lang_items.RangeToInclusiveCopy
+                } else {
+                    lang_items.RangeToInclusive
+                }
+            }
+            (RangeOp::Inclusive, true, true) => {
+                if has_new_range {
+                    lang_items.RangeInclusiveCopy
+                } else {
+                    lang_items.RangeInclusiveStruct
+                }
+            }
             // [E0586] inclusive ranges must be bounded at the end
-            (RangeOp::Inclusive, None, None) => return None,
-            (RangeOp::Inclusive, Some(_), None) => return None,
-        };
-        self.resolver.resolve_known_struct(db, &path)
+            (RangeOp::Inclusive, false, false) => None,
+            (RangeOp::Inclusive, true, false) => None,
+        }
     }
 
     pub(crate) fn resolve_await_to_poll(
diff --git a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs
index 781e58a1204e..513f51faba7f 100644
--- a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs
+++ b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs
@@ -525,6 +525,12 @@ define_symbols! {
     arbitrary_self_types,
     arbitrary_self_types_pointers,
     supertrait_item_shadowing,
+    new_range,
+    range,
+    RangeCopy,
+    RangeFromCopy,
+    RangeInclusiveCopy,
+    RangeToInclusiveCopy,
     hash,
     partial_cmp,
     cmp,
diff --git a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
index c3429356d9e5..26e125f8f2d6 100644
--- a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
+++ b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
@@ -58,6 +58,7 @@
 //!     pin:
 //!     pointee: copy, send, sync, ord, hash, unpin, phantom_data
 //!     range:
+//!     new_range:
 //!     receiver: deref
 //!     result:
 //!     send: sized
@@ -175,7 +176,9 @@ pub mod marker {
 
     // region:clone
     impl Clone for PhantomData {
-        fn clone(&self) -> Self { Self }
+        fn clone(&self) -> Self {
+            Self
+        }
     }
     // endregion:clone
 
@@ -1128,6 +1131,32 @@ pub mod ops {
     // endregion:dispatch_from_dyn
 }
 
+// region:new_range
+pub mod range {
+    #[lang = "RangeCopy"]
+    pub struct Range {
+        pub start: Idx,
+        pub end: Idx,
+    }
+
+    #[lang = "RangeFromCopy"]
+    pub struct RangeFrom {
+        pub start: Idx,
+    }
+
+    #[lang = "RangeInclusiveCopy"]
+    pub struct RangeInclusive {
+        pub start: Idx,
+        pub end: Idx,
+    }
+
+    #[lang = "RangeToInclusiveCopy"]
+    pub struct RangeToInclusive {
+        pub end: Idx,
+    }
+}
+// endregion:new_range
+
 // region:eq
 pub mod cmp {
     use crate::marker::PointeeSized;
@@ -1144,7 +1173,9 @@ pub mod cmp {
 
     // region:builtin_impls
     impl PartialEq for () {
-        fn eq(&self, other: &()) -> bool { true }
+        fn eq(&self, other: &()) -> bool {
+            true
+        }
     }
     // endregion:builtin_impls
 
@@ -1567,10 +1598,7 @@ pub mod pin {
     }
     // endregion:dispatch_from_dyn
     // region:coerce_unsized
-    impl crate::ops::CoerceUnsized> for Pin where
-        Ptr: crate::ops::CoerceUnsized
-    {
-    }
+    impl crate::ops::CoerceUnsized> for Pin where Ptr: crate::ops::CoerceUnsized {}
     // endregion:coerce_unsized
 }
 // endregion:pin
@@ -1792,9 +1820,9 @@ pub mod iter {
                 fn from_iter>(iter: T) -> Self;
             }
         }
-        pub use self::collect::{IntoIterator, FromIterator};
+        pub use self::collect::{FromIterator, IntoIterator};
     }
-    pub use self::traits::{IntoIterator, FromIterator, Iterator};
+    pub use self::traits::{FromIterator, IntoIterator, Iterator};
 }
 // endregion:iterator
 
@@ -2087,30 +2115,30 @@ macro_rules! column {
 pub mod prelude {
     pub mod v1 {
         pub use crate::{
-            clone::Clone,                            // :clone
-            cmp::{Eq, PartialEq},                    // :eq
-            cmp::{Ord, PartialOrd},                  // :ord
-            convert::AsMut,                          // :as_mut
-            convert::AsRef,                          // :as_ref
-            convert::{From, Into, TryFrom, TryInto}, // :from
-            default::Default,                        // :default
-            iter::{IntoIterator, Iterator, FromIterator}, // :iterator
-            macros::builtin::{derive, derive_const}, // :derive
-            marker::Copy,                            // :copy
-            marker::Send,                            // :send
-            marker::Sized,                           // :sized
-            marker::Sync,                            // :sync
-            mem::drop,                               // :drop
-            mem::size_of,                            // :size_of
-            ops::Drop,                               // :drop
-            ops::{AsyncFn, AsyncFnMut, AsyncFnOnce}, // :async_fn
-            ops::{Fn, FnMut, FnOnce},                // :fn
-            option::Option::{self, None, Some},      // :option
-            panic,                                   // :panic
-            result::Result::{self, Err, Ok},         // :result
-            str::FromStr,                            // :str
-            fmt::derive::Debug,                      // :fmt, derive
-            hash::derive::Hash,                      // :hash, derive
+            clone::Clone,                                 // :clone
+            cmp::{Eq, PartialEq},                         // :eq
+            cmp::{Ord, PartialOrd},                       // :ord
+            convert::AsMut,                               // :as_mut
+            convert::AsRef,                               // :as_ref
+            convert::{From, Into, TryFrom, TryInto},      // :from
+            default::Default,                             // :default
+            fmt::derive::Debug,                           // :fmt, derive
+            hash::derive::Hash,                           // :hash, derive
+            iter::{FromIterator, IntoIterator, Iterator}, // :iterator
+            macros::builtin::{derive, derive_const},      // :derive
+            marker::Copy,                                 // :copy
+            marker::Send,                                 // :send
+            marker::Sized,                                // :sized
+            marker::Sync,                                 // :sync
+            mem::drop,                                    // :drop
+            mem::size_of,                                 // :size_of
+            ops::Drop,                                    // :drop
+            ops::{AsyncFn, AsyncFnMut, AsyncFnOnce},      // :async_fn
+            ops::{Fn, FnMut, FnOnce},                     // :fn
+            option::Option::{self, None, Some},           // :option
+            panic,                                        // :panic
+            result::Result::{self, Err, Ok},              // :result
+            str::FromStr,                                 // :str
         };
     }
 

From 596ea17eae592b68a16a7661bf26c10a5155da89 Mon Sep 17 00:00:00 2001
From: rami3l 
Date: Tue, 13 Jan 2026 21:05:44 +0100
Subject: [PATCH 0235/1843] fix(build-manifest): enable docs target fallback
 for `rustc-docs`

---
 src/tools/build-manifest/src/main.rs     | 12 ++++++------
 src/tools/build-manifest/src/versions.rs |  5 +----
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 4cec1b1f164b..58f7487a0fda 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -14,11 +14,11 @@ use crate::versions::{PkgType, Versions};
 
 include!(concat!(env!("OUT_DIR"), "/targets.rs"));
 
-/// This allows the manifest to contain rust-docs for hosts that don't build
-/// docs.
+/// This allows the manifest to contain rust-docs and rustc-docs for hosts
+/// that don't build certain docs.
 ///
 /// Tuples of `(host_partial, host_instead)`. If the host does not have the
-/// rust-docs component available, then if the host name contains
+/// corresponding docs component available, then if the host name contains
 /// `host_partial`, it will use the docs from `host_instead` instead.
 ///
 /// The order here matters, more specific entries should be first.
@@ -392,9 +392,9 @@ impl Builder {
                     let t = Target::from_compressed_tar(self, &tarball_name!(fallback_target));
                     // Fallbacks should typically be available on 'production' builds
                     // but may not be available for try builds, which only build one target by
-                    // default. Ideally we'd gate this being a hard error on whether we're in a
-                    // production build or not, but it's not information that's readily available
-                    // here.
+                    // default. It is also possible that `rust-docs` and `rustc-docs` differ in
+                    // availability per target. Thus, we take the first available fallback we can
+                    // find.
                     if !t.available {
                         eprintln!(
                             "{:?} not available for fallback",
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 6ef8a0e83de3..da887a66d98e 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -133,10 +133,7 @@ impl PkgType {
 
     /// Whether to package these target-specific docs for another similar target.
     pub(crate) fn use_docs_fallback(&self) -> bool {
-        match self {
-            PkgType::JsonDocs | PkgType::HtmlDocs => true,
-            _ => false,
-        }
+        matches!(self, PkgType::JsonDocs | PkgType::HtmlDocs | PkgType::RustcDocs)
     }
 }
 

From 6948a66ad81e149d8a0af5dcc0888b9a44f3d2b4 Mon Sep 17 00:00:00 2001
From: Shoyu Vanilla 
Date: Wed, 14 Jan 2026 05:40:27 +0900
Subject: [PATCH 0236/1843] fix: Sync cast checks to rustc again

---
 .../crates/hir-ty/src/infer/cast.rs           | 270 +++++++++++++-----
 .../crates/hir-ty/src/infer/unify.rs          |  10 -
 .../src/handlers/invalid_cast.rs              |  10 +-
 3 files changed, 201 insertions(+), 89 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer/cast.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer/cast.rs
index d073b06ccc8a..d69b00adb7f7 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/infer/cast.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer/cast.rs
@@ -2,8 +2,10 @@
 
 use hir_def::{AdtId, hir::ExprId, signatures::TraitFlags};
 use rustc_ast_ir::Mutability;
+use rustc_hash::FxHashSet;
 use rustc_type_ir::{
-    Flags, InferTy, TypeFlags, UintTy,
+    InferTy, TypeVisitableExt, UintTy, elaborate,
+    error::TypeError,
     inherent::{AdtDef, BoundExistentialPredicates as _, IntoKind, Ty as _},
 };
 use stdx::never;
@@ -12,7 +14,10 @@ use crate::{
     InferenceDiagnostic,
     db::HirDatabase,
     infer::{AllowTwoPhase, InferenceContext, expr::ExprIsRead},
-    next_solver::{BoundExistentialPredicates, DbInterner, ParamTy, Ty, TyKind},
+    next_solver::{
+        BoundExistentialPredicates, ExistentialPredicate, ParamTy, Region, Ty, TyKind,
+        infer::traits::ObligationCause,
+    },
 };
 
 #[derive(Debug)]
@@ -66,12 +71,13 @@ pub enum CastError {
     DifferingKinds,
     SizedUnsizedCast,
     IllegalCast,
-    IntToFatCast,
+    IntToWideCast,
     NeedDeref,
     NeedViaPtr,
     NeedViaThinPtr,
     NeedViaInt,
     NonScalar,
+    PtrPtrAddingAutoTraits,
     // We don't want to report errors with unknown types currently.
     // UnknownCastPtrKind,
     // UnknownExprPtrKind,
@@ -137,22 +143,13 @@ impl<'db> CastCheck<'db> {
             return Ok(());
         }
 
-        if !self.cast_ty.flags().contains(TypeFlags::HAS_TY_INFER)
-            && !ctx.table.is_sized(self.cast_ty)
-        {
+        if !self.cast_ty.has_infer_types() && !ctx.table.is_sized(self.cast_ty) {
             return Err(InferenceDiagnostic::CastToUnsized {
                 expr: self.expr,
                 cast_ty: self.cast_ty.store(),
             });
         }
 
-        // Chalk doesn't support trait upcasting and fails to solve some obvious goals
-        // when the trait environment contains some recursive traits (See issue #18047)
-        // We skip cast checks for such cases for now, until the next-gen solver.
-        if contains_dyn_trait(self.cast_ty) {
-            return Ok(());
-        }
-
         self.do_check(ctx).map_err(|e| e.into_diagnostic(self.expr, self.expr_ty, self.cast_ty))
     }
 
@@ -162,22 +159,23 @@ impl<'db> CastCheck<'db> {
                 (Some(t_from), Some(t_cast)) => (t_from, t_cast),
                 (None, Some(t_cast)) => match self.expr_ty.kind() {
                     TyKind::FnDef(..) => {
-                        let sig =
-                            self.expr_ty.callable_sig(ctx.interner()).expect("FnDef had no sig");
-                        let sig = ctx.table.normalize_associated_types_in(sig);
+                        // rustc calls `FnCtxt::normalize` on this but it's a no-op in next-solver
+                        let sig = self.expr_ty.fn_sig(ctx.interner());
                         let fn_ptr = Ty::new_fn_ptr(ctx.interner(), sig);
-                        if ctx
-                            .coerce(
-                                self.source_expr.into(),
-                                self.expr_ty,
-                                fn_ptr,
-                                AllowTwoPhase::No,
-                                ExprIsRead::Yes,
-                            )
-                            .is_ok()
-                        {
-                        } else {
-                            return Err(CastError::IllegalCast);
+                        match ctx.coerce(
+                            self.source_expr.into(),
+                            self.expr_ty,
+                            fn_ptr,
+                            AllowTwoPhase::No,
+                            ExprIsRead::Yes,
+                        ) {
+                            Ok(_) => {}
+                            Err(TypeError::IntrinsicCast) => {
+                                return Err(CastError::IllegalCast);
+                            }
+                            Err(_) => {
+                                return Err(CastError::NonScalar);
+                            }
                         }
 
                         (CastTy::FnPtr, t_cast)
@@ -213,23 +211,41 @@ impl<'db> CastCheck<'db> {
         // rustc checks whether the `expr_ty` is foreign adt with `non_exhaustive` sym
 
         match (t_from, t_cast) {
+            // These types have invariants! can't cast into them.
             (_, CastTy::Int(Int::CEnum) | CastTy::FnPtr) => Err(CastError::NonScalar),
+
+            // * -> Bool
             (_, CastTy::Int(Int::Bool)) => Err(CastError::CastToBool),
-            (CastTy::Int(Int::U(UintTy::U8)), CastTy::Int(Int::Char)) => Ok(()),
+
+            // * -> Char
+            (CastTy::Int(Int::U(UintTy::U8)), CastTy::Int(Int::Char)) => Ok(()), // u8-char-cast
             (_, CastTy::Int(Int::Char)) => Err(CastError::CastToChar),
+
+            // prim -> float,ptr
             (CastTy::Int(Int::Bool | Int::CEnum | Int::Char), CastTy::Float) => {
                 Err(CastError::NeedViaInt)
             }
+
             (CastTy::Int(Int::Bool | Int::CEnum | Int::Char) | CastTy::Float, CastTy::Ptr(..))
             | (CastTy::Ptr(..) | CastTy::FnPtr, CastTy::Float) => Err(CastError::IllegalCast),
-            (CastTy::Ptr(src, _), CastTy::Ptr(dst, _)) => self.check_ptr_ptr_cast(ctx, src, dst),
+
+            // ptr -> ptr
+            (CastTy::Ptr(src, _), CastTy::Ptr(dst, _)) => self.check_ptr_ptr_cast(ctx, src, dst), // ptr-ptr-cast
+
+            // // ptr-addr-cast
             (CastTy::Ptr(src, _), CastTy::Int(_)) => self.check_ptr_addr_cast(ctx, src),
+            (CastTy::FnPtr, CastTy::Int(_)) => Ok(()),
+
+            // addr-ptr-cast
             (CastTy::Int(_), CastTy::Ptr(dst, _)) => self.check_addr_ptr_cast(ctx, dst),
+
+            // fn-ptr-cast
             (CastTy::FnPtr, CastTy::Ptr(dst, _)) => self.check_fptr_ptr_cast(ctx, dst),
+
+            // prim -> prim
             (CastTy::Int(Int::CEnum), CastTy::Int(_)) => Ok(()),
             (CastTy::Int(Int::Char | Int::Bool), CastTy::Int(_)) => Ok(()),
             (CastTy::Int(_) | CastTy::Float, CastTy::Int(_) | CastTy::Float) => Ok(()),
-            (CastTy::FnPtr, CastTy::Int(_)) => Ok(()),
         }
     }
 
@@ -241,10 +257,16 @@ impl<'db> CastCheck<'db> {
         t_cast: Ty<'db>,
         m_cast: Mutability,
     ) -> Result<(), CastError> {
-        // Mutability order is opposite to rustc. `Mut < Not`
-        if m_expr <= m_cast
+        let t_expr = ctx.table.try_structurally_resolve_type(t_expr);
+        let t_cast = ctx.table.try_structurally_resolve_type(t_cast);
+
+        if m_expr >= m_cast
             && let TyKind::Array(ety, _) = t_expr.kind()
+            && ctx.infcx().can_eq(ctx.table.param_env, ety, t_cast)
         {
+            // Due to historical reasons we allow directly casting references of
+            // arrays into raw pointers of their element type.
+
             // Coerce to a raw pointer so that we generate RawPtr in MIR.
             let array_ptr_type = Ty::new_ptr(ctx.interner(), t_expr, m_expr);
             if ctx
@@ -265,14 +287,9 @@ impl<'db> CastCheck<'db> {
                 );
             }
 
-            // This is a less strict condition than rustc's `demand_eqtype`,
-            // but false negative is better than false positive
-            if ctx
-                .coerce(self.source_expr.into(), ety, t_cast, AllowTwoPhase::No, ExprIsRead::Yes)
-                .is_ok()
-            {
-                return Ok(());
-            }
+            // this will report a type mismatch if needed
+            let _ = ctx.demand_eqtype(self.expr.into(), ety, t_cast);
+            return Ok(());
         }
 
         Err(CastError::IllegalCast)
@@ -289,30 +306,147 @@ impl<'db> CastCheck<'db> {
 
         match (src_kind, dst_kind) {
             (Some(PointerKind::Error), _) | (_, Some(PointerKind::Error)) => Ok(()),
+
             // (_, None) => Err(CastError::UnknownCastPtrKind),
             // (None, _) => Err(CastError::UnknownExprPtrKind),
             (_, None) | (None, _) => Ok(()),
+
+            // Cast to thin pointer is OK
             (_, Some(PointerKind::Thin)) => Ok(()),
+
+            // thin -> fat? report invalid cast (don't complain about vtable kinds)
             (Some(PointerKind::Thin), _) => Err(CastError::SizedUnsizedCast),
+
+            // trait object -> trait object? need to do additional checks
             (Some(PointerKind::VTable(src_tty)), Some(PointerKind::VTable(dst_tty))) => {
                 match (src_tty.principal_def_id(), dst_tty.principal_def_id()) {
+                    // A + SrcAuto> -> B + DstAuto>. need to make sure
+                    // - `Src` and `Dst` traits are the same
+                    // - traits have the same generic arguments
+                    // - projections are the same
+                    // - `SrcAuto` (+auto traits implied by `Src`) is a superset of `DstAuto`
+                    //
+                    // Note that trait upcasting goes through a different mechanism (`coerce_unsized`)
+                    // and is unaffected by this check.
                     (Some(src_principal), Some(dst_principal)) => {
                         if src_principal == dst_principal {
                             return Ok(());
                         }
-                        let src_principal = ctx.db.trait_signature(src_principal.0);
-                        let dst_principal = ctx.db.trait_signature(dst_principal.0);
-                        if src_principal.flags.contains(TraitFlags::AUTO)
-                            && dst_principal.flags.contains(TraitFlags::AUTO)
+
+                        // We need to reconstruct trait object types.
+                        // `m_src` and `m_dst` won't work for us here because they will potentially
+                        // contain wrappers, which we do not care about.
+                        //
+                        // e.g. we want to allow `dyn T -> (dyn T,)`, etc.
+                        //
+                        // We also need to skip auto traits to emit an FCW and not an error.
+                        let src_obj = Ty::new_dynamic(
+                            ctx.interner(),
+                            BoundExistentialPredicates::new_from_iter(
+                                ctx.interner(),
+                                src_tty.iter().filter(|pred| {
+                                    !matches!(
+                                        pred.skip_binder(),
+                                        ExistentialPredicate::AutoTrait(_)
+                                    )
+                                }),
+                            ),
+                            Region::new_erased(ctx.interner()),
+                        );
+                        let dst_obj = Ty::new_dynamic(
+                            ctx.interner(),
+                            BoundExistentialPredicates::new_from_iter(
+                                ctx.interner(),
+                                dst_tty.iter().filter(|pred| {
+                                    !matches!(
+                                        pred.skip_binder(),
+                                        ExistentialPredicate::AutoTrait(_)
+                                    )
+                                }),
+                            ),
+                            Region::new_erased(ctx.interner()),
+                        );
+
+                        // `dyn Src = dyn Dst`, this checks for matching traits/generics/projections
+                        // This is `fcx.demand_eqtype`, but inlined to give a better error.
+                        if ctx
+                            .table
+                            .at(&ObligationCause::dummy())
+                            .eq(src_obj, dst_obj)
+                            .map(|infer_ok| ctx.table.register_infer_ok(infer_ok))
+                            .is_err()
                         {
-                            Ok(())
-                        } else {
-                            Err(CastError::DifferingKinds)
+                            return Err(CastError::DifferingKinds);
                         }
+
+                        // Check that `SrcAuto` (+auto traits implied by `Src`) is a superset of `DstAuto`.
+                        // Emit an FCW otherwise.
+                        let src_auto: FxHashSet<_> = src_tty
+                            .auto_traits()
+                            .into_iter()
+                            .chain(
+                                elaborate::supertrait_def_ids(ctx.interner(), src_principal)
+                                    .filter(|trait_| {
+                                        ctx.db
+                                            .trait_signature(trait_.0)
+                                            .flags
+                                            .contains(TraitFlags::AUTO)
+                                    }),
+                            )
+                            .collect();
+
+                        let added = dst_tty
+                            .auto_traits()
+                            .into_iter()
+                            .any(|trait_| !src_auto.contains(&trait_));
+
+                        if added {
+                            return Err(CastError::PtrPtrAddingAutoTraits);
+                        }
+
+                        Ok(())
                     }
-                    _ => Err(CastError::Unknown),
+
+                    // dyn Auto -> dyn Auto'? ok.
+                    (None, None) => Ok(()),
+
+                    // dyn Trait -> dyn Auto? not ok (for now).
+                    //
+                    // Although dropping the principal is already allowed for unsizing coercions
+                    // (e.g. `*const (dyn Trait + Auto)` to `*const dyn Auto`), dropping it is
+                    // currently **NOT** allowed for (non-coercion) ptr-to-ptr casts (e.g
+                    // `*const Foo` to `*const Bar` where `Foo` has a `dyn Trait + Auto` tail
+                    // and `Bar` has a `dyn Auto` tail), because the underlying MIR operations
+                    // currently work very differently:
+                    //
+                    // * A MIR unsizing coercion on raw pointers to trait objects (`*const dyn Src`
+                    //   to `*const dyn Dst`) is currently equivalent to downcasting the source to
+                    //   the concrete sized type that it was originally unsized from first (via a
+                    //   ptr-to-ptr cast from `*const Src` to `*const T` with `T: Sized`) and then
+                    //   unsizing this thin pointer to the target type (unsizing `*const T` to
+                    //   `*const Dst`). In particular, this means that the pointer's metadata
+                    //   (vtable) will semantically change, e.g. for const eval and miri, even
+                    //   though the vtables will always be merged for codegen.
+                    //
+                    // * A MIR ptr-to-ptr cast is currently equivalent to a transmute and does not
+                    //   change the pointer metadata (vtable) at all.
+                    //
+                    // In addition to this potentially surprising difference between coercion and
+                    // non-coercion casts, casting away the principal with a MIR ptr-to-ptr cast
+                    // is currently considered undefined behavior:
+                    //
+                    // As a validity invariant of pointers to trait objects, we currently require
+                    // that the principal of the vtable in the pointer metadata exactly matches
+                    // the principal of the pointee type, where "no principal" is also considered
+                    // a kind of principal.
+                    (Some(_), None) => Err(CastError::DifferingKinds),
+
+                    // dyn Auto -> dyn Trait? not ok.
+                    (None, Some(_)) => Err(CastError::DifferingKinds),
                 }
             }
+
+            // fat -> fat? metadata kinds must match
             (Some(src_kind), Some(dst_kind)) if src_kind == dst_kind => Ok(()),
             (_, _) => Err(CastError::DifferingKinds),
         }
@@ -342,9 +476,9 @@ impl<'db> CastCheck<'db> {
             None => Ok(()),
             Some(PointerKind::Error) => Ok(()),
             Some(PointerKind::Thin) => Ok(()),
-            Some(PointerKind::VTable(_)) => Err(CastError::IntToFatCast),
-            Some(PointerKind::Length) => Err(CastError::IntToFatCast),
-            Some(PointerKind::OfAlias | PointerKind::OfParam(_)) => Err(CastError::IntToFatCast),
+            Some(PointerKind::VTable(_)) => Err(CastError::IntToWideCast),
+            Some(PointerKind::Length) => Err(CastError::IntToWideCast),
+            Some(PointerKind::OfAlias | PointerKind::OfParam(_)) => Err(CastError::IntToWideCast),
         }
     }
 
@@ -363,15 +497,20 @@ impl<'db> CastCheck<'db> {
     }
 }
 
+/// The kind of pointer and associated metadata (thin, length or vtable) - we
+/// only allow casts between wide pointers if their metadata have the same
+/// kind.
 #[derive(Debug, PartialEq, Eq)]
 enum PointerKind<'db> {
-    // thin pointer
+    /// No metadata attached, ie pointer to sized type or foreign type
     Thin,
-    // trait object
+    /// A trait object
     VTable(BoundExistentialPredicates<'db>),
-    // slice
+    /// Slice
     Length,
+    /// The unsize info of this projection or opaque type
     OfAlias,
+    /// The unsize info of this parameter
     OfParam(ParamTy),
     Error,
 }
@@ -439,24 +578,3 @@ fn pointer_kind<'db>(
         }
     }
 }
-
-fn contains_dyn_trait<'db>(ty: Ty<'db>) -> bool {
-    use std::ops::ControlFlow;
-
-    use rustc_type_ir::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
-
-    struct DynTraitVisitor;
-
-    impl<'db> TypeVisitor> for DynTraitVisitor {
-        type Result = ControlFlow<()>;
-
-        fn visit_ty(&mut self, ty: Ty<'db>) -> ControlFlow<()> {
-            match ty.kind() {
-                TyKind::Dynamic(..) => ControlFlow::Break(()),
-                _ => ty.super_visit_with(self),
-            }
-        }
-    }
-
-    ty.visit_with(&mut DynTraitVisitor).is_break()
-}
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer/unify.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer/unify.rs
index d55fc0ab0da6..2057159c46d2 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/infer/unify.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer/unify.rs
@@ -261,16 +261,6 @@ impl<'db> InferenceTable<'db> {
         self.infer_ctxt.canonicalize_response(t)
     }
 
-    // FIXME: We should get rid of this method. We cannot deeply normalize during inference, only when finishing.
-    // Inference should use shallow normalization (`try_structurally_resolve_type()`) only, when needed.
-    pub(crate) fn normalize_associated_types_in(&mut self, ty: T) -> T
-    where
-        T: TypeFoldable> + Clone,
-    {
-        let ty = self.resolve_vars_with_obligations(ty);
-        self.at(&ObligationCause::new()).deeply_normalize(ty.clone()).unwrap_or(ty)
-    }
-
     pub(crate) fn normalize_alias_ty(&mut self, alias: Ty<'db>) -> Ty<'db> {
         self.infer_ctxt
             .at(&ObligationCause::new(), self.param_env)
diff --git a/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/invalid_cast.rs b/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/invalid_cast.rs
index a59077b757b1..7479f8147d2e 100644
--- a/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/invalid_cast.rs
+++ b/src/tools/rust-analyzer/crates/ide-diagnostics/src/handlers/invalid_cast.rs
@@ -51,7 +51,7 @@ pub(crate) fn invalid_cast(ctx: &DiagnosticsContext<'_>, d: &hir::InvalidCast<'_
             DiagnosticCode::RustcHardError("E0606"),
             format_ty!(ctx, "casting `{}` as `{}` is invalid", d.expr_ty, d.cast_ty),
         ),
-        CastError::IntToFatCast => (
+        CastError::IntToWideCast => (
             DiagnosticCode::RustcHardError("E0606"),
             format_ty!(ctx, "cannot cast `{}` to a fat pointer `{}`", d.expr_ty, d.cast_ty),
         ),
@@ -95,6 +95,10 @@ pub(crate) fn invalid_cast(ctx: &DiagnosticsContext<'_>, d: &hir::InvalidCast<'_
             DiagnosticCode::RustcHardError("E0605"),
             format_ty!(ctx, "non-primitive cast: `{}` as `{}`", d.expr_ty, d.cast_ty),
         ),
+        CastError::PtrPtrAddingAutoTraits => (
+            DiagnosticCode::RustcHardError("E0804"),
+            "cannot add auto trait to dyn bound via pointer cast".to_owned(),
+        ),
         // CastError::UnknownCastPtrKind | CastError::UnknownExprPtrKind => (
         //     DiagnosticCode::RustcHardError("E0641"),
         //     "cannot cast to a pointer of an unknown kind".to_owned(),
@@ -444,8 +448,8 @@ fn main() {
     q as *const [i32];
   //^^^^^^^^^^^^^^^^^ error: cannot cast thin pointer `*const i32` to fat pointer `*const [i32]`
 
-    // FIXME: This should emit diagnostics but disabled to prevent many false positives
     let t: *mut (dyn Trait + 'static) = 0 as *mut _;
+                                      //^^^^^^^^^^^ error: cannot cast `usize` to a fat pointer `*mut (dyn Trait + 'static)`
 
     let mut fail: *const str = 0 as *const str;
                              //^^^^^^^^^^^^^^^ error: cannot cast `usize` to a fat pointer `*const str`
@@ -543,7 +547,7 @@ fn main() {
     fn ptr_to_trait_obj_ok() {
         check_diagnostics(
             r#"
-//- minicore: pointee
+//- minicore: pointee, send, sync
 trait Trait<'a> {}
 
 fn remove_auto<'a>(x: *mut (dyn Trait<'a> + Send)) -> *mut dyn Trait<'a> {

From 6166b619794ba095919deaad239628ad2ddce3b3 Mon Sep 17 00:00:00 2001
From: reddevilmidzy 
Date: Mon, 12 Jan 2026 00:45:50 +0900
Subject: [PATCH 0237/1843] Add crashes mGCA tests

---
 tests/crashes/138226-2.rs                     | 13 ++++++++
 tests/crashes/138226.rs                       | 13 ++++++++
 tests/crashes/149809.rs                       | 13 ++++++++
 tests/crashes/150960.rs                       | 11 +++++++
 .../mgca/cast-with-type-mismatched.rs         | 14 --------
 .../mgca/cast-with-type-mismatched.stderr     | 20 ------------
 .../mgca/const-eval-with-invalid-args.rs      | 26 ---------------
 .../mgca/const-eval-with-invalid-args.stderr  | 32 -------------------
 .../recursive-self-referencing-const-param.rs | 11 -------
 ...ursive-self-referencing-const-param.stderr | 23 -------------
 .../mgca/size-of-generic-ptr-in-array-len.rs  |  2 ++
 .../size-of-generic-ptr-in-array-len.stderr   |  8 ++++-
 12 files changed, 59 insertions(+), 127 deletions(-)
 create mode 100644 tests/crashes/138226-2.rs
 create mode 100644 tests/crashes/138226.rs
 create mode 100644 tests/crashes/149809.rs
 create mode 100644 tests/crashes/150960.rs
 delete mode 100644 tests/ui/const-generics/mgca/cast-with-type-mismatched.rs
 delete mode 100644 tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr
 delete mode 100644 tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs
 delete mode 100644 tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr
 delete mode 100644 tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs
 delete mode 100644 tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr

diff --git a/tests/crashes/138226-2.rs b/tests/crashes/138226-2.rs
new file mode 100644
index 000000000000..a2ebbdefdf3f
--- /dev/null
+++ b/tests/crashes/138226-2.rs
@@ -0,0 +1,13 @@
+//@ known-bug: #138226
+//@ needs-rustc-debug-assertions
+#![feature(min_generic_const_args)]
+#![feature(inherent_associated_types)]
+struct Bar;
+impl Bar {
+    #[type_const]
+    const LEN: usize = 4;
+
+    fn bar() {
+        let _ = [0; Self::LEN];
+    }
+}
diff --git a/tests/crashes/138226.rs b/tests/crashes/138226.rs
new file mode 100644
index 000000000000..7d13461a56b3
--- /dev/null
+++ b/tests/crashes/138226.rs
@@ -0,0 +1,13 @@
+//@ known-bug: #138226
+//@ needs-rustc-debug-assertions
+#![feature(min_generic_const_args)]
+#![feature(inherent_associated_types)]
+struct Foo(A, B);
+impl Foo {
+    #[type_const]
+    const LEN: usize = 4;
+
+    fn foo() {
+        let _ = [5; Self::LEN];
+    }
+}
diff --git a/tests/crashes/149809.rs b/tests/crashes/149809.rs
new file mode 100644
index 000000000000..2b948e9079c3
--- /dev/null
+++ b/tests/crashes/149809.rs
@@ -0,0 +1,13 @@
+//@ known-bug: #149809
+#![feature(min_generic_const_args)]
+#![feature(inherent_associated_types)]
+struct Qux<'a> {
+    x: &'a (),
+}
+impl<'a> Qux<'a> {
+    #[type_const]
+    const LEN: usize = 4;
+    fn foo(_: [u8; Qux::LEN]) {}
+}
+
+fn main() {}
diff --git a/tests/crashes/150960.rs b/tests/crashes/150960.rs
new file mode 100644
index 000000000000..2d46eea67989
--- /dev/null
+++ b/tests/crashes/150960.rs
@@ -0,0 +1,11 @@
+//@ known-bug: #150960
+#![feature(min_generic_const_args)]
+struct Baz;
+impl Baz {
+    #[type_const]
+    const LEN: usize = 4;
+
+    fn baz() {
+        let _ = [0; const { Self::LEN }];
+    }
+}
diff --git a/tests/ui/const-generics/mgca/cast-with-type-mismatched.rs b/tests/ui/const-generics/mgca/cast-with-type-mismatched.rs
deleted file mode 100644
index 1f499c222e7f..000000000000
--- a/tests/ui/const-generics/mgca/cast-with-type-mismatched.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-//! regression test for 
-#![expect(incomplete_features)]
-#![feature(generic_const_exprs)]
-#![feature(min_generic_const_args)]
-
-fn foo(a: [(); N as usize]) {}
-//~^ ERROR: complex const arguments must be placed inside of a `const` block
-
-const C: f32 = 1.0;
-
-fn main() {
-    foo::([]);
-    //~^ ERROR: the constant `C` is not of type `u32`
-}
diff --git a/tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr b/tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr
deleted file mode 100644
index 7e1b21138ec8..000000000000
--- a/tests/ui/const-generics/mgca/cast-with-type-mismatched.stderr
+++ /dev/null
@@ -1,20 +0,0 @@
-error: complex const arguments must be placed inside of a `const` block
-  --> $DIR/cast-with-type-mismatched.rs:6:30
-   |
-LL | fn foo(a: [(); N as usize]) {}
-   |                              ^^^^^^^^^^
-
-error: the constant `C` is not of type `u32`
-  --> $DIR/cast-with-type-mismatched.rs:12:11
-   |
-LL |     foo::([]);
-   |           ^ expected `u32`, found `f32`
-   |
-note: required by a const generic parameter in `foo`
-  --> $DIR/cast-with-type-mismatched.rs:6:8
-   |
-LL | fn foo(a: [(); N as usize]) {}
-   |        ^^^^^^^^^^^^ required by this const generic parameter in `foo`
-
-error: aborting due to 2 previous errors
-
diff --git a/tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs b/tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs
deleted file mode 100644
index 3bf951e0c212..000000000000
--- a/tests/ui/const-generics/mgca/const-eval-with-invalid-args.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-//! regression test for 
-#![expect(incomplete_features)]
-#![feature(generic_const_exprs)]
-#![feature(min_generic_const_args)]
-
-// The previous ICE was an "invalid field access on immediate".
-// If we remove `val: i32` from the field, another ICE occurs.
-// "assertion `left == right` failed: invalid field type in
-// Immediate::offset: scalar value has wrong size"
-struct A {
-    arr: usize,
-    val: i32,
-}
-
-struct B {
-    //~^ ERROR: `A` is forbidden as the type of a const generic parameter
-    arr: [u8; N.arr],
-    //~^ ERROR: complex const arguments must be placed inside of a `const` block
-}
-
-const C: u32 = 1;
-fn main() {
-    let b = B:: {arr: [1]};
-    //~^ ERROR: the constant `C` is not of type `A`
-    let _ = b.arr.len();
-}
diff --git a/tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr b/tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr
deleted file mode 100644
index a226718ccf1b..000000000000
--- a/tests/ui/const-generics/mgca/const-eval-with-invalid-args.stderr
+++ /dev/null
@@ -1,32 +0,0 @@
-error: complex const arguments must be placed inside of a `const` block
-  --> $DIR/const-eval-with-invalid-args.rs:17:15
-   |
-LL |     arr: [u8; N.arr],
-   |               ^^^^^
-
-error: `A` is forbidden as the type of a const generic parameter
-  --> $DIR/const-eval-with-invalid-args.rs:15:19
-   |
-LL | struct B {
-   |                   ^
-   |
-   = note: the only supported types are integers, `bool`, and `char`
-help: add `#![feature(adt_const_params)]` to the crate attributes to enable more complex and user defined types
-   |
-LL + #![feature(adt_const_params)]
-   |
-
-error: the constant `C` is not of type `A`
-  --> $DIR/const-eval-with-invalid-args.rs:23:17
-   |
-LL |     let b = B:: {arr: [1]};
-   |                 ^ expected `A`, found `u32`
-   |
-note: required by a const generic parameter in `B`
-  --> $DIR/const-eval-with-invalid-args.rs:15:10
-   |
-LL | struct B {
-   |          ^^^^^^^^^^ required by this const generic parameter in `B`
-
-error: aborting due to 3 previous errors
-
diff --git a/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs b/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs
deleted file mode 100644
index 37dcc58ff468..000000000000
--- a/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-//! regression test for 
-#![expect(incomplete_features)]
-#![feature(generic_const_exprs)]
-#![feature(min_generic_const_args)]
-
-fn identity }>() }>>();
-//~^ ERROR: free function without a body
-//~| ERROR: expected type, found function `identity`
-//~| ERROR: complex const arguments must be placed inside of a `const` block
-
-fn main() {}
diff --git a/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr b/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr
deleted file mode 100644
index d1899c476ec6..000000000000
--- a/tests/ui/const-generics/mgca/recursive-self-referencing-const-param.stderr
+++ /dev/null
@@ -1,23 +0,0 @@
-error: free function without a body
-  --> $DIR/recursive-self-referencing-const-param.rs:6:1
-   |
-LL | fn identity }>() }>>();
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-
-   |                                                                       |
-   |                                                                       help: provide a definition for the function: `{  }`
-
-error[E0573]: expected type, found function `identity`
-  --> $DIR/recursive-self-referencing-const-param.rs:6:22
-   |
-LL | fn identity }>() }>>();
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ not a type
-
-error: complex const arguments must be placed inside of a `const` block
-  --> $DIR/recursive-self-referencing-const-param.rs:6:57
-   |
-LL | fn identity }>() }>>();
-   |                                                         ^^
-
-error: aborting due to 3 previous errors
-
-For more information about this error, try `rustc --explain E0573`.
diff --git a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs
index 2a7c23929845..22963b6438c0 100644
--- a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs
+++ b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.rs
@@ -5,6 +5,8 @@
 fn foo() {
     [0; size_of::<*mut T>()];
     //~^ ERROR: tuple constructor with invalid base path
+    [0; const { size_of::<*mut T>() }];
+    //~^ ERROR: generic parameters may not be used in const operations
 }
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
index dcdc56a1cf47..913d8195fe21 100644
--- a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
+++ b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
@@ -4,5 +4,11 @@ error: tuple constructor with invalid base path
 LL |     [0; size_of::<*mut T>()];
    |         ^^^^^^^^^^^^^^^^^^^
 
-error: aborting due to 1 previous error
+error: generic parameters may not be used in const operations
+  --> $DIR/size-of-generic-ptr-in-array-len.rs:8:32
+   |
+LL |     [0; const { size_of::<*mut T>() }];
+   |                                ^
+
+error: aborting due to 2 previous errors
 

From 6c4cdfaf42bef13a33935835e09bedd353791af6 Mon Sep 17 00:00:00 2001
From: Samuel Tardieu 
Date: Tue, 13 Jan 2026 23:11:42 +0100
Subject: [PATCH 0238/1843] =?UTF-8?q?Replace=20complex=20`matches!(?=
 =?UTF-8?q?=E2=80=A6,=20=E2=80=A6=20if=20=E2=80=A6)`=20by=20`if=20let`=20c?=
 =?UTF-8?q?hain?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 clippy_lints/src/slow_vector_initialization.rs | 4 +++-
 clippy_lints/src/unnested_or_patterns.rs       | 7 ++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clippy_lints/src/slow_vector_initialization.rs b/clippy_lints/src/slow_vector_initialization.rs
index b25fa0905feb..8524497c387c 100644
--- a/clippy_lints/src/slow_vector_initialization.rs
+++ b/clippy_lints/src/slow_vector_initialization.rs
@@ -150,7 +150,9 @@ impl SlowVectorInit {
             && func.ty_rel_def(cx).is_diag_item(cx, sym::vec_with_capacity)
         {
             Some(InitializedSize::Initialized(len_expr))
-        } else if matches!(expr.kind, ExprKind::Call(func, []) if func.ty_rel_def(cx).is_diag_item(cx, sym::vec_new)) {
+        } else if let ExprKind::Call(func, []) = expr.kind
+            && func.ty_rel_def(cx).is_diag_item(cx, sym::vec_new)
+        {
             Some(InitializedSize::Uninitialized)
         } else {
             None
diff --git a/clippy_lints/src/unnested_or_patterns.rs b/clippy_lints/src/unnested_or_patterns.rs
index 975dd332ad06..12f8bd50e144 100644
--- a/clippy_lints/src/unnested_or_patterns.rs
+++ b/clippy_lints/src/unnested_or_patterns.rs
@@ -152,7 +152,12 @@ fn insert_necessary_parens(pat: &mut Pat) {
             walk_pat(self, pat);
             let target = match &mut pat.kind {
                 // `i @ a | b`, `box a | b`, and `& mut? a | b`.
-                Ident(.., Some(p)) | Box(p) | Ref(p, _, _) if matches!(&p.kind, Or(ps) if ps.len() > 1) => p,
+                Ident(.., Some(p)) | Box(p) | Ref(p, _, _)
+                    if let Or(ps) = &p.kind
+                        && ps.len() > 1 =>
+                {
+                    p
+                },
                 // `&(mut x)`
                 Ref(p, Pinnedness::Not, Mutability::Not) if matches!(p.kind, Ident(BindingMode::MUT, ..)) => p,
                 _ => return,

From 03cc50fc57424001285b72c02af2e19f36aabfd5 Mon Sep 17 00:00:00 2001
From: Josh Stone 
Date: Tue, 13 Jan 2026 15:23:59 -0800
Subject: [PATCH 0239/1843] update to indexmap v2.13.0

---
 Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 45d01c486faf..4297c1b32e35 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1953,9 +1953,9 @@ checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5"
 
 [[package]]
 name = "indexmap"
-version = "2.12.1"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2"
+checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
 dependencies = [
  "equivalent",
  "hashbrown 0.16.1",

From b202eee0816938126195e95488ce6df8e9602214 Mon Sep 17 00:00:00 2001
From: Taeyoon Kim 
Date: Wed, 14 Jan 2026 10:06:39 +0900
Subject: [PATCH 0240/1843] Add Korean translation to Rust By Example

Add korean translation. Thanks in advanced
---
 src/bootstrap/src/core/build_steps/doc.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
index b86582807f72..9bb16fe33fd5 100644
--- a/src/bootstrap/src/core/build_steps/doc.rs
+++ b/src/bootstrap/src/core/build_steps/doc.rs
@@ -75,7 +75,7 @@ book!(
     EditionGuide, "src/doc/edition-guide", "edition-guide", &[];
     EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[];
     Nomicon, "src/doc/nomicon", "nomicon", &[];
-    RustByExample, "src/doc/rust-by-example", "rust-by-example", &["es", "ja", "zh"];
+    RustByExample, "src/doc/rust-by-example", "rust-by-example", &["es", "ja", "zh", "ko"];
     RustdocBook, "src/doc/rustdoc", "rustdoc", &[];
     StyleGuide, "src/doc/style-guide", "style-guide", &[];
 );

From 795745ccf1f7af296334204931ae1d0e467d62c2 Mon Sep 17 00:00:00 2001
From: Nicholas Bishop 
Date: Tue, 13 Jan 2026 19:48:38 -0500
Subject: [PATCH 0241/1843] remote-test-server: Fix compilation on UEFI targets

Tested with:
./x build src/tools/remote-test-server --target x86_64-unknown-uefi
---
 src/tools/remote-test-server/src/main.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/tools/remote-test-server/src/main.rs b/src/tools/remote-test-server/src/main.rs
index bfe8f54937f6..a15f30d4659e 100644
--- a/src/tools/remote-test-server/src/main.rs
+++ b/src/tools/remote-test-server/src/main.rs
@@ -10,13 +10,13 @@
 //! themselves having support libraries. All data over the TCP sockets is in a
 //! basically custom format suiting our needs.
 
-#[cfg(all(not(windows), not(target_os = "motor")))]
+#[cfg(not(any(windows, target_os = "motor", target_os = "uefi")))]
 use std::fs::Permissions;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::io::{self, BufReader};
 use std::net::{SocketAddr, TcpListener, TcpStream};
-#[cfg(all(not(windows), not(target_os = "motor")))]
+#[cfg(not(any(windows, target_os = "motor", target_os = "uefi")))]
 use std::os::unix::prelude::*;
 use std::path::{Path, PathBuf};
 use std::process::{Command, ExitStatus, Stdio};
@@ -325,7 +325,7 @@ fn handle_run(socket: TcpStream, work: &Path, tmp: &Path, lock: &Mutex<()>, conf
     ]));
 }
 
-#[cfg(all(not(windows), not(target_os = "motor")))]
+#[cfg(not(any(windows, target_os = "motor", target_os = "uefi")))]
 fn get_status_code(status: &ExitStatus) -> (u8, i32) {
     match status.code() {
         Some(n) => (0, n),
@@ -333,7 +333,7 @@ fn get_status_code(status: &ExitStatus) -> (u8, i32) {
     }
 }
 
-#[cfg(any(windows, target_os = "motor"))]
+#[cfg(any(windows, target_os = "motor", target_os = "uefi"))]
 fn get_status_code(status: &ExitStatus) -> (u8, i32) {
     (0, status.code().unwrap())
 }
@@ -359,11 +359,11 @@ fn recv(dir: &Path, io: &mut B) -> PathBuf {
     dst
 }
 
-#[cfg(all(not(windows), not(target_os = "motor")))]
+#[cfg(not(any(windows, target_os = "motor", target_os = "uefi")))]
 fn set_permissions(path: &Path) {
     t!(fs::set_permissions(&path, Permissions::from_mode(0o755)));
 }
-#[cfg(any(windows, target_os = "motor"))]
+#[cfg(any(windows, target_os = "motor", target_os = "uefi"))]
 fn set_permissions(_path: &Path) {}
 
 fn my_copy(src: &mut dyn Read, which: u8, dst: &Mutex) {

From cd6c41283838023ee3b22709106d666869ec9a6f Mon Sep 17 00:00:00 2001
From: Zachary S 
Date: Tue, 13 Jan 2026 22:40:12 -0600
Subject: [PATCH 0242/1843] Add test for feature-gating `mut ref` patterns in
 struct field shorthand.

---
 tests/ui/feature-gates/feature-gate-mut-ref.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/ui/feature-gates/feature-gate-mut-ref.rs b/tests/ui/feature-gates/feature-gate-mut-ref.rs
index 752ae35d8a9a..eac2a59002e2 100644
--- a/tests/ui/feature-gates/feature-gate-mut-ref.rs
+++ b/tests/ui/feature-gates/feature-gate-mut-ref.rs
@@ -10,4 +10,7 @@ fn main() {
     let mut ref x = 10; //~  ERROR [E0658]
     #[cfg(false)]
     let mut ref mut y = 10; //~  ERROR [E0658]
+
+    struct Foo { x: i32 }
+    let Foo { mut ref x } = Foo { x: 10 };
 }

From f809e332d895bc80d28d8d2cb10297a3fe9ca4f2 Mon Sep 17 00:00:00 2001
From: Zachary S 
Date: Tue, 13 Jan 2026 12:57:04 -0600
Subject: [PATCH 0243/1843] Feature-gate `mut ref` patterns in struct field
 shorthand.

---
 compiler/rustc_parse/src/parser/pat.rs             |  6 ++++++
 tests/ui/feature-gates/feature-gate-mut-ref.rs     |  2 +-
 tests/ui/feature-gates/feature-gate-mut-ref.stderr | 12 +++++++++++-
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index d7f3a36122e5..f3e12d48a6a5 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -1750,6 +1750,12 @@ impl<'a> Parser<'a> {
             hi = self.prev_token.span;
             let ann = BindingMode(by_ref, mutability);
             let fieldpat = self.mk_pat_ident(boxed_span.to(hi), ann, fieldname);
+            if matches!(
+                fieldpat.kind,
+                PatKind::Ident(BindingMode(ByRef::Yes(..), Mutability::Mut), ..)
+            ) {
+                self.psess.gated_spans.gate(sym::mut_ref, fieldpat.span);
+            }
             let subpat = if is_box {
                 self.mk_pat(lo.to(hi), PatKind::Box(Box::new(fieldpat)))
             } else {
diff --git a/tests/ui/feature-gates/feature-gate-mut-ref.rs b/tests/ui/feature-gates/feature-gate-mut-ref.rs
index eac2a59002e2..74b1fba5dfea 100644
--- a/tests/ui/feature-gates/feature-gate-mut-ref.rs
+++ b/tests/ui/feature-gates/feature-gate-mut-ref.rs
@@ -12,5 +12,5 @@ fn main() {
     let mut ref mut y = 10; //~  ERROR [E0658]
 
     struct Foo { x: i32 }
-    let Foo { mut ref x } = Foo { x: 10 };
+    let Foo { mut ref x } = Foo { x: 10 }; //~  ERROR [E0658]
 }
diff --git a/tests/ui/feature-gates/feature-gate-mut-ref.stderr b/tests/ui/feature-gates/feature-gate-mut-ref.stderr
index d3eb674e92de..921ff878bf4d 100644
--- a/tests/ui/feature-gates/feature-gate-mut-ref.stderr
+++ b/tests/ui/feature-gates/feature-gate-mut-ref.stderr
@@ -38,6 +38,16 @@ LL |     let mut ref mut y = 10;
    = help: add `#![feature(mut_ref)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error: aborting due to 4 previous errors
+error[E0658]: mutable by-reference bindings are experimental
+  --> $DIR/feature-gate-mut-ref.rs:15:15
+   |
+LL |     let Foo { mut ref x } = Foo { x: 10 };
+   |               ^^^^^^^^^
+   |
+   = note: see issue #123076  for more information
+   = help: add `#![feature(mut_ref)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 5 previous errors
 
 For more information about this error, try `rustc --explain E0658`.

From c9a063ced93ec7dce1dfaca57f818dba5334cfca Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Wed, 14 Jan 2026 10:30:08 +1100
Subject: [PATCH 0244/1843] Remove `Deref`/`DerefMut` impl for `Providers`.

It's described as a "backwards compatibility hack to keep the diff
small". Removing it requires only a modest amount of churn, and the
resulting code is clearer without the invisible derefs.
---
 src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib.rs b/src/lib.rs
index 96d3a0024f41..cf1be1806235 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -286,7 +286,8 @@ impl CodegenBackend for GccCodegenBackend {
     }
 
     fn provide(&self, providers: &mut Providers) {
-        providers.global_backend_features = |tcx, ()| gcc_util::global_gcc_features(tcx.sess)
+        providers.queries.global_backend_features =
+            |tcx, ()| gcc_util::global_gcc_features(tcx.sess)
     }
 
     fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box {

From 05939a8d38959b1ff418a0f54663eaba0f5e3108 Mon Sep 17 00:00:00 2001
From: A4-Tacks 
Date: Wed, 14 Jan 2026 17:54:23 +0800
Subject: [PATCH 0245/1843] Fix false positive precedence in `(2 as i32) < 3`

Example
---
```rust
fn f() { _ = $0(1 as u32) << 10; }
```

**Before this PR**

This is syntax error

```rust
fn f() { _ = 1 as u32 << 10; }
```

**After this PR**

Assist not applicable
---
 .../src/handlers/remove_parentheses.rs        |  6 +++++
 .../crates/syntax/src/ast/prec.rs             | 24 +++++++++++++++----
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/remove_parentheses.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/remove_parentheses.rs
index aa4d2bcadb01..f07da489e23a 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/remove_parentheses.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/remove_parentheses.rs
@@ -321,6 +321,12 @@ mod tests {
         );
     }
 
+    #[test]
+    fn remove_parens_conflict_cast_before_l_angle() {
+        check_assist_not_applicable(remove_parentheses, r#"fn f() { _ = $0(1 as u32) << 10; }"#);
+        check_assist_not_applicable(remove_parentheses, r#"fn f() { _ = $0(1 as u32) < 10; }"#);
+    }
+
     #[test]
     fn remove_parens_double_paren_stmt() {
         check_assist(
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs
index 8c88224a761a..d99cf492616e 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs
@@ -154,6 +154,11 @@ fn check_ancestry(ancestor: &SyntaxNode, descendent: &SyntaxNode) -> bool {
     bail()
 }
 
+fn next_token_of(node: &SyntaxNode) -> Option {
+    let last = node.last_token()?;
+    skip_trivia_token(last.next_token()?, Direction::Next)
+}
+
 impl Expr {
     pub fn precedence(&self) -> ExprPrecedence {
         precedence(self)
@@ -197,6 +202,8 @@ impl Expr {
         if is_parent_call_expr && is_field_expr {
             return true;
         }
+        let place_of_parent =
+            || place_of.ancestors().find(|it| it.parent().is_none_or(|p| &p == parent.syntax()));
 
         // Special-case block weirdness
         if parent.child_is_followed_by_a_block() {
@@ -226,15 +233,24 @@ impl Expr {
         // For `&&`, we avoid introducing ` && ` into a binary chain.
 
         if self.precedence() == ExprPrecedence::Jump
-            && let Some(node) =
-                place_of.ancestors().find(|it| it.parent().is_none_or(|p| &p == parent.syntax()))
-            && let Some(next) =
-                node.last_token().and_then(|t| skip_trivia_token(t.next_token()?, Direction::Next))
+            && let Some(node) = place_of_parent()
+            && let Some(next) = next_token_of(&node)
             && matches!(next.kind(), T![||] | T![&&])
         {
             return true;
         }
 
+        // Special-case `2 as x < 3`
+        if let ast::Expr::CastExpr(it) = self
+            && let Some(ty) = it.ty()
+            && ty.syntax().last_token().and_then(|it| ast::NameLike::cast(it.parent()?)).is_some()
+            && let Some(node) = place_of_parent()
+            && let Some(next) = next_token_of(&node)
+            && matches!(next.kind(), T![<] | T![<<])
+        {
+            return true;
+        }
+
         if self.is_paren_like()
             || parent.is_paren_like()
             || self.is_prefix()

From bc751adcdb7cd5cce59a6aec36e74b959132e0af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcelo=20Dom=C3=ADnguez?= 
Date: Sun, 4 Jan 2026 17:26:21 +0100
Subject: [PATCH 0246/1843] Minor doc and ty fixes

---
 compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
index b8eb4f038216..7817755dafe4 100644
--- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
+++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
@@ -445,9 +445,8 @@ fn declare_offload_fn<'ll>(
 // the gpu. For now, we only handle the data transfer part of it.
 // If two consecutive kernels use the same memory, we still move it to the host and back to the gpu.
 // Since in our frontend users (by default) don't have to specify data transfer, this is something
-// we should optimize in the future! We also assume that everything should be copied back and forth,
-// but sometimes we can directly zero-allocate on the device and only move back, or if something is
-// immutable, we might only copy it to the device, but not back.
+// we should optimize in the future! In some cases we can directly zero-allocate ont he device and
+// only move data back, or if something is immutable, we might only copy it to the device.
 //
 // Current steps:
 // 0. Alloca some variables for the following steps

From 2c9c5d14a210767f83358c6cb17c7a81ec7ae41c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcelo=20Dom=C3=ADnguez?= 
Date: Tue, 6 Jan 2026 19:14:14 +0100
Subject: [PATCH 0247/1843] Allow bounded types

---
 compiler/rustc_codegen_llvm/src/intrinsic.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 481f75f337d6..59cbcd78dd0f 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -1388,7 +1388,8 @@ fn codegen_offload<'ll, 'tcx>(
     let args = get_args_from_tuple(bx, args[3], fn_target);
     let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE);
 
-    let sig = tcx.fn_sig(fn_target.def_id()).skip_binder().skip_binder();
+    let sig = tcx.fn_sig(fn_target.def_id()).instantiate_identity();
+    let sig = tcx.instantiate_bound_regions_with_erased(sig);
     let inputs = sig.inputs();
 
     let metadata = inputs.iter().map(|ty| OffloadMetadata::from_ty(tcx, *ty)).collect::>();

From 4b5b42c82c1cf2095fcb183c72406eb82c908b21 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Wed, 14 Jan 2026 16:34:31 +0530
Subject: [PATCH 0248/1843] remove postcard from legacy

---
 .../src/bidirectional_protocol.rs             |  9 +---
 .../proc-macro-api/src/legacy_protocol.rs     |  8 +---
 .../crates/proc-macro-api/src/lib.rs          |  3 --
 .../crates/proc-macro-api/src/process.rs      | 45 ++++++-------------
 .../crates/proc-macro-srv-cli/src/main.rs     |  5 ---
 .../proc-macro-srv-cli/src/main_loop.rs       |  1 -
 6 files changed, 16 insertions(+), 55 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index e44723a6a389..5996f882981c 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -212,14 +212,7 @@ fn run_request(
     if let Some(err) = srv.exited() {
         return Err(err.clone());
     }
-
-    match srv.use_postcard() {
-        true => srv.run_bidirectional::(msg, callback),
-        false => Err(ServerError {
-            message: "bidirectional messaging does not support JSON".to_owned(),
-            io: None,
-        }),
-    }
+    srv.run_bidirectional::(msg, callback)
 }
 
 pub fn reject_subrequests(req: SubRequest) -> Result {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
index 4524d1b66bfe..aabe5a011851 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
@@ -19,7 +19,7 @@ use crate::{
     },
     process::ProcMacroServerProcess,
     transport::codec::Codec,
-    transport::codec::{json::JsonProtocol, postcard::PostcardProtocol},
+    transport::codec::json::JsonProtocol,
     version,
 };
 
@@ -148,11 +148,7 @@ fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result(send_request::, req)
-    } else {
-        srv.send_task::<_, _, JsonProtocol>(send_request::, req)
-    }
+    srv.send_task::<_, _, JsonProtocol>(send_request::, req)
 }
 
 /// Sends a request to the server and reads the response.
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index 822809943a36..01195c10feef 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -49,8 +49,6 @@ pub mod version {
 pub enum ProtocolFormat {
     /// JSON-based legacy protocol (newline-delimited JSON).
     JsonLegacy,
-    /// Postcard-based legacy protocol (COBS-encoded postcard).
-    PostcardLegacy,
     /// Bidirectional postcard protocol with sub-request support.
     BidirectionalPostcardPrototype,
 }
@@ -59,7 +57,6 @@ impl fmt::Display for ProtocolFormat {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             ProtocolFormat::JsonLegacy => write!(f, "json-legacy"),
-            ProtocolFormat::PostcardLegacy => write!(f, "postcard-legacy"),
             ProtocolFormat::BidirectionalPostcardPrototype => {
                 write!(f, "bidirectional-postcard-prototype")
             }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index 4f8762158790..cd387dad0d0b 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -43,7 +43,6 @@ impl std::fmt::Debug for ProcMacroServerProcess {
 #[derive(Debug, Clone)]
 pub(crate) enum Protocol {
     LegacyJson { mode: SpanMode },
-    LegacyPostcard { mode: SpanMode },
     BidirectionalPostcardPrototype { mode: SpanMode },
 }
 
@@ -136,7 +135,6 @@ impl ProcMacroServerProcess {
         {
             &[
                 Some(ProtocolFormat::BidirectionalPostcardPrototype),
-                Some(ProtocolFormat::PostcardLegacy),
                 Some(ProtocolFormat::JsonLegacy),
             ]
         } else {
@@ -155,9 +153,6 @@ impl ProcMacroServerProcess {
                         Some(ProtocolFormat::BidirectionalPostcardPrototype) => {
                             Protocol::BidirectionalPostcardPrototype { mode: SpanMode::Id }
                         }
-                        Some(ProtocolFormat::PostcardLegacy) => {
-                            Protocol::LegacyPostcard { mode: SpanMode::Id }
-                        }
                         Some(ProtocolFormat::JsonLegacy) | None => {
                             Protocol::LegacyJson { mode: SpanMode::Id }
                         }
@@ -185,7 +180,6 @@ impl ProcMacroServerProcess {
                     {
                         match &mut srv.protocol {
                             Protocol::LegacyJson { mode }
-                            | Protocol::LegacyPostcard { mode }
                             | Protocol::BidirectionalPostcardPrototype { mode } => *mode = new_mode,
                         }
                     }
@@ -208,10 +202,6 @@ impl ProcMacroServerProcess {
         self.exited.get().map(|it| &it.0)
     }
 
-    pub(crate) fn use_postcard(&self) -> bool {
-        matches!(self.protocol, Protocol::LegacyPostcard { .. })
-    }
-
     /// Retrieves the API version of the proc-macro server.
     pub(crate) fn version(&self) -> u32 {
         self.version
@@ -221,7 +211,6 @@ impl ProcMacroServerProcess {
     pub(crate) fn rust_analyzer_spans(&self) -> bool {
         match self.protocol {
             Protocol::LegacyJson { mode } => mode == SpanMode::RustAnalyzer,
-            Protocol::LegacyPostcard { mode } => mode == SpanMode::RustAnalyzer,
             Protocol::BidirectionalPostcardPrototype { mode } => mode == SpanMode::RustAnalyzer,
         }
     }
@@ -229,9 +218,7 @@ impl ProcMacroServerProcess {
     /// Checks the API version of the running proc-macro server.
     fn version_check(&self, callback: Option>) -> Result {
         match self.protocol {
-            Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
-                legacy_protocol::version_check(self)
-            }
+            Protocol::LegacyJson { .. } => legacy_protocol::version_check(self),
             Protocol::BidirectionalPostcardPrototype { .. } => {
                 let cb = callback.expect("callback required for bidirectional protocol");
                 bidirectional_protocol::version_check(self, cb)
@@ -245,9 +232,7 @@ impl ProcMacroServerProcess {
         callback: Option>,
     ) -> Result {
         match self.protocol {
-            Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
-                legacy_protocol::enable_rust_analyzer_spans(self)
-            }
+            Protocol::LegacyJson { .. } => legacy_protocol::enable_rust_analyzer_spans(self),
             Protocol::BidirectionalPostcardPrototype { .. } => {
                 let cb = callback.expect("callback required for bidirectional protocol");
                 bidirectional_protocol::enable_rust_analyzer_spans(self, cb)
@@ -262,9 +247,7 @@ impl ProcMacroServerProcess {
         callback: Option>,
     ) -> Result, String>, ServerError> {
         match self.protocol {
-            Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
-                legacy_protocol::find_proc_macros(self, dylib_path)
-            }
+            Protocol::LegacyJson { .. } => legacy_protocol::find_proc_macros(self, dylib_path),
             Protocol::BidirectionalPostcardPrototype { .. } => {
                 let cb = callback.expect("callback required for bidirectional protocol");
                 bidirectional_protocol::find_proc_macros(self, dylib_path, cb)
@@ -285,18 +268,16 @@ impl ProcMacroServerProcess {
         callback: Option>,
     ) -> Result, ServerError> {
         match self.protocol {
-            Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
-                legacy_protocol::expand(
-                    proc_macro,
-                    subtree,
-                    attr,
-                    env,
-                    def_site,
-                    call_site,
-                    mixed_site,
-                    current_dir,
-                )
-            }
+            Protocol::LegacyJson { .. } => legacy_protocol::expand(
+                proc_macro,
+                subtree,
+                attr,
+                env,
+                def_site,
+                call_site,
+                mixed_site,
+                current_dir,
+            ),
             Protocol::BidirectionalPostcardPrototype { .. } => bidirectional_protocol::expand(
                 proc_macro,
                 subtree,
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs
index a246d4d3f28f..928753659f1c 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs
@@ -67,7 +67,6 @@ impl ValueEnum for ProtocolFormatArg {
     fn value_variants<'a>() -> &'a [Self] {
         &[
             ProtocolFormatArg(ProtocolFormat::JsonLegacy),
-            ProtocolFormatArg(ProtocolFormat::PostcardLegacy),
             ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype),
         ]
     }
@@ -75,9 +74,6 @@ impl ValueEnum for ProtocolFormatArg {
     fn to_possible_value(&self) -> Option {
         match self.0 {
             ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")),
-            ProtocolFormat::PostcardLegacy => {
-                Some(clap::builder::PossibleValue::new("postcard-legacy"))
-            }
             ProtocolFormat::BidirectionalPostcardPrototype => {
                 Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype"))
             }
@@ -87,7 +83,6 @@ impl ValueEnum for ProtocolFormatArg {
     fn from_str(input: &str, _ignore_case: bool) -> Result {
         match input {
             "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)),
-            "postcard-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::PostcardLegacy)),
             "bidirectional-postcard-prototype" => {
                 Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype))
             }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs
index 5180ede9fb90..70e1e091c197 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs
@@ -41,7 +41,6 @@ pub fn run(
 ) -> io::Result<()> {
     match format {
         ProtocolFormat::JsonLegacy => run_old::(stdin, stdout),
-        ProtocolFormat::PostcardLegacy => run_old::(stdin, stdout),
         ProtocolFormat::BidirectionalPostcardPrototype => {
             run_new::(stdin, stdout)
         }

From 31817f6deaf57c58f713bb61666d32bbd8306a3b Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Wed, 14 Jan 2026 16:34:57 +0530
Subject: [PATCH 0249/1843] remove flatten from ExpandMacro message in
 bidirectional messages

---
 .../crates/proc-macro-api/src/bidirectional_protocol/msg.rs      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
index 57e7b1ee8f68..c56ed5191694 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
@@ -70,7 +70,6 @@ pub struct ExpandMacro {
     pub lib: Utf8PathBuf,
     pub env: Vec<(String, String)>,
     pub current_dir: Option,
-    #[serde(flatten)]
     pub data: ExpandMacroData,
 }
 

From 98e1f7103b5d6bfcfcad849fa30a19614303160d Mon Sep 17 00:00:00 2001
From: Roberto Aloi 
Date: Wed, 14 Jan 2026 13:15:09 +0100
Subject: [PATCH 0250/1843] Bump camino to 1.2.2

---
 src/tools/rust-analyzer/Cargo.lock | 4 ++--
 src/tools/rust-analyzer/Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock
index d6c6250e13dc..a2a18cf8eeea 100644
--- a/src/tools/rust-analyzer/Cargo.lock
+++ b/src/tools/rust-analyzer/Cargo.lock
@@ -178,9 +178,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "camino"
-version = "1.2.0"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1de8bc0aa9e9385ceb3bf0c152e3a9b9544f6c4a912c8ae504e80c1f0368603"
+checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48"
 dependencies = [
  "serde_core",
 ]
diff --git a/src/tools/rust-analyzer/Cargo.toml b/src/tools/rust-analyzer/Cargo.toml
index 8003cb2fba8e..04b513b38b58 100644
--- a/src/tools/rust-analyzer/Cargo.toml
+++ b/src/tools/rust-analyzer/Cargo.toml
@@ -107,7 +107,7 @@ anyhow = "1.0.98"
 arrayvec = "0.7.6"
 bitflags = "2.9.1"
 cargo_metadata = "0.23.0"
-camino = "1.1.10"
+camino = "1.2.2"
 crossbeam-channel = "0.5.15"
 dissimilar = "1.0.10"
 dot = "0.1.4"

From 0d8aa8991c33f7dec592856760b3a1c32251bfba Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Wed, 14 Jan 2026 09:16:17 +0100
Subject: [PATCH 0251/1843] fix: Fix path symbol search not respecting
 re-exports

---
 src/tools/rust-analyzer/crates/hir/src/lib.rs | 17 ++++
 .../crates/ide-db/src/symbol_index.rs         | 97 ++++++++++++++-----
 2 files changed, 91 insertions(+), 23 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir/src/lib.rs b/src/tools/rust-analyzer/crates/hir/src/lib.rs
index 78be5a7e8fa9..252d71fb80a4 100644
--- a/src/tools/rust-analyzer/crates/hir/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/lib.rs
@@ -610,6 +610,23 @@ impl Module {
         res
     }
 
+    pub fn modules_in_scope(&self, db: &dyn HirDatabase, pub_only: bool) -> Vec<(Name, Module)> {
+        let def_map = self.id.def_map(db);
+        let scope = &def_map[self.id].scope;
+
+        let mut res = Vec::new();
+
+        for (name, item) in scope.types() {
+            if let ModuleDefId::ModuleId(m) = item.def
+                && (!pub_only || item.vis == Visibility::Public)
+            {
+                res.push((name.clone(), Module { id: m }));
+            }
+        }
+
+        res
+    }
+
     /// Returns a `ModuleScope`: a set of items, visible in this module.
     pub fn scope(
         self,
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
index c95b541748ec..d7f4c66f465b 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
+++ b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
@@ -35,6 +35,7 @@ use hir::{
     import_map::{AssocSearchMode, SearchMode},
     symbols::{FileSymbol, SymbolCollector},
 };
+use itertools::Itertools;
 use rayon::prelude::*;
 use salsa::Update;
 
@@ -224,12 +225,10 @@ pub fn world_symbols(db: &RootDatabase, mut query: Query) -> Vec>
     // Search for crates by name (handles "::" and "::foo" queries)
     let indices: Vec<_> = if query.is_crate_search() {
         query.only_types = false;
-        query.libs = true;
         vec![SymbolIndex::extern_prelude_symbols(db)]
         // If we have a path filter, resolve it to target modules
     } else if !query.path_filter.is_empty() {
         query.only_types = false;
-        query.libs = true;
         let target_modules = resolve_path_to_modules(
             db,
             &query.path_filter,
@@ -313,11 +312,11 @@ fn resolve_path_to_modules(
 
     // If anchor_to_crate is true, first segment MUST be a crate name
     // If anchor_to_crate is false, first segment could be a crate OR a module in local crates
-    let mut candidate_modules: Vec = vec![];
+    let mut candidate_modules: Vec<(Module, bool)> = vec![];
 
     // Add crate root modules for matching crates
     for krate in matching_crates {
-        candidate_modules.push(krate.root_module(db));
+        candidate_modules.push((krate.root_module(db), krate.origin(db).is_local()));
     }
 
     // If not anchored to crate, also search for modules matching first segment in local crates
@@ -329,7 +328,7 @@ fn resolve_path_to_modules(
                     if let Some(name) = child.name(db)
                         && names_match(name.as_str(), first_segment)
                     {
-                        candidate_modules.push(child);
+                        candidate_modules.push((child, true));
                     }
                 }
             }
@@ -340,11 +339,14 @@ fn resolve_path_to_modules(
     for segment in rest_segments {
         candidate_modules = candidate_modules
             .into_iter()
-            .flat_map(|module| {
-                module.children(db).filter(|child| {
-                    child.name(db).is_some_and(|name| names_match(name.as_str(), segment))
-                })
+            .flat_map(|(module, local)| {
+                module
+                    .modules_in_scope(db, !local)
+                    .into_iter()
+                    .filter(|(name, _)| names_match(name.as_str(), segment))
+                    .map(move |(_, module)| (module, local))
             })
+            .unique()
             .collect();
 
         if candidate_modules.is_empty() {
@@ -352,7 +354,7 @@ fn resolve_path_to_modules(
         }
     }
 
-    candidate_modules
+    candidate_modules.into_iter().map(|(module, _)| module).collect()
 }
 
 #[derive(Default)]
@@ -839,7 +841,7 @@ pub struct Foo;
         assert_eq!(item, "foo");
         assert!(anchor);
 
-        // Trailing :: (module browsing)
+        // Trailing ::
         let (path, item, anchor) = Query::parse_path_query("foo::");
         assert_eq!(path, vec!["foo"]);
         assert_eq!(item, "");
@@ -909,7 +911,7 @@ pub mod nested {
     }
 
     #[test]
-    fn test_module_browsing() {
+    fn test_path_search_module() {
         let (mut db, _) = RootDatabase::with_many_files(
             r#"
 //- /lib.rs crate:main
@@ -1066,20 +1068,11 @@ pub fn root_fn() {}
         let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect();
         assert!(names.contains(&"RootItem"), "Expected RootItem at crate root in {:?}", names);
 
-        // Browse crate root
         let query = Query::new("mylib::".to_owned());
         let symbols = world_symbols(&db, query);
         let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect();
-        assert!(
-            names.contains(&"RootItem"),
-            "Expected RootItem when browsing crate root in {:?}",
-            names
-        );
-        assert!(
-            names.contains(&"root_fn"),
-            "Expected root_fn when browsing crate root in {:?}",
-            names
-        );
+        assert!(names.contains(&"RootItem"), "Expected RootItem {:?}", names);
+        assert!(names.contains(&"root_fn"), "Expected root_fn {:?}", names);
     }
 
     #[test]
@@ -1163,4 +1156,62 @@ pub struct FooStruct;
         let symbols = world_symbols(&db, query);
         assert!(symbols.is_empty(), "Expected empty results for non-matching crate pattern");
     }
+
+    #[test]
+    fn test_path_search_with_use_reexport() {
+        // Test that module resolution works for `use` items (re-exports), not just `mod` items
+        let (mut db, _) = RootDatabase::with_many_files(
+            r#"
+//- /lib.rs crate:main
+mod inner;
+pub use inner::nested;
+
+//- /inner.rs
+pub mod nested {
+    pub struct NestedStruct;
+    pub fn nested_fn() {}
+}
+"#,
+        );
+
+        let mut local_roots = FxHashSet::default();
+        local_roots.insert(WORKSPACE);
+        LocalRoots::get(&db).set_roots(&mut db).to(local_roots);
+
+        // Search via the re-exported path (main::nested::NestedStruct)
+        // This should work because `nested` is in scope via `pub use inner::nested`
+        let query = Query::new("main::nested::NestedStruct".to_owned());
+        let symbols = world_symbols(&db, query);
+        let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect();
+        assert!(
+            names.contains(&"NestedStruct"),
+            "Expected NestedStruct via re-exported path in {:?}",
+            names
+        );
+
+        // Also verify the original path still works
+        let query = Query::new("main::inner::nested::NestedStruct".to_owned());
+        let symbols = world_symbols(&db, query);
+        let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect();
+        assert!(
+            names.contains(&"NestedStruct"),
+            "Expected NestedStruct via original path in {:?}",
+            names
+        );
+
+        // Browse the re-exported module
+        let query = Query::new("main::nested::".to_owned());
+        let symbols = world_symbols(&db, query);
+        let names: Vec<_> = symbols.iter().map(|s| s.name.as_str()).collect();
+        assert!(
+            names.contains(&"NestedStruct"),
+            "Expected NestedStruct when browsing re-exported module in {:?}",
+            names
+        );
+        assert!(
+            names.contains(&"nested_fn"),
+            "Expected nested_fn when browsing re-exported module in {:?}",
+            names
+        );
+    }
 }

From 6ecee2a415db5f3abf32414372e8325d3ff47f7b Mon Sep 17 00:00:00 2001
From: Wilfred Hughes 
Date: Wed, 14 Jan 2026 18:26:02 +0000
Subject: [PATCH 0252/1843] internal: Improve docs for discoverConfig

Add concrete examples of CLI invocations and JSONL outputs, use BUCK
for consistency with the first example, and polish the wording.
---
 .../crates/rust-analyzer/src/config.rs        | 116 +++++++++++-------
 .../docs/book/src/configuration_generated.md  | 116 +++++++++++-------
 .../docs/book/src/non_cargo_based_projects.md |   2 +-
 .../rust-analyzer/editors/code/package.json   |   2 +-
 4 files changed, 142 insertions(+), 94 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index 28ac94e4deb6..8d6b19a84caa 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -478,14 +478,26 @@ config_data! {
         typing_triggerChars: Option = Some("=.".to_owned()),
 
 
-        /// Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].
+        /// Configure a command that rust-analyzer can invoke to
+        /// obtain configuration.
         ///
-        /// [`DiscoverWorkspaceConfig`] also requires setting `progressLabel` and `filesToWatch`.
-        /// `progressLabel` is used for the title in progress indicators, whereas `filesToWatch`
-        /// is used to determine which build system-specific files should be watched in order to
-        /// reload rust-analyzer.
+        /// This is an alternative to manually generating
+        /// `rust-project.json`: it enables rust-analyzer to generate
+        /// rust-project.json on the fly, and regenerate it when
+        /// switching or modifying projects.
+        ///
+        /// This is an object with three fields:
+        ///
+        /// * `command`: the shell command to invoke
+        ///
+        /// * `filesToWatch`: which build system-specific files should
+        /// be watched to trigger regenerating the configuration
+        ///
+        /// * `progressLabel`: the name of the command, used in
+        /// progress indicators in the IDE
+        ///
+        /// Here's an example of a valid configuration:
         ///
-        /// Below is an example of a valid configuration:
         /// ```json
         /// "rust-analyzer.workspace.discoverConfig": {
         ///     "command": [
@@ -500,12 +512,49 @@ config_data! {
         /// }
         /// ```
         ///
-        /// ## Workspace Discovery Protocol
+        /// ## Argument Substitutions
+        ///
+        /// If `command` includes the argument `{arg}`, that argument will be substituted
+        /// with the JSON-serialized form of the following enum:
+        ///
+        /// ```norun
+        /// #[derive(PartialEq, Clone, Debug, Serialize)]
+        /// #[serde(rename_all = "camelCase")]
+        /// pub enum DiscoverArgument {
+        ///    Path(AbsPathBuf),
+        ///    Buildfile(AbsPathBuf),
+        /// }
+        /// ```
+        ///
+        /// rust-analyzer will use the path invocation to find and
+        /// generate a `rust-project.json` and therefore a
+        /// workspace. Example:
+        ///
+        ///
+        /// ```norun
+        /// rust-project develop-json '{ "path": "myproject/src/main.rs" }'
+        /// ```
+        ///
+        /// rust-analyzer will use build file invocations to update an
+        /// existing workspace. Example:
+        ///
+        /// Or with a build file and the configuration above:
+        ///
+        /// ```norun
+        /// rust-project develop-json '{ "buildfile": "myproject/BUCK" }'
+        /// ```
+        ///
+        /// As a reference for implementors, buck2's `rust-project`
+        /// will likely be useful:
+        /// .
+        ///
+        /// ## Discover Command Output
         ///
         /// **Warning**: This format is provisional and subject to change.
         ///
-        /// [`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to
-        /// `DiscoverProjectData::Finished`:
+        /// The discover command should output JSON objects, one per
+        /// line (JSONL format). These objects should correspond to
+        /// this Rust data type:
         ///
         /// ```norun
         /// #[derive(Debug, Clone, Deserialize, Serialize)]
@@ -518,7 +567,14 @@ config_data! {
         /// }
         /// ```
         ///
-        /// As JSON, `DiscoverProjectData::Finished` is:
+        /// For example, a progress event:
+        ///
+        /// ```json
+        /// {"kind":"progress","message":"generating rust-project.json"}
+        /// ```
+        ///
+        /// A finished event can look like this (expanded and
+        /// commented for readability):
         ///
         /// ```json
         /// {
@@ -526,7 +582,7 @@ config_data! {
         ///     "kind": "finished",
         ///     // the file used by a non-Cargo build system to define
         ///     // a package or target.
-        ///     "buildfile": "rust-analyzer/BUILD",
+        ///     "buildfile": "rust-analyzer/BUCK",
         ///     // the contents of a rust-project.json, elided for brevity
         ///     "project": {
         ///         "sysroot": "foo",
@@ -535,41 +591,9 @@ config_data! {
         /// }
         /// ```
         ///
-        /// It is encouraged, but not required, to use the other variants on `DiscoverProjectData`
-        /// to provide a more polished end-user experience.
-        ///
-        /// `DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be
-        /// substituted with the JSON-serialized form of the following enum:
-        ///
-        /// ```norun
-        /// #[derive(PartialEq, Clone, Debug, Serialize)]
-        /// #[serde(rename_all = "camelCase")]
-        /// pub enum DiscoverArgument {
-        ///    Path(AbsPathBuf),
-        ///    Buildfile(AbsPathBuf),
-        /// }
-        /// ```
-        ///
-        /// The JSON representation of `DiscoverArgument::Path` is:
-        ///
-        /// ```json
-        /// {
-        ///     "path": "src/main.rs"
-        /// }
-        /// ```
-        ///
-        /// Similarly, the JSON representation of `DiscoverArgument::Buildfile` is:
-        ///
-        /// ```json
-        /// {
-        ///     "buildfile": "BUILD"
-        /// }
-        /// ```
-        ///
-        /// `DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and
-        /// therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an
-        /// existing workspace. As a reference for implementors, buck2's `rust-project` will likely
-        /// be useful: .
+        /// Only the finished event is required, but the other
+        /// variants are encouraged to give users more feedback about
+        /// progress or errors.
         workspace_discoverConfig: Option = None,
     }
 }
diff --git a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
index c4124aaae075..9bc412631039 100644
--- a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
+++ b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
@@ -1619,14 +1619,26 @@ though Cargo might be the eventual consumer.
 
 Default: `null`
 
-Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].
+Configure a command that rust-analyzer can invoke to
+obtain configuration.
 
-[`DiscoverWorkspaceConfig`] also requires setting `progressLabel` and `filesToWatch`.
-`progressLabel` is used for the title in progress indicators, whereas `filesToWatch`
-is used to determine which build system-specific files should be watched in order to
-reload rust-analyzer.
+This is an alternative to manually generating
+`rust-project.json`: it enables rust-analyzer to generate
+rust-project.json on the fly, and regenerate it when
+switching or modifying projects.
+
+This is an object with three fields:
+
+* `command`: the shell command to invoke
+
+* `filesToWatch`: which build system-specific files should
+be watched to trigger regenerating the configuration
+
+* `progressLabel`: the name of the command, used in
+progress indicators in the IDE
+
+Here's an example of a valid configuration:
 
-Below is an example of a valid configuration:
 ```json
 "rust-analyzer.workspace.discoverConfig": {
     "command": [
@@ -1641,12 +1653,49 @@ Below is an example of a valid configuration:
 }
 ```
 
-## Workspace Discovery Protocol
+## Argument Substitutions
+
+If `command` includes the argument `{arg}`, that argument will be substituted
+with the JSON-serialized form of the following enum:
+
+```norun
+#[derive(PartialEq, Clone, Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub enum DiscoverArgument {
+   Path(AbsPathBuf),
+   Buildfile(AbsPathBuf),
+}
+```
+
+rust-analyzer will use the path invocation to find and
+generate a `rust-project.json` and therefore a
+workspace. Example:
+
+
+```norun
+rust-project develop-json '{ "path": "myproject/src/main.rs" }'
+```
+
+rust-analyzer will use build file invocations to update an
+existing workspace. Example:
+
+Or with a build file and the configuration above:
+
+```norun
+rust-project develop-json '{ "buildfile": "myproject/BUCK" }'
+```
+
+As a reference for implementors, buck2's `rust-project`
+will likely be useful:
+.
+
+## Discover Command Output
 
 **Warning**: This format is provisional and subject to change.
 
-[`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to
-`DiscoverProjectData::Finished`:
+The discover command should output JSON objects, one per
+line (JSONL format). These objects should correspond to
+this Rust data type:
 
 ```norun
 #[derive(Debug, Clone, Deserialize, Serialize)]
@@ -1659,7 +1708,14 @@ enum DiscoverProjectData {
 }
 ```
 
-As JSON, `DiscoverProjectData::Finished` is:
+For example, a progress event:
+
+```json
+{"kind":"progress","message":"generating rust-project.json"}
+```
+
+A finished event can look like this (expanded and
+commented for readability):
 
 ```json
 {
@@ -1667,7 +1723,7 @@ As JSON, `DiscoverProjectData::Finished` is:
     "kind": "finished",
     // the file used by a non-Cargo build system to define
     // a package or target.
-    "buildfile": "rust-analyzer/BUILD",
+    "buildfile": "rust-analyzer/BUCK",
     // the contents of a rust-project.json, elided for brevity
     "project": {
         "sysroot": "foo",
@@ -1676,41 +1732,9 @@ As JSON, `DiscoverProjectData::Finished` is:
 }
 ```
 
-It is encouraged, but not required, to use the other variants on `DiscoverProjectData`
-to provide a more polished end-user experience.
-
-`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be
-substituted with the JSON-serialized form of the following enum:
-
-```norun
-#[derive(PartialEq, Clone, Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub enum DiscoverArgument {
-   Path(AbsPathBuf),
-   Buildfile(AbsPathBuf),
-}
-```
-
-The JSON representation of `DiscoverArgument::Path` is:
-
-```json
-{
-    "path": "src/main.rs"
-}
-```
-
-Similarly, the JSON representation of `DiscoverArgument::Buildfile` is:
-
-```json
-{
-    "buildfile": "BUILD"
-}
-```
-
-`DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and
-therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an
-existing workspace. As a reference for implementors, buck2's `rust-project` will likely
-be useful: .
+Only the finished event is required, but the other
+variants are encouraged to give users more feedback about
+progress or errors.
 
 
 ## rust-analyzer.workspace.symbol.search.excludeImports {#workspace.symbol.search.excludeImports}
diff --git a/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md b/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md
index a48b025c7b3a..f1f10ae33653 100644
--- a/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md
+++ b/src/tools/rust-analyzer/docs/book/src/non_cargo_based_projects.md
@@ -237,7 +237,7 @@ There are four ways to feed `rust-project.json` to rust-analyzer:
     [`"rust-analyzer.workspace.discoverConfig": … }`](./configuration.md#workspace.discoverConfig)
     to specify a workspace discovery command to generate project descriptions
     on-the-fly. Please note that the command output is message-oriented and must
-    follow [the discovery protocol](./configuration.md#workspace-discovery-protocol).
+    output JSONL [as described in the configuration docs](./configuration.md#workspace.discoverConfig).
 
 -   Place `rust-project.json` file at the root of the project, and
     rust-analyzer will discover it.
diff --git a/src/tools/rust-analyzer/editors/code/package.json b/src/tools/rust-analyzer/editors/code/package.json
index 0d91378706a4..a197b7abd84c 100644
--- a/src/tools/rust-analyzer/editors/code/package.json
+++ b/src/tools/rust-analyzer/editors/code/package.json
@@ -3135,7 +3135,7 @@
                 "title": "Workspace",
                 "properties": {
                     "rust-analyzer.workspace.discoverConfig": {
-                        "markdownDescription": "Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].\n\n[`DiscoverWorkspaceConfig`] also requires setting `progressLabel` and `filesToWatch`.\n`progressLabel` is used for the title in progress indicators, whereas `filesToWatch`\nis used to determine which build system-specific files should be watched in order to\nreload rust-analyzer.\n\nBelow is an example of a valid configuration:\n```json\n\"rust-analyzer.workspace.discoverConfig\": {\n        \"command\": [\n                \"rust-project\",\n                \"develop-json\",\n                \"{arg}\"\n        ],\n        \"progressLabel\": \"buck2/rust-project\",\n        \"filesToWatch\": [\n                \"BUCK\"\n        ]\n}\n```\n\n## Workspace Discovery Protocol\n\n**Warning**: This format is provisional and subject to change.\n\n[`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to\n`DiscoverProjectData::Finished`:\n\n```norun\n#[derive(Debug, Clone, Deserialize, Serialize)]\n#[serde(tag = \"kind\")]\n#[serde(rename_all = \"snake_case\")]\nenum DiscoverProjectData {\n        Finished { buildfile: Utf8PathBuf, project: ProjectJsonData },\n        Error { error: String, source: Option },\n        Progress { message: String },\n}\n```\n\nAs JSON, `DiscoverProjectData::Finished` is:\n\n```json\n{\n        // the internally-tagged representation of the enum.\n        \"kind\": \"finished\",\n        // the file used by a non-Cargo build system to define\n        // a package or target.\n        \"buildfile\": \"rust-analyzer/BUILD\",\n        // the contents of a rust-project.json, elided for brevity\n        \"project\": {\n                \"sysroot\": \"foo\",\n                \"crates\": []\n        }\n}\n```\n\nIt is encouraged, but not required, to use the other variants on `DiscoverProjectData`\nto provide a more polished end-user experience.\n\n`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be\nsubstituted with the JSON-serialized form of the following enum:\n\n```norun\n#[derive(PartialEq, Clone, Debug, Serialize)]\n#[serde(rename_all = \"camelCase\")]\npub enum DiscoverArgument {\n     Path(AbsPathBuf),\n     Buildfile(AbsPathBuf),\n}\n```\n\nThe JSON representation of `DiscoverArgument::Path` is:\n\n```json\n{\n        \"path\": \"src/main.rs\"\n}\n```\n\nSimilarly, the JSON representation of `DiscoverArgument::Buildfile` is:\n\n```json\n{\n        \"buildfile\": \"BUILD\"\n}\n```\n\n`DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and\ntherefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an\nexisting workspace. As a reference for implementors, buck2's `rust-project` will likely\nbe useful: .",
+                        "markdownDescription": "Configure a command that rust-analyzer can invoke to\nobtain configuration.\n\nThis is an alternative to manually generating\n`rust-project.json`: it enables rust-analyzer to generate\nrust-project.json on the fly, and regenerate it when\nswitching or modifying projects.\n\nThis is an object with three fields:\n\n* `command`: the shell command to invoke\n\n* `filesToWatch`: which build system-specific files should\nbe watched to trigger regenerating the configuration\n\n* `progressLabel`: the name of the command, used in\nprogress indicators in the IDE\n\nHere's an example of a valid configuration:\n\n```json\n\"rust-analyzer.workspace.discoverConfig\": {\n        \"command\": [\n                \"rust-project\",\n                \"develop-json\",\n                \"{arg}\"\n        ],\n        \"progressLabel\": \"buck2/rust-project\",\n        \"filesToWatch\": [\n                \"BUCK\"\n        ]\n}\n```\n\n## Argument Substitutions\n\nIf `command` includes the argument `{arg}`, that argument will be substituted\nwith the JSON-serialized form of the following enum:\n\n```norun\n#[derive(PartialEq, Clone, Debug, Serialize)]\n#[serde(rename_all = \"camelCase\")]\npub enum DiscoverArgument {\n     Path(AbsPathBuf),\n     Buildfile(AbsPathBuf),\n}\n```\n\nrust-analyzer will use the path invocation to find and\ngenerate a `rust-project.json` and therefore a\nworkspace. Example:\n\n\n```norun\nrust-project develop-json '{ \"path\": \"myproject/src/main.rs\" }'\n```\n\nrust-analyzer will use build file invocations to update an\nexisting workspace. Example:\n\nOr with a build file and the configuration above:\n\n```norun\nrust-project develop-json '{ \"buildfile\": \"myproject/BUCK\" }'\n```\n\nAs a reference for implementors, buck2's `rust-project`\nwill likely be useful:\n.\n\n## Discover Command Output\n\n**Warning**: This format is provisional and subject to change.\n\nThe discover command should output JSON objects, one per\nline (JSONL format). These objects should correspond to\nthis Rust data type:\n\n```norun\n#[derive(Debug, Clone, Deserialize, Serialize)]\n#[serde(tag = \"kind\")]\n#[serde(rename_all = \"snake_case\")]\nenum DiscoverProjectData {\n        Finished { buildfile: Utf8PathBuf, project: ProjectJsonData },\n        Error { error: String, source: Option },\n        Progress { message: String },\n}\n```\n\nFor example, a progress event:\n\n```json\n{\"kind\":\"progress\",\"message\":\"generating rust-project.json\"}\n```\n\nA finished event can look like this (expanded and\ncommented for readability):\n\n```json\n{\n        // the internally-tagged representation of the enum.\n        \"kind\": \"finished\",\n        // the file used by a non-Cargo build system to define\n        // a package or target.\n        \"buildfile\": \"rust-analyzer/BUCK\",\n        // the contents of a rust-project.json, elided for brevity\n        \"project\": {\n                \"sysroot\": \"foo\",\n                \"crates\": []\n        }\n}\n```\n\nOnly the finished event is required, but the other\nvariants are encouraged to give users more feedback about\nprogress or errors.",
                         "default": null,
                         "anyOf": [
                             {

From a1a944880f89b275745d1c5838463433980b3f72 Mon Sep 17 00:00:00 2001
From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com>
Date: Wed, 14 Jan 2026 21:16:54 +0100
Subject: [PATCH 0253/1843] checksum-freshness: Add binary file inclusion to
 integration test

---
 tests/run-make/checksum-freshness/binary_file | 1 +
 tests/run-make/checksum-freshness/expected.d  | 6 ++++--
 tests/run-make/checksum-freshness/lib.rs      | 3 ++-
 3 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 tests/run-make/checksum-freshness/binary_file

diff --git a/tests/run-make/checksum-freshness/binary_file b/tests/run-make/checksum-freshness/binary_file
new file mode 100644
index 000000000000..45f1873fb781
--- /dev/null
+++ b/tests/run-make/checksum-freshness/binary_file
@@ -0,0 +1 @@
+binary
\ No newline at end of file
diff --git a/tests/run-make/checksum-freshness/expected.d b/tests/run-make/checksum-freshness/expected.d
index 51467af53a20..4554c509e36e 100644
--- a/tests/run-make/checksum-freshness/expected.d
+++ b/tests/run-make/checksum-freshness/expected.d
@@ -1,6 +1,8 @@
-lib.d: lib.rs foo.rs
+lib.d: lib.rs foo.rs binary_file
 
 lib.rs:
 foo.rs:
-# checksum:blake3=94af75ee4ed805434484c3de51c9025278e5c3ada2315e2592052e102168a503 file_len:120 lib.rs
+binary_file:
+# checksum:blake3=4ac56f3f877798fb762d714c7bcb72e70133f4cc585f80dbd99c07755ae2c7f6 file_len:222 lib.rs
 # checksum:blake3=2720e17bfda4f3b2a5c96bb61b7e76ed8ebe3359b34128c0e5d8032c090a4f1a file_len:119 foo.rs
+# checksum:blake3=119a5db8711914922c5b1c1908be4958175c5afa95c08888de594725329b5439 file_len:7 binary_file
diff --git a/tests/run-make/checksum-freshness/lib.rs b/tests/run-make/checksum-freshness/lib.rs
index 7bc6757959b1..0cd4243423de 100644
--- a/tests/run-make/checksum-freshness/lib.rs
+++ b/tests/run-make/checksum-freshness/lib.rs
@@ -1,7 +1,8 @@
 // A basic library to be used in tests with no real purpose.
 
 mod foo;
-
+// Binary file with invalid UTF-8 sequence.
+static BINARY_FILE: &[u8] = include_bytes!("binary_file");
 pub fn sum(a: i32, b: i32) -> i32 {
     a + b
 }

From ed7479c658484647858e08342a9aa00aa0fb5b30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Wed, 14 Jan 2026 21:47:24 +0100
Subject: [PATCH 0254/1843] Add the GCC codegen backend to build-manifest

---
 src/tools/build-manifest/src/main.rs     | 3 ++-
 src/tools/build-manifest/src/versions.rs | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 4cec1b1f164b..7c0c528bcc04 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -32,7 +32,7 @@ static DOCS_FALLBACK: &[(&str, &str)] = &[
 static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
 
 static NIGHTLY_ONLY_COMPONENTS: &[PkgType] =
-    &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift];
+    &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift, PkgType::RustcCodegenGcc];
 
 macro_rules! t {
     ($e:expr) => {
@@ -302,6 +302,7 @@ impl Builder {
                 | PkgType::RustAnalysis
                 | PkgType::JsonDocs
                 | PkgType::RustcCodegenCranelift
+                | PkgType::RustcCodegenGcc
                 | PkgType::LlvmBitcodeLinker => {
                     extensions.push(host_component(pkg));
                 }
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 6ef8a0e83de3..1e55d1d467e6 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -59,6 +59,7 @@ pkg_type! {
     JsonDocs = "rust-docs-json"; preview = true,
     RustcCodegenCranelift = "rustc-codegen-cranelift"; preview = true,
     LlvmBitcodeLinker = "llvm-bitcode-linker"; preview = true,
+    RustcCodegenGcc = "rustc-codegen-gcc"; preview = true,
 }
 
 impl PkgType {
@@ -82,6 +83,7 @@ impl PkgType {
             PkgType::LlvmTools => false,
             PkgType::Miri => false,
             PkgType::RustcCodegenCranelift => false,
+            PkgType::RustcCodegenGcc => false,
 
             PkgType::Rust => true,
             PkgType::RustStd => true,
@@ -111,6 +113,7 @@ impl PkgType {
             RustcDocs => HOSTS,
             Cargo => HOSTS,
             RustcCodegenCranelift => HOSTS,
+            RustcCodegenGcc => HOSTS,
             RustMingw => MINGW,
             RustStd => TARGETS,
             HtmlDocs => HOSTS,

From 4ceb13807e8062d5a4a3e9adecbd1d42a61eecf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Wed, 14 Jan 2026 21:49:44 +0100
Subject: [PATCH 0255/1843] Forbid distributing GCC on CI if
 `gcc.download-ci-gcc` is enabled

---
 src/bootstrap/src/core/build_steps/dist.rs | 10 +++++++++-
 src/bootstrap/src/core/config/mod.rs       |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index cfcb144e0993..bfffc958f858 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -30,7 +30,7 @@ use crate::core::build_steps::tool::{
 use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor};
 use crate::core::build_steps::{compile, llvm};
 use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata};
-use crate::core::config::TargetSelection;
+use crate::core::config::{GccCiMode, TargetSelection};
 use crate::utils::build_stamp::{self, BuildStamp};
 use crate::utils::channel::{self, Info};
 use crate::utils::exec::{BootstrapCommand, command};
@@ -3035,6 +3035,14 @@ impl Step for Gcc {
             return None;
         }
 
+        if builder.config.is_running_on_ci {
+            assert_eq!(
+                builder.config.gcc_ci_mode,
+                GccCiMode::BuildLocally,
+                "Cannot use gcc.download-ci-gcc when distributing GCC on CI"
+            );
+        }
+
         // We need the GCC sources to build GCC and also to add its license and README
         // files to the tarball
         builder.require_submodule(
diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs
index 007ed4aaba13..7651def62672 100644
--- a/src/bootstrap/src/core/config/mod.rs
+++ b/src/bootstrap/src/core/config/mod.rs
@@ -425,7 +425,7 @@ impl std::str::FromStr for RustcLto {
 }
 
 /// Determines how will GCC be provided.
-#[derive(Default, Clone)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub enum GccCiMode {
     /// Build GCC from the local `src/gcc` submodule.
     BuildLocally,

From 4ded6397b376b0a005c26381ed92c944e3702019 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Wed, 14 Jan 2026 00:45:16 +0100
Subject: [PATCH 0256/1843] Add new "hide deprecated items" setting in rustdoc

---
 src/librustdoc/clean/auto_trait.rs         |  1 +
 src/librustdoc/clean/blanket_impl.rs       |  3 ++
 src/librustdoc/clean/inline.rs             |  3 ++
 src/librustdoc/clean/mod.rs                |  4 ++
 src/librustdoc/clean/types.rs              |  8 ++++
 src/librustdoc/html/render/mod.rs          | 33 +++++++++++----
 src/librustdoc/html/render/print_item.rs   | 49 +++++++++++++++-------
 src/librustdoc/html/static/css/rustdoc.css |  9 ++++
 src/librustdoc/html/static/js/search.js    |  3 ++
 src/librustdoc/html/static/js/settings.js  | 11 +++++
 src/librustdoc/html/static/js/storage.js   |  3 ++
 src/librustdoc/json/conversions.rs         |  3 +-
 12 files changed, 107 insertions(+), 23 deletions(-)

diff --git a/src/librustdoc/clean/auto_trait.rs b/src/librustdoc/clean/auto_trait.rs
index 6c67916571a4..847e688d03d0 100644
--- a/src/librustdoc/clean/auto_trait.rs
+++ b/src/librustdoc/clean/auto_trait.rs
@@ -126,6 +126,7 @@ fn synthesize_auto_trait_impl<'tcx>(
                 items: Vec::new(),
                 polarity,
                 kind: clean::ImplKind::Auto,
+                is_deprecated: false,
             })),
             item_id: clean::ItemId::Auto { trait_: trait_def_id, for_: item_def_id },
             cfg: None,
diff --git a/src/librustdoc/clean/blanket_impl.rs b/src/librustdoc/clean/blanket_impl.rs
index ddfce7aeb92d..de45922e856f 100644
--- a/src/librustdoc/clean/blanket_impl.rs
+++ b/src/librustdoc/clean/blanket_impl.rs
@@ -117,6 +117,9 @@ pub(crate) fn synthesize_blanket_impls(
                             None,
                             None,
                         ))),
+                        is_deprecated: tcx
+                            .lookup_deprecation(impl_def_id)
+                            .is_some_and(|deprecation| deprecation.is_in_effect()),
                     })),
                     cfg: None,
                     inline_stmt_id: None,
diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs
index c86a655c083a..19974f4847d4 100644
--- a/src/librustdoc/clean/inline.rs
+++ b/src/librustdoc/clean/inline.rs
@@ -645,6 +645,9 @@ pub(crate) fn build_impl(
             } else {
                 ImplKind::Normal
             },
+            is_deprecated: tcx
+                .lookup_deprecation(did)
+                .is_some_and(|deprecation| deprecation.is_in_effect()),
         })),
         merged_attrs,
         cfg,
diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs
index 817eda4c52ec..0e970a042ff0 100644
--- a/src/librustdoc/clean/mod.rs
+++ b/src/librustdoc/clean/mod.rs
@@ -2882,6 +2882,9 @@ fn clean_impl<'tcx>(
             )),
             _ => None,
         });
+    let is_deprecated = tcx
+        .lookup_deprecation(def_id.to_def_id())
+        .is_some_and(|deprecation| deprecation.is_in_effect());
     let mut make_item = |trait_: Option, for_: Type, items: Vec| {
         let kind = ImplItem(Box::new(Impl {
             safety: match impl_.of_trait {
@@ -2902,6 +2905,7 @@ fn clean_impl<'tcx>(
             } else {
                 ImplKind::Normal
             },
+            is_deprecated,
         }));
         Item::from_def_id_and_parts(def_id.to_def_id(), None, kind, cx)
     };
diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs
index c3bafd3db13a..93863ea41396 100644
--- a/src/librustdoc/clean/types.rs
+++ b/src/librustdoc/clean/types.rs
@@ -427,6 +427,10 @@ impl Item {
         })
     }
 
+    pub(crate) fn is_deprecated(&self, tcx: TyCtxt<'_>) -> bool {
+        self.deprecation(tcx).is_some_and(|deprecation| deprecation.is_in_effect())
+    }
+
     pub(crate) fn inner_docs(&self, tcx: TyCtxt<'_>) -> bool {
         self.item_id.as_def_id().map(|did| inner_docs(tcx.get_all_attrs(did))).unwrap_or(false)
     }
@@ -1270,6 +1274,9 @@ impl Trait {
     pub(crate) fn is_dyn_compatible(&self, tcx: TyCtxt<'_>) -> bool {
         tcx.is_dyn_compatible(self.def_id)
     }
+    pub(crate) fn is_deprecated(&self, tcx: TyCtxt<'_>) -> bool {
+        tcx.lookup_deprecation(self.def_id).is_some_and(|deprecation| deprecation.is_in_effect())
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -2254,6 +2261,7 @@ pub(crate) struct Impl {
     pub(crate) items: Vec,
     pub(crate) polarity: ty::ImplPolarity,
     pub(crate) kind: ImplKind,
+    pub(crate) is_deprecated: bool,
 }
 
 impl Impl {
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index 63de870f07f4..106aefa4626b 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -1794,12 +1794,14 @@ fn render_impl(
             let mut info_buffer = String::new();
             let mut short_documented = true;
 
+            let mut trait_item_deprecated = false;
             if render_method_item {
                 if !is_default_item {
                     if let Some(t) = trait_ {
                         // The trait item may have been stripped so we might not
                         // find any documentation or stability for it.
                         if let Some(it) = t.items.iter().find(|i| i.name == item.name) {
+                            trait_item_deprecated = it.is_deprecated(cx.tcx());
                             // We need the stability of the item from the trait
                             // because impls can't have a stability.
                             if !item.doc_value().is_empty() {
@@ -1839,10 +1841,20 @@ fn render_impl(
                 Either::Right(boring)
             };
 
+            let mut deprecation_class = if trait_item_deprecated || item.is_deprecated(cx.tcx()) {
+                " deprecated"
+            } else {
+                ""
+            };
+
             let toggled = !doc_buffer.is_empty();
             if toggled {
                 let method_toggle_class = if item_type.is_method() { " method-toggle" } else { "" };
-                write!(w, "
")?;
+                write!(
+                    w,
+                    "
"
+                )?;
+                deprecation_class = "";
             }
             match &item.kind {
                 clean::MethodItem(..) | clean::RequiredMethodItem(_) => {
@@ -1859,7 +1871,7 @@ fn render_impl(
                             .map(|item| format!("{}.{name}", item.type_()));
                         write!(
                             w,
-                            "
\
+                            "
\
                                 {}",
                             render_rightside(cx, item, render_mode)
                         )?;
@@ -1885,7 +1897,7 @@ fn render_impl(
                     let id = cx.derive_id(&source_id);
                     write!(
                         w,
-                        "
\
+                        "
\
                             {}",
                         render_rightside(cx, item, render_mode)
                     )?;
@@ -1912,7 +1924,7 @@ fn render_impl(
                     let id = cx.derive_id(&source_id);
                     write!(
                         w,
-                        "
\
+                        "
\
                             {}",
                         render_rightside(cx, item, render_mode),
                     )?;
@@ -1944,7 +1956,7 @@ fn render_impl(
                     let id = cx.derive_id(&source_id);
                     write!(
                         w,
-                        "
\
+                        "
\
                             {}",
                         render_rightside(cx, item, render_mode),
                     )?;
@@ -1971,7 +1983,7 @@ fn render_impl(
                     let id = cx.derive_id(&source_id);
                     write!(
                         w,
-                        "
\
+                        "
\
                             {}",
                         render_rightside(cx, item, render_mode),
                     )?;
@@ -2143,11 +2155,18 @@ fn render_impl(
         }
         if render_mode == RenderMode::Normal {
             let toggled = !(impl_items.is_empty() && default_impl_items.is_empty());
+            let deprecation_attr = if impl_.is_deprecated
+                || trait_.is_some_and(|trait_| trait_.is_deprecated(cx.tcx()))
+            {
+                " deprecated"
+            } else {
+                ""
+            };
             if toggled {
                 close_tags.push("
");
                 write!(
                     w,
-                    "
\
+                    "
\
                         ",
                     if rendering_params.toggle_open_by_default { " open" } else { "" }
                 )?;
diff --git a/src/librustdoc/html/render/print_item.rs b/src/librustdoc/html/render/print_item.rs
index 84e93a479b5b..55e36c462f96 100644
--- a/src/librustdoc/html/render/print_item.rs
+++ b/src/librustdoc/html/render/print_item.rs
@@ -217,6 +217,10 @@ fn toggle_close(mut w: impl fmt::Write) {
 }
 
 fn item_module(cx: &Context<'_>, item: &clean::Item, items: &[clean::Item]) -> impl fmt::Display {
+    fn deprecation_class_attr(is_deprecated: bool) -> &'static str {
+        if is_deprecated { " class=\"deprecated\"" } else { "" }
+    }
+
     fmt::from_fn(|w| {
         write!(w, "{}", document(cx, item, None, HeadingOffset::H2))?;
 
@@ -370,11 +374,18 @@ fn item_module(cx: &Context<'_>, item: &clean::Item, items: &[clean::Item]) -> i
                         write!(w, "")?
                     }
                     clean::ImportItem(ref import) => {
-                        let stab_tags =
-                            import.source.did.map_or_else(String::new, |import_def_id| {
-                                print_extra_info_tags(tcx, myitem, item, Some(import_def_id))
-                                    .to_string()
-                            });
+                        let (stab_tags, deprecation) = match import.source.did {
+                            Some(import_def_id) => {
+                                let stab_tags =
+                                    print_extra_info_tags(tcx, myitem, item, Some(import_def_id))
+                                        .to_string();
+                                let deprecation = tcx
+                                    .lookup_deprecation(import_def_id)
+                                    .is_some_and(|deprecation| deprecation.is_in_effect());
+                                (stab_tags, deprecation)
+                            }
+                            None => (String::new(), item.is_deprecated(tcx)),
+                        };
                         let id = match import.kind {
                             clean::ImportKind::Simple(s) => {
                                 format!(" id=\"{}\"", cx.derive_id(format!("reexport.{s}")))
@@ -383,8 +394,8 @@ fn item_module(cx: &Context<'_>, item: &clean::Item, items: &[clean::Item]) -> i
                         };
                         write!(
                             w,
-                            "\
-                                "
+                            "",
+                            deprecation_attr = deprecation_class_attr(deprecation)
                         )?;
                         render_attributes_in_code(w, myitem, "", cx)?;
                         write!(
@@ -396,9 +407,7 @@ fn item_module(cx: &Context<'_>, item: &clean::Item, items: &[clean::Item]) -> i
                         )?;
                     }
                     _ => {
-                        if myitem.name.is_none() {
-                            continue;
-                        }
+                        let Some(item_name) = myitem.name else { continue };
 
                         let unsafety_flag = match myitem.kind {
                             clean::FunctionItem(_) | clean::ForeignFunctionItem(..)
@@ -431,9 +440,10 @@ fn item_module(cx: &Context<'_>, item: &clean::Item, items: &[clean::Item]) -> i
                             .into_string();
                         let (docs_before, docs_after) =
                             if docs.is_empty() { ("", "") } else { ("
", "
") };
+                        let deprecation_attr = deprecation_class_attr(myitem.is_deprecated(tcx));
                         write!(
                             w,
-                            "
\
+                            "\
                                 \
                                 {name}\
                                 \
@@ -442,12 +452,12 @@ fn item_module(cx: &Context<'_>, item: &clean::Item, items: &[clean::Item]) -> i
                                 {stab_tags}\
                             
\
                             {docs_before}{docs}{docs_after}",
-                            name = EscapeBodyTextWithWbr(myitem.name.unwrap().as_str()),
+                            name = EscapeBodyTextWithWbr(item_name.as_str()),
                             visibility_and_hidden = visibility_and_hidden,
                             stab_tags = print_extra_info_tags(tcx, myitem, item, None),
                             class = type_,
                             unsafety_flag = unsafety_flag,
-                            href = print_item_path(type_, myitem.name.unwrap().as_str()),
+                            href = print_item_path(type_, item_name.as_str()),
                             title1 = myitem.type_(),
                             title2 = full_path(cx, myitem),
                         )?;
@@ -778,15 +788,24 @@ fn item_trait(cx: &Context<'_>, it: &clean::Item, t: &clean::Trait) -> impl fmt:
 
                 let content = document_full(m, cx, HeadingOffset::H5).to_string();
 
+                let mut deprecation_class =
+                    if m.is_deprecated(cx.tcx()) { " deprecated" } else { "" };
+
                 let toggled = !content.is_empty();
                 if toggled {
                     let method_toggle_class =
                         if item_type.is_method() { " method-toggle" } else { "" };
-                    write!(w, "
")?;
+                    write!(
+                        w,
+                        "
"
+                    )?;
+                    deprecation_class = "";
                 }
                 write!(
                     w,
-                    "
\
+                    "
\
                     {}\
                     
{}
",
                     render_rightside(cx, m, RenderMode::Normal),
diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index b770a0e2a0e4..73eb260a2b8e 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -2642,6 +2642,15 @@ However, it's not needed with smaller screen width because the doc/code block is
 	}
 }
 
+/* Items on module pages */
+.hide-deprecated-items dt.deprecated,
+.hide-deprecated-items dt.deprecated + dd,
+/* Items on item pages */
+.hide-deprecated-items .deprecated,
+.hide-deprecated-items .deprecated + .item-info {
+	display: none;
+}
+
 /*
 WARNING: RUSTDOC_MOBILE_BREAKPOINT MEDIA QUERY
 If you update this line, then you also need to update the line with the same warning
diff --git a/src/librustdoc/html/static/js/search.js b/src/librustdoc/html/static/js/search.js
index b2880a2f4c2b..9961c1447ec2 100644
--- a/src/librustdoc/html/static/js/search.js
+++ b/src/librustdoc/html/static/js/search.js
@@ -4920,6 +4920,9 @@ async function addTab(results, query, display, finishedCallback, isTypeSearch) {
 
         const link = document.createElement("a");
         link.className = "result-" + type;
+        if (obj.item.deprecated) {
+            link.className += " deprecated";
+        }
         link.href = obj.href;
 
         const resultName = document.createElement("span");
diff --git a/src/librustdoc/html/static/js/settings.js b/src/librustdoc/html/static/js/settings.js
index 347d3d0750ec..b28b76b0d2e7 100644
--- a/src/librustdoc/html/static/js/settings.js
+++ b/src/librustdoc/html/static/js/settings.js
@@ -78,6 +78,12 @@
                     removeClass(document.documentElement, "word-wrap-source-code");
                 }
                 break;
+            case "hide-deprecated-items":
+                if (value === true) {
+                    addClass(document.documentElement, "hide-deprecated-items");
+                } else {
+                    removeClass(document.documentElement, "hide-deprecated-items");
+                }
         }
     }
 
@@ -274,6 +280,11 @@
                 "js_name": "word-wrap-source-code",
                 "default": false,
             },
+            {
+                "name": "Hide deprecated items",
+                "js_name": "hide-deprecated-items",
+                "default": false,
+            },
         ];
 
         // Then we build the DOM.
diff --git a/src/librustdoc/html/static/js/storage.js b/src/librustdoc/html/static/js/storage.js
index 40ab8be03c93..7cc70374378b 100644
--- a/src/librustdoc/html/static/js/storage.js
+++ b/src/librustdoc/html/static/js/storage.js
@@ -334,6 +334,9 @@ if (getSettingValue("sans-serif-fonts") === "true") {
 if (getSettingValue("word-wrap-source-code") === "true") {
     addClass(document.documentElement, "word-wrap-source-code");
 }
+if (getSettingValue("hide-deprecated-items") === "true") {
+    addClass(document.documentElement, "hide-deprecated-items");
+}
 function updateSidebarWidth() {
     const desktopSidebarWidth = getSettingValue("desktop-sidebar-width");
     if (desktopSidebarWidth && desktopSidebarWidth !== "null") {
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index 892cc483dbd6..2edf7891be40 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -711,7 +711,8 @@ impl FromClean for PolyTrait {
 impl FromClean for Impl {
     fn from_clean(impl_: &clean::Impl, renderer: &JsonRenderer<'_>) -> Self {
         let provided_trait_methods = impl_.provided_trait_methods(renderer.tcx);
-        let clean::Impl { safety, generics, trait_, for_, items, polarity, kind } = impl_;
+        let clean::Impl { safety, generics, trait_, for_, items, polarity, kind, is_deprecated: _ } =
+            impl_;
         // FIXME: use something like ImplKind in JSON?
         let (is_synthetic, blanket_impl) = match kind {
             clean::ImplKind::Normal | clean::ImplKind::FakeVariadic => (false, None),

From df386c5b48e2b3a3dabae22f2b018501f5abcbe8 Mon Sep 17 00:00:00 2001
From: Ben Kimock 
Date: Tue, 30 Dec 2025 16:57:56 -0500
Subject: [PATCH 0257/1843] Test that -Zbuild-std=core works on a variety of
 profiles

---
 Cargo.lock                                    |   1 +
 .../src/spec/targets/i586_unknown_redox.rs    |   2 +-
 .../spec/targets/x86_64_unknown_linux_none.rs |   2 +-
 .../src/spec/targets/xtensa_esp32_espidf.rs   |   2 +-
 .../src/spec/targets/xtensa_esp32s2_espidf.rs |   2 +-
 .../src/spec/targets/xtensa_esp32s3_espidf.rs |   2 +-
 src/bootstrap/mk/Makefile.in                  |   5 +
 src/bootstrap/src/core/build_steps/test.rs    |   8 +-
 src/bootstrap/src/core/builder/cli_paths.rs   |   1 +
 .../cli_paths/snapshots/x_test_tests.snap     |   3 +
 .../snapshots/x_test_tests_skip_coverage.snap |   3 +
 src/bootstrap/src/core/builder/mod.rs         |   1 +
 src/bootstrap/src/core/builder/tests.rs       |   4 +-
 .../rustc-dev-guide/src/tests/compiletest.md  |  10 ++
 src/tools/compiletest/src/common.rs           |   1 +
 src/tools/compiletest/src/runtest/run_make.rs |   4 +-
 src/tools/run-make-support/Cargo.toml         |   1 +
 src/tools/run-make-support/src/command.rs     |  13 ++
 src/tools/run-make-support/src/lib.rs         |   4 +-
 src/tools/run-make-support/src/util.rs        |   3 +
 tests/build-std/configurations/rmake.rs       | 131 ++++++++++++++++++
 21 files changed, 192 insertions(+), 11 deletions(-)
 create mode 100644 tests/build-std/configurations/rmake.rs

diff --git a/Cargo.lock b/Cargo.lock
index bf28939ac87b..bf0bc9234d39 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3334,6 +3334,7 @@ dependencies = [
  "rustdoc-json-types",
  "serde_json",
  "similar",
+ "tempfile",
  "wasmparser 0.236.1",
 ]
 
diff --git a/compiler/rustc_target/src/spec/targets/i586_unknown_redox.rs b/compiler/rustc_target/src/spec/targets/i586_unknown_redox.rs
index 935666630dae..56e1c8a9705d 100644
--- a/compiler/rustc_target/src/spec/targets/i586_unknown_redox.rs
+++ b/compiler/rustc_target/src/spec/targets/i586_unknown_redox.rs
@@ -11,7 +11,7 @@ pub(crate) fn target() -> Target {
 
     Target {
         llvm_target: "i586-unknown-redox".into(),
-        metadata: TargetMetadata { description: None, tier: None, host_tools: None, std: None },
+        metadata: TargetMetadata { description: None, tier: Some(3), host_tools: None, std: None },
         pointer_width: 32,
         data_layout:
             "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_none.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_none.rs
index ecf232f1ab2f..768b1a1ba112 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_none.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_none.rs
@@ -15,7 +15,7 @@ pub(crate) fn target() -> Target {
         llvm_target: "x86_64-unknown-linux-none".into(),
         metadata: TargetMetadata {
             description: None,
-            tier: None,
+            tier: Some(3),
             host_tools: None,
             std: Some(false),
         },
diff --git a/compiler/rustc_target/src/spec/targets/xtensa_esp32_espidf.rs b/compiler/rustc_target/src/spec/targets/xtensa_esp32_espidf.rs
index 83835705f3da..0fb4d186087b 100644
--- a/compiler/rustc_target/src/spec/targets/xtensa_esp32_espidf.rs
+++ b/compiler/rustc_target/src/spec/targets/xtensa_esp32_espidf.rs
@@ -9,7 +9,7 @@ pub(crate) fn target() -> Target {
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-v1:8:8-i64:64-i128:128-n32".into(),
         arch: Arch::Xtensa,
-        metadata: TargetMetadata { description: None, tier: None, host_tools: None, std: None },
+        metadata: TargetMetadata { description: None, tier: Some(3), host_tools: None, std: None },
 
         options: TargetOptions {
             endian: Endian::Little,
diff --git a/compiler/rustc_target/src/spec/targets/xtensa_esp32s2_espidf.rs b/compiler/rustc_target/src/spec/targets/xtensa_esp32s2_espidf.rs
index 42e4c7bf8bec..81eab657db71 100644
--- a/compiler/rustc_target/src/spec/targets/xtensa_esp32s2_espidf.rs
+++ b/compiler/rustc_target/src/spec/targets/xtensa_esp32s2_espidf.rs
@@ -9,7 +9,7 @@ pub(crate) fn target() -> Target {
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-v1:8:8-i64:64-i128:128-n32".into(),
         arch: Arch::Xtensa,
-        metadata: TargetMetadata { description: None, tier: None, host_tools: None, std: None },
+        metadata: TargetMetadata { description: None, tier: Some(3), host_tools: None, std: None },
 
         options: TargetOptions {
             endian: Endian::Little,
diff --git a/compiler/rustc_target/src/spec/targets/xtensa_esp32s3_espidf.rs b/compiler/rustc_target/src/spec/targets/xtensa_esp32s3_espidf.rs
index 498daf4e6063..fb1f4f471979 100644
--- a/compiler/rustc_target/src/spec/targets/xtensa_esp32s3_espidf.rs
+++ b/compiler/rustc_target/src/spec/targets/xtensa_esp32s3_espidf.rs
@@ -9,7 +9,7 @@ pub(crate) fn target() -> Target {
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-v1:8:8-i64:64-i128:128-n32".into(),
         arch: Arch::Xtensa,
-        metadata: TargetMetadata { description: None, tier: None, host_tools: None, std: None },
+        metadata: TargetMetadata { description: None, tier: Some(3), host_tools: None, std: None },
 
         options: TargetOptions {
             endian: Endian::Little,
diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
index 6bef58a0ac20..5f956f03ecb1 100644
--- a/src/bootstrap/mk/Makefile.in
+++ b/src/bootstrap/mk/Makefile.in
@@ -53,6 +53,11 @@ check-aux:
 		src/tools/cargotest \
 		src/tools/test-float-parse \
 		$(BOOTSTRAP_ARGS)
+	# The build-std suite is off by default because it is uncommonly slow
+	# and memory-hungry.
+	$(Q)$(BOOTSTRAP) test --stage 2 \
+		build-std \
+		$(BOOTSTRAP_ARGS)
 	# Run standard library tests in Miri.
 	$(Q)MIRIFLAGS="-Zmiri-strict-provenance" \
 		$(BOOTSTRAP) miri --stage 2 \
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index f3a1c6b0e3dd..080931fc0a93 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -1625,6 +1625,12 @@ test!(RunMakeCargo {
     suite: "run-make-cargo",
     default: true
 });
+test!(BuildStd {
+    path: "tests/build-std",
+    mode: CompiletestMode::RunMake,
+    suite: "build-std",
+    default: false
+});
 
 test!(AssemblyLlvm {
     path: "tests/assembly-llvm",
@@ -1948,7 +1954,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
             let stage0_rustc_path = builder.compiler(0, test_compiler.host);
             cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path));
 
-            if suite == "run-make-cargo" {
+            if matches!(suite, "run-make-cargo" | "build-std") {
                 let cargo_path = if test_compiler.stage == 0 {
                     // If we're using `--stage 0`, we should provide the bootstrap cargo.
                     builder.initial_cargo.clone()
diff --git a/src/bootstrap/src/core/builder/cli_paths.rs b/src/bootstrap/src/core/builder/cli_paths.rs
index 5ff2b380e4b9..1b0caa980e1d 100644
--- a/src/bootstrap/src/core/builder/cli_paths.rs
+++ b/src/bootstrap/src/core/builder/cli_paths.rs
@@ -20,6 +20,7 @@ pub(crate) const PATH_REMAP: &[(&str, &[&str])] = &[
         &[
             // tidy-alphabetical-start
             "tests/assembly-llvm",
+            "tests/build-std",
             "tests/codegen-llvm",
             "tests/codegen-units",
             "tests/coverage",
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests.snap
index ad9660ef5c91..65349a59a1e8 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests.snap
@@ -5,6 +5,9 @@ expression: test tests
 [Test] test::AssemblyLlvm
     targets: [aarch64-unknown-linux-gnu]
     - Suite(test::tests/assembly-llvm)
+[Test] test::BuildStd
+    targets: [aarch64-unknown-linux-gnu]
+    - Suite(test::tests/build-std)
 [Test] test::CodegenLlvm
     targets: [aarch64-unknown-linux-gnu]
     - Suite(test::tests/codegen-llvm)
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests_skip_coverage.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests_skip_coverage.snap
index 4572f089b0ae..694bb0672ee4 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests_skip_coverage.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_tests_skip_coverage.snap
@@ -5,6 +5,9 @@ expression: test tests --skip=coverage
 [Test] test::AssemblyLlvm
     targets: [aarch64-unknown-linux-gnu]
     - Suite(test::tests/assembly-llvm)
+[Test] test::BuildStd
+    targets: [aarch64-unknown-linux-gnu]
+    - Suite(test::tests/build-std)
 [Test] test::CodegenLlvm
     targets: [aarch64-unknown-linux-gnu]
     - Suite(test::tests/codegen-llvm)
diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs
index 4a04b97c549a..7f98e42227af 100644
--- a/src/bootstrap/src/core/builder/mod.rs
+++ b/src/bootstrap/src/core/builder/mod.rs
@@ -927,6 +927,7 @@ impl<'a> Builder<'a> {
                 test::CollectLicenseMetadata,
                 test::RunMake,
                 test::RunMakeCargo,
+                test::BuildStd,
             ),
             Kind::Miri => describe!(test::Crate),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc, test::CrateRustdoc),
diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs
index 66614cc6cced..af26c2f87d02 100644
--- a/src/bootstrap/src/core/builder/tests.rs
+++ b/src/bootstrap/src/core/builder/tests.rs
@@ -2077,7 +2077,7 @@ mod snapshot {
         let ctx = TestCtx::new();
         insta::assert_snapshot!(
             prepare_test_config(&ctx)
-                .render_steps(), @r"
+                .render_steps(), @"
         [build] rustc 0  -> Tidy 1 
         [test] tidy <>
         [build] rustdoc 0 
@@ -2255,7 +2255,7 @@ mod snapshot {
         insta::assert_snapshot!(
             prepare_test_config(&ctx)
                 .stage(2)
-                .render_steps(), @r"
+                .render_steps(), @"
         [build] rustc 0  -> Tidy 1 
         [test] tidy <>
         [build] rustdoc 0 
diff --git a/src/doc/rustc-dev-guide/src/tests/compiletest.md b/src/doc/rustc-dev-guide/src/tests/compiletest.md
index d69e0a5ce988..7f22bc27600c 100644
--- a/src/doc/rustc-dev-guide/src/tests/compiletest.md
+++ b/src/doc/rustc-dev-guide/src/tests/compiletest.md
@@ -78,6 +78,10 @@ The following test suites are available, with links for more information:
 
 [`run-make`](#run-make-tests) are general purpose tests using Rust programs.
 
+### The build-std test suite
+
+[`build-std`](#build-std-tests) test that -Zbuild-std works.
+
 ### Rustdoc test suites
 
 | Test suite                           | Purpose                                                                  |
@@ -429,6 +433,12 @@ use cases that require testing in-tree `cargo` in conjunction with in-tree `rust
 The `run-make` test suite does not have access to in-tree `cargo` (so it can be the
 faster-to-iterate test suite).
 
+### `build-std` tests
+
+The tests in [`tests/build-std`] check that `-Zbuild-std` works. This is currently
+just a run-make test suite with a single recipe. The recipe generates test cases
+and runs them in parallel.
+
 #### Using Rust recipes
 
 Each test should be in a separate directory with a `rmake.rs` Rust program,
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index a2f3a8f00dac..843b4ad2975d 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -77,6 +77,7 @@ string_enum! {
         RustdocUi => "rustdoc-ui",
         Ui => "ui",
         UiFullDeps => "ui-fulldeps",
+        BuildStd => "build-std",
     }
 }
 
diff --git a/src/tools/compiletest/src/runtest/run_make.rs b/src/tools/compiletest/src/runtest/run_make.rs
index 4eb8f91fe894..ac8846a263c0 100644
--- a/src/tools/compiletest/src/runtest/run_make.rs
+++ b/src/tools/compiletest/src/runtest/run_make.rs
@@ -190,8 +190,8 @@ impl TestCx<'_> {
             // through a specific CI runner).
             .env("LLVM_COMPONENTS", &self.config.llvm_components);
 
-        // Only `run-make-cargo` test suite gets an in-tree `cargo`, not `run-make`.
-        if self.config.suite == TestSuite::RunMakeCargo {
+        // The `run-make-cargo` and `build-std` suites need an in-tree `cargo`, `run-make` does not.
+        if matches!(self.config.suite, TestSuite::RunMakeCargo | TestSuite::BuildStd) {
             cmd.env(
                 "CARGO",
                 self.config.cargo_path.as_ref().expect("cargo must be built and made available"),
diff --git a/src/tools/run-make-support/Cargo.toml b/src/tools/run-make-support/Cargo.toml
index 9d9cd656f570..918f5ef0d506 100644
--- a/src/tools/run-make-support/Cargo.toml
+++ b/src/tools/run-make-support/Cargo.toml
@@ -17,6 +17,7 @@ object = "0.37"
 regex = "1.11"
 serde_json = "1.0"
 similar = "2.7"
+tempfile = "3"
 wasmparser = { version = "0.236", default-features = false, features = ["std", "features", "validate"] }
 # tidy-alphabetical-end
 
diff --git a/src/tools/run-make-support/src/command.rs b/src/tools/run-make-support/src/command.rs
index 0aeb189bb6a5..f4a09f9faae8 100644
--- a/src/tools/run-make-support/src/command.rs
+++ b/src/tools/run-make-support/src/command.rs
@@ -46,6 +46,8 @@ pub struct Command {
     // Emulate linear type semantics.
     drop_bomb: DropBomb,
     already_executed: bool,
+
+    context: String,
 }
 
 impl Command {
@@ -60,6 +62,7 @@ impl Command {
             stdout: None,
             stderr: None,
             already_executed: false,
+            context: String::new(),
         }
     }
 
@@ -69,6 +72,16 @@ impl Command {
         self.cmd
     }
 
+    pub(crate) fn get_context(&self) -> &str {
+        &self.context
+    }
+
+    /// Appends context to the command, to provide a better error message if the command fails.
+    pub fn context(&mut self, ctx: &str) -> &mut Self {
+        self.context.push_str(&format!("{ctx}\n"));
+        self
+    }
+
     /// Specify a stdin input buffer. This is a convenience helper,
     pub fn stdin_buf>(&mut self, input: I) -> &mut Self {
         self.stdin_buf = Some(input.as_ref().to_vec().into_boxed_slice());
diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs
index 5253dc04a93e..b19d73b78a94 100644
--- a/src/tools/run-make-support/src/lib.rs
+++ b/src/tools/run-make-support/src/lib.rs
@@ -34,7 +34,9 @@ pub mod rfs {
 }
 
 // Re-exports of third-party library crates.
-pub use {bstr, gimli, libc, object, regex, rustdoc_json_types, serde_json, similar, wasmparser};
+pub use {
+    bstr, gimli, libc, object, regex, rustdoc_json_types, serde_json, similar, tempfile, wasmparser,
+};
 
 // Helpers for building names of output artifacts that are potentially target-specific.
 pub use crate::artifact_names::{
diff --git a/src/tools/run-make-support/src/util.rs b/src/tools/run-make-support/src/util.rs
index af01758447b9..7908dc1f7b3d 100644
--- a/src/tools/run-make-support/src/util.rs
+++ b/src/tools/run-make-support/src/util.rs
@@ -21,6 +21,9 @@ pub(crate) fn handle_failed_output(
     eprintln!("output status: `{}`", output.status());
     eprintln!("=== STDOUT ===\n{}\n\n", output.stdout_utf8());
     eprintln!("=== STDERR ===\n{}\n\n", output.stderr_utf8());
+    if !cmd.get_context().is_empty() {
+        eprintln!("Context:\n{}", cmd.get_context());
+    }
     std::process::exit(1)
 }
 
diff --git a/tests/build-std/configurations/rmake.rs b/tests/build-std/configurations/rmake.rs
new file mode 100644
index 000000000000..99cb9f9b0f4d
--- /dev/null
+++ b/tests/build-std/configurations/rmake.rs
@@ -0,0 +1,131 @@
+// This test ensures we are able to compile -Zbuild-std=core under a variety of profiles.
+// Currently, it tests that we can compile to all Tier 1 targets, and it does this by checking what
+// the tier metadata in target-spec JSON. This means that all in-tree targets must have a tier set.
+
+#![deny(warnings)]
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::thread;
+
+use run_make_support::serde_json::{self, Value};
+use run_make_support::tempfile::TempDir;
+use run_make_support::{cargo, rfs, rustc};
+
+#[derive(Clone)]
+struct Task {
+    target: String,
+    opt_level: u8,
+    debug: u8,
+    panic: &'static str,
+}
+
+fn manifest(task: &Task) -> String {
+    let Task { opt_level, debug, panic, target: _ } = task;
+    format!(
+        r#"[package]
+name = "scratch"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+path = "lib.rs"
+
+[profile.release]
+opt-level = {opt_level}
+debug = {debug}
+panic = "{panic}"
+"#
+    )
+}
+
+fn main() {
+    let mut targets = Vec::new();
+    let all_targets =
+        rustc().args(&["--print=all-target-specs-json", "-Zunstable-options"]).run().stdout_utf8();
+    let all_targets: HashMap = serde_json::from_str(&all_targets).unwrap();
+    for (target, spec) in all_targets {
+        let metadata = spec.as_object().unwrap()["metadata"].as_object().unwrap();
+        let tier = metadata["tier"]
+            .as_u64()
+            .expect(&format!("Target {} is missing tier metadata", target));
+        if tier == 1 {
+            targets.push(target);
+        }
+    }
+
+    let mut tasks = Vec::new();
+
+    // Testing every combination of compiler flags is infeasible. So we are making some attempt to
+    // choose combinations that will tend to run into problems.
+    //
+    // The particular combination of settings below is tuned to look for problems generating the
+    // code for compiler-builtins.
+    // We only exercise opt-level 0 and 3 to exercise mir-opt-level 1 and 2.
+    // We only exercise debug 0 and 2 because level 2 turns off some MIR optimizations.
+    // We only test abort and immediate-abort because abort vs unwind doesn't change MIR much at
+    // all. but immediate-abort does.
+    //
+    // Currently this only tests that we can compile the tier 1 targets. But since we are using
+    // -Zbuild-std=core, we could have any list of targets.
+
+    for opt_level in [0, 3] {
+        for debug in [0, 2] {
+            for panic in ["abort", "immediate-abort"] {
+                for target in &targets {
+                    tasks.push(Task { target: target.clone(), opt_level, debug, panic });
+                }
+            }
+        }
+    }
+
+    let tasks = Arc::new(Mutex::new(tasks));
+    let mut threads = Vec::new();
+
+    // Try to obey the -j argument passed to bootstrap, otherwise fall back to using all the system
+    // resouces. This test can be rather memory-hungry (~1 GB/thread); if it causes trouble in
+    // practice do not hesitate to limit its parallelism.
+    for _ in 0..run_make_support::env::jobs() {
+        let tasks = Arc::clone(&tasks);
+        let handle = thread::spawn(move || {
+            loop {
+                let maybe_task = tasks.lock().unwrap().pop();
+                if let Some(task) = maybe_task {
+                    test(task);
+                } else {
+                    break;
+                }
+            }
+        });
+        threads.push(handle);
+    }
+
+    for t in threads {
+        t.join().unwrap();
+    }
+}
+
+fn test(task: Task) {
+    let dir = TempDir::new().unwrap();
+
+    let manifest = manifest(&task);
+    rfs::write(dir.path().join("Cargo.toml"), &manifest);
+    rfs::write(dir.path().join("lib.rs"), "#![no_std]");
+
+    let mut args = vec!["build", "--release", "-Zbuild-std=core", "--target", &task.target, "-j1"];
+    if task.panic == "immediate-abort" {
+        args.push("-Zpanic-immediate-abort");
+    }
+    cargo()
+        .current_dir(dir.path())
+        .args(&args)
+        .env("RUSTC_BOOTSTRAP", "1")
+        // Visual Studio 2022 requires that the LIB env var be set so it can
+        // find the Windows SDK.
+        .env("LIB", std::env::var("LIB").unwrap_or_default())
+        .context(&format!(
+            "build-std for target `{}` failed with the following Cargo.toml:\n\n{manifest}",
+            task.target
+        ))
+        .run();
+}

From c31add03870e80556703be3c9544109328559379 Mon Sep 17 00:00:00 2001
From: The rustc-josh-sync Cronjob Bot 
Date: Thu, 15 Jan 2026 04:21:53 +0000
Subject: [PATCH 0258/1843] Prepare for merging from rust-lang/rust

This updates the rust-version file to b6fdaf2a15736cbccf248b532f48e33179614d40.
---
 src/tools/rust-analyzer/rust-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rust-analyzer/rust-version b/src/tools/rust-analyzer/rust-version
index b53a66c66751..a6ccd9bab393 100644
--- a/src/tools/rust-analyzer/rust-version
+++ b/src/tools/rust-analyzer/rust-version
@@ -1 +1 @@
-44a5b55557c26353f388400d7da95527256fe260
+b6fdaf2a15736cbccf248b532f48e33179614d40

From d59c47c2bc933f46e63ea5f23618f9618c0b9f51 Mon Sep 17 00:00:00 2001
From: The Miri Cronjob Bot 
Date: Thu, 15 Jan 2026 05:01:05 +0000
Subject: [PATCH 0259/1843] Prepare for merging from rust-lang/rust

This updates the rust-version file to b6fdaf2a15736cbccf248b532f48e33179614d40.
---
 src/tools/miri/rust-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/miri/rust-version b/src/tools/miri/rust-version
index b53a66c66751..a6ccd9bab393 100644
--- a/src/tools/miri/rust-version
+++ b/src/tools/miri/rust-version
@@ -1 +1 @@
-44a5b55557c26353f388400d7da95527256fe260
+b6fdaf2a15736cbccf248b532f48e33179614d40

From d7e5996e4f55158af402fc77322812bea46eb8c2 Mon Sep 17 00:00:00 2001
From: Hugh 
Date: Wed, 14 Jan 2026 23:12:46 -0800
Subject: [PATCH 0260/1843] Skip elidable_lifetime_names lint for proc-macro
 generated code

When linting code generated by proc macros (like `derivative`), the
`elidable_lifetime_names` lint can produce fix suggestions with
overlapping spans. This causes `cargo clippy --fix` to fail with
"cannot replace slice of data that was already replaced".

This change adds `is_from_proc_macro` checks to the three lint entry
points (`check_item`, `check_impl_item`, `check_trait_item`) to skip
linting proc-macro generated code entirely.

Fixes #16316
---
 clippy_lints/src/lifetimes.rs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/clippy_lints/src/lifetimes.rs b/clippy_lints/src/lifetimes.rs
index 727e9b172a87..8917c90262a4 100644
--- a/clippy_lints/src/lifetimes.rs
+++ b/clippy_lints/src/lifetimes.rs
@@ -1,7 +1,7 @@
 use clippy_config::Conf;
 use clippy_utils::diagnostics::{span_lint, span_lint_and_then};
 use clippy_utils::msrvs::{self, Msrv};
-use clippy_utils::trait_ref_of_method;
+use clippy_utils::{is_from_proc_macro, trait_ref_of_method};
 use itertools::Itertools;
 use rustc_ast::visit::{try_visit, walk_list};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
@@ -149,16 +149,22 @@ impl<'tcx> LateLintPass<'tcx> for Lifetimes {
             ..
         } = item.kind
         {
+            if is_from_proc_macro(cx, item) {
+                return;
+            }
             check_fn_inner(cx, sig, Some(id), None, generics, item.span, true, self.msrv);
         } else if let ItemKind::Impl(impl_) = &item.kind
             && !item.span.from_expansion()
+            && !is_from_proc_macro(cx, item)
         {
             report_extra_impl_lifetimes(cx, impl_);
         }
     }
 
     fn check_impl_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx ImplItem<'_>) {
-        if let ImplItemKind::Fn(ref sig, id) = item.kind {
+        if let ImplItemKind::Fn(ref sig, id) = item.kind
+            && !is_from_proc_macro(cx, item)
+        {
             let report_extra_lifetimes = trait_ref_of_method(cx, item.owner_id).is_none();
             check_fn_inner(
                 cx,
@@ -174,7 +180,9 @@ impl<'tcx> LateLintPass<'tcx> for Lifetimes {
     }
 
     fn check_trait_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx TraitItem<'_>) {
-        if let TraitItemKind::Fn(ref sig, ref body) = item.kind {
+        if let TraitItemKind::Fn(ref sig, ref body) = item.kind
+            && !is_from_proc_macro(cx, item)
+        {
             let (body, trait_sig) = match *body {
                 TraitFn::Required(sig) => (None, Some(sig)),
                 TraitFn::Provided(id) => (Some(id), None),

From ea9b062a8a71d16f8327ddf7d9ceac405a9a3bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Thu, 15 Jan 2026 11:15:26 +0100
Subject: [PATCH 0261/1843] Add GCC to build-manifest

---
 src/tools/build-manifest/src/main.rs     |  7 +--
 src/tools/build-manifest/src/versions.rs | 56 +++++++++++++++++++-----
 2 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 7c0c528bcc04..cd6efad01141 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -158,7 +158,7 @@ impl Builder {
     }
 
     fn add_packages_to(&mut self, manifest: &mut Manifest) {
-        for pkg in PkgType::all() {
+        for pkg in &PkgType::all() {
             self.package(pkg, &mut manifest.pkg);
         }
     }
@@ -227,7 +227,7 @@ impl Builder {
         };
         for pkg in PkgType::all() {
             if pkg.is_preview() {
-                rename(pkg.tarball_component_name(), &pkg.manifest_component_name());
+                rename(&pkg.tarball_component_name(), &pkg.manifest_component_name());
             }
         }
     }
@@ -263,7 +263,7 @@ impl Builder {
 
         let host_component = |pkg: &_| Component::from_pkg(pkg, host);
 
-        for pkg in PkgType::all() {
+        for pkg in &PkgType::all() {
             match pkg {
                 // rustc/rust-std/cargo/docs are all required
                 PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => {
@@ -303,6 +303,7 @@ impl Builder {
                 | PkgType::JsonDocs
                 | PkgType::RustcCodegenCranelift
                 | PkgType::RustcCodegenGcc
+                | PkgType::Gcc { .. }
                 | PkgType::LlvmBitcodeLinker => {
                     extensions.push(host_component(pkg));
                 }
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 1e55d1d467e6..b53f6c5edc5d 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -11,29 +11,54 @@ use xz2::read::XzDecoder;
 const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
 
 macro_rules! pkg_type {
-    ( $($variant:ident = $component:literal $(; preview = true $(@$is_preview:tt)? )? ),+ $(,)? ) => {
+    ( $($variant:ident = $component:literal $(; preview = true $(@$is_preview:tt)? )? $(; suffixes = [$($suffixes:literal),+] $(@$is_suffixed:tt)? )? ),+ $(,)? ) => {
         #[derive(Debug, Hash, Eq, PartialEq, Clone)]
         pub(crate) enum PkgType {
-            $($variant,)+
+            $($variant $( $($is_suffixed)? { suffix: &'static str })?,)+
         }
 
         impl PkgType {
             pub(crate) fn is_preview(&self) -> bool {
                 match self {
-                    $( $( $($is_preview)? PkgType::$variant => true, )? )+
-                    _ => false,
+                    $( PkgType::$variant $($($is_suffixed)? { .. })? => false $( $($is_preview)? || true)?, )+
                 }
             }
 
-            /// First part of the tarball name.
-            pub(crate) fn tarball_component_name(&self) -> &str {
+            /// First part of the tarball name. May include a suffix, if the package has one.
+            pub(crate) fn tarball_component_name(&self) -> String {
                 match self {
-                    $( PkgType::$variant => $component,)+
+                    $( PkgType::$variant $($($is_suffixed)? { suffix })? => {
+                        #[allow(unused_mut)]
+                        let mut name = $component.to_owned();
+                        $($($is_suffixed)?
+                        name.push('-');
+                        name.push_str(suffix);
+                        )?
+                        name
+                    },)+
                 }
             }
 
-            pub(crate) fn all() -> &'static [PkgType] {
-                &[ $(PkgType::$variant),+ ]
+            pub(crate) fn all() -> Vec {
+                let mut packages = vec![];
+                $(
+                    // Push the single variant
+                    packages.push(PkgType::$variant $($($is_suffixed)? { suffix: "" })?);
+                    // Macro hell, we have to remove the fake empty suffix if we actually have
+                    // suffixes
+                    $(
+                        $($is_suffixed)?
+                        packages.pop();
+                    )?
+                    // And now add the suffixes, if any
+                    $(
+                        $($is_suffixed)?
+                        $(
+                            packages.push(PkgType::$variant { suffix: $suffixes });
+                        )+
+                    )?
+                )+
+                packages
             }
         }
     }
@@ -60,6 +85,9 @@ pkg_type! {
     RustcCodegenCranelift = "rustc-codegen-cranelift"; preview = true,
     LlvmBitcodeLinker = "llvm-bitcode-linker"; preview = true,
     RustcCodegenGcc = "rustc-codegen-gcc"; preview = true,
+    Gcc = "gcc"; preview = true; suffixes = [
+        "x86_64-unknown-linux-gnu"
+    ],
 }
 
 impl PkgType {
@@ -68,7 +96,7 @@ impl PkgType {
         if self.is_preview() {
             format!("{}-preview", self.tarball_component_name())
         } else {
-            self.tarball_component_name().to_string()
+            self.tarball_component_name()
         }
     }
 
@@ -84,6 +112,7 @@ impl PkgType {
             PkgType::Miri => false,
             PkgType::RustcCodegenCranelift => false,
             PkgType::RustcCodegenGcc => false,
+            PkgType::Gcc { suffix: _ } => false,
 
             PkgType::Rust => true,
             PkgType::RustStd => true,
@@ -114,6 +143,13 @@ impl PkgType {
             Cargo => HOSTS,
             RustcCodegenCranelift => HOSTS,
             RustcCodegenGcc => HOSTS,
+            // Gcc is "special", because we need a separate libgccjit.so for each
+            // (host, target) compilation pair. So it's even more special than stdlib, which has a
+            // separate component per target. This component thus hardcodes its compilation
+            // target in its name, and we thus ship it for HOSTS only. So we essentially have
+            // gcc-T1, gcc-T2, a separate *component/package* per each compilation target.
+            // So on host T1, if you want to compile for T2, you would install gcc-T2.
+            Gcc { suffix: _ } => HOSTS,
             RustMingw => MINGW,
             RustStd => TARGETS,
             HtmlDocs => HOSTS,

From 55abc484d7243eea77fd40247b531bcb0796e697 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Thu, 15 Jan 2026 11:29:07 +0100
Subject: [PATCH 0262/1843] Extend build-manifest local test guide

Fill in more blanks about how to test build-manifest changes with Rustup.
---
 src/tools/build-manifest/README.md | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
index bc1992ef80cc..f88949f48d8b 100644
--- a/src/tools/build-manifest/README.md
+++ b/src/tools/build-manifest/README.md
@@ -18,7 +18,7 @@ This gets called by `promote-release`  build/dist/channel-rust-nightly.toml.sha256
+```
+
+And start a HTTP server from the `build` directory:
+```sh
+cd build
+python3 -m http.server 8000
+```
+
+After you do all that, you can then install the locally generated components with rustup:
+```
+rustup uninstall nightly
+RUSTUP_DIST_SERVER=http://localhost:8000 rustup toolchain install nightly --profile minimal
+RUSTUP_DIST_SERVER=http://localhost:8000 rustup +nightly component add 
+```
+
+Note that generally it will not work to combine components built locally and those built from CI (nightly). Ideally, if you want to ship new rustup components, first dist them in nightly, and then test everything from nightly here after it's available on CI.

From 316627ee24c2214037631c227dc073bde46001e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Tue, 16 Dec 2025 21:19:20 +0100
Subject: [PATCH 0263/1843] Refactor `Enzyme` step

---
 src/bootstrap/src/core/build_steps/compile.rs | 22 +++------
 src/bootstrap/src/core/build_steps/llvm.rs    | 49 +++++++++++++++----
 2 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 11f2a28bb935..651ff03a8690 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -2292,23 +2292,13 @@ impl Step for Assemble {
             builder.compiler(target_compiler.stage - 1, builder.config.host_target);
 
         // Build enzyme
-        if builder.config.llvm_enzyme && !builder.config.dry_run() {
+        if builder.config.llvm_enzyme {
             debug!("`llvm_enzyme` requested");
-            let enzyme_install = builder.ensure(llvm::Enzyme { target: build_compiler.host });
-            if let Some(llvm_config) = builder.llvm_config(builder.config.host_target) {
-                let llvm_version_major = llvm::get_llvm_version_major(builder, &llvm_config);
-                let lib_ext = std::env::consts::DLL_EXTENSION;
-                let libenzyme = format!("libEnzyme-{llvm_version_major}");
-                let src_lib =
-                    enzyme_install.join("build/Enzyme").join(&libenzyme).with_extension(lib_ext);
-                let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host);
-                let target_libdir =
-                    builder.sysroot_target_libdir(target_compiler, target_compiler.host);
-                let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext);
-                let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext);
-                builder.copy_link(&src_lib, &dst_lib, FileType::NativeLibrary);
-                builder.copy_link(&src_lib, &target_dst_lib, FileType::NativeLibrary);
-            }
+            let enzyme = builder.ensure(llvm::Enzyme { target: build_compiler.host });
+            let target_libdir =
+                builder.sysroot_target_libdir(target_compiler, target_compiler.host);
+            let target_dst_lib = target_libdir.join(enzyme.enzyme_filename());
+            builder.copy_link(&enzyme.enzyme_path(), &target_dst_lib, FileType::NativeLibrary);
         }
 
         if builder.config.llvm_offload && !builder.config.dry_run() {
diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs
index c3935d9810e9..f6a763b94075 100644
--- a/src/bootstrap/src/core/build_steps/llvm.rs
+++ b/src/bootstrap/src/core/build_steps/llvm.rs
@@ -17,6 +17,7 @@ use std::{env, fs};
 use build_helper::exit;
 use build_helper::git::PathFreshness;
 
+use crate::core::build_steps::llvm;
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step, StepMetadata};
 use crate::core::config::{Config, TargetSelection};
 use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
@@ -1077,13 +1078,28 @@ impl Step for OmpOffload {
     }
 }
 
+#[derive(Clone)]
+pub struct BuiltEnzyme {
+    /// Path to the libEnzyme dylib.
+    enzyme: PathBuf,
+}
+
+impl BuiltEnzyme {
+    pub fn enzyme_path(&self) -> PathBuf {
+        self.enzyme.clone()
+    }
+    pub fn enzyme_filename(&self) -> String {
+        self.enzyme.file_name().unwrap().to_str().unwrap().to_owned()
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Enzyme {
     pub target: TargetSelection,
 }
 
 impl Step for Enzyme {
-    type Output = PathBuf;
+    type Output = BuiltEnzyme;
     const IS_HOST: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
@@ -1095,17 +1111,17 @@ impl Step for Enzyme {
     }
 
     /// Compile Enzyme for `target`.
-    fn run(self, builder: &Builder<'_>) -> PathBuf {
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
         builder.require_submodule(
             "src/tools/enzyme",
             Some("The Enzyme sources are required for autodiff."),
         );
-        if builder.config.dry_run() {
-            let out_dir = builder.enzyme_out(self.target);
-            return out_dir;
-        }
         let target = self.target;
 
+        if builder.config.dry_run() {
+            return BuiltEnzyme { enzyme: builder.config.tempdir().join("enzyme-dryrun") };
+        }
+
         let LlvmResult { host_llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target });
 
         static STAMP_HASH_MEMO: OnceLock = OnceLock::new();
@@ -1120,6 +1136,12 @@ impl Step for Enzyme {
         let out_dir = builder.enzyme_out(target);
         let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash);
 
+        let llvm_version_major = llvm::get_llvm_version_major(builder, &host_llvm_config);
+        let lib_ext = std::env::consts::DLL_EXTENSION;
+        let libenzyme = format!("libEnzyme-{llvm_version_major}");
+        let build_dir = out_dir.join("lib");
+        let dylib = build_dir.join(&libenzyme).with_extension(lib_ext);
+
         trace!("checking build stamp to see if we need to rebuild enzyme artifacts");
         if stamp.is_up_to_date() {
             trace!(?out_dir, "enzyme build artifacts are up to date");
@@ -1133,7 +1155,7 @@ impl Step for Enzyme {
                     stamp.path().display()
                 ));
             }
-            return out_dir;
+            return BuiltEnzyme { enzyme: dylib };
         }
 
         if !builder.config.dry_run() && !llvm_cmake_dir.is_dir() {
@@ -1149,7 +1171,6 @@ impl Step for Enzyme {
         let _time = helpers::timeit(builder);
         t!(fs::create_dir_all(&out_dir));
 
-        builder.config.update_submodule("src/tools/enzyme");
         let mut cfg = cmake::Config::new(builder.src.join("src/tools/enzyme/enzyme/"));
         // Enzyme devs maintain upstream compatibility, but only fix deprecations when they are about
         // to turn into a hard error. As such, Enzyme generates various warnings which could make it
@@ -1178,8 +1199,18 @@ impl Step for Enzyme {
 
         cfg.build();
 
+        // At this point, `out_dir` should contain the built libEnzyme-.
+        // file.
+        if !dylib.exists() {
+            eprintln!(
+                "`{libenzyme}` not found in `{}`. Either the build has failed or Enzyme was built with a wrong version of LLVM",
+                build_dir.display()
+            );
+            exit!(1);
+        }
+
         t!(stamp.write());
-        out_dir
+        BuiltEnzyme { enzyme: dylib }
     }
 }
 

From 85e01e3c4e328a63b39ea1189de03b79cc02b0cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Tue, 16 Dec 2025 21:55:56 +0100
Subject: [PATCH 0264/1843] Add dist step for `Enzyme`

---
 src/bootstrap/src/core/build_steps/dist.rs | 49 ++++++++++++++++++++++
 src/bootstrap/src/core/builder/mod.rs      |  1 +
 src/bootstrap/src/utils/tarball.rs         |  3 ++
 3 files changed, 53 insertions(+)

diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index cfcb144e0993..8ba05a7fa3a7 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -2717,6 +2717,55 @@ impl Step for LlvmBitcodeLinker {
     }
 }
 
+/// Distributes the `enzyme` library so that it can be used by a compiler whose host
+/// is `target`.
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Enzyme {
+    /// Enzyme will by usable by rustc on this host.
+    pub target: TargetSelection,
+}
+
+impl Step for Enzyme {
+    type Output = Option;
+    const IS_HOST: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("enzyme")
+    }
+
+    fn is_default_step(builder: &Builder<'_>) -> bool {
+        builder.config.llvm_enzyme
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Enzyme { target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Option {
+        // This prevents Enzyme from being built for "dist"
+        // or "install" on the stable/beta channels. It is not yet stable and
+        // should not be included.
+        if !builder.build.unstable_features() {
+            return None;
+        }
+
+        let target = self.target;
+
+        let enzyme = builder.ensure(llvm::Enzyme { target });
+
+        let target_libdir = format!("lib/rustlib/{}/lib", target.triple);
+
+        // Prepare the image directory
+        let mut tarball = Tarball::new(builder, "enzyme", &target.triple);
+        tarball.set_overlay(OverlayKind::Enzyme);
+        tarball.is_preview(true);
+
+        tarball.add_file(enzyme.enzyme_path(), target_libdir, FileType::NativeLibrary);
+
+        Some(tarball.generate())
+    }
+}
+
 /// Tarball intended for internal consumption to ease rustc/std development.
 ///
 /// Should not be considered stable by end users.
diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs
index f63b8e044550..14cae2d0efc4 100644
--- a/src/bootstrap/src/core/builder/mod.rs
+++ b/src/bootstrap/src/core/builder/mod.rs
@@ -980,6 +980,7 @@ impl<'a> Builder<'a> {
                 dist::LlvmTools,
                 dist::LlvmBitcodeLinker,
                 dist::RustDev,
+                dist::Enzyme,
                 dist::Bootstrap,
                 dist::Extended,
                 // It seems that PlainSourceTarball somehow changes how some of the tools
diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs
index cd6a03c84870..17d75e83daea 100644
--- a/src/bootstrap/src/utils/tarball.rs
+++ b/src/bootstrap/src/utils/tarball.rs
@@ -28,6 +28,7 @@ pub(crate) enum OverlayKind {
     RustcCodegenGcc,
     Gcc,
     LlvmBitcodeLinker,
+    Enzyme,
 }
 
 impl OverlayKind {
@@ -37,6 +38,7 @@ impl OverlayKind {
             OverlayKind::Llvm => {
                 &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
             }
+            OverlayKind::Enzyme => &["src/tools/enzyme/LICENSE", "src/tools/enzyme/Readme.md"],
             OverlayKind::Cargo => &[
                 "src/tools/cargo/README.md",
                 "src/tools/cargo/LICENSE-MIT",
@@ -111,6 +113,7 @@ impl OverlayKind {
             OverlayKind::RustcCodegenGcc => builder.rust_version(),
             OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
             OverlayKind::Gcc => builder.rust_version(),
+            OverlayKind::Enzyme => builder.rust_version(),
         }
     }
 }

From d85965634996e60e80440b4fef7415b6379d2487 Mon Sep 17 00:00:00 2001
From: lummax 
Date: Thu, 15 Jan 2026 12:54:01 +0100
Subject: [PATCH 0265/1843] fix: lookup flycheck by ID instead of vector index

After a recent introduction of per-package flycheck for JSON projects, the code
assumed that `world.flycheck` indices matched `world.workspaces` indices.
However, not all workspaces have flycheck enabled (e.g., JSON projects
without a flycheck template configured), so the flycheck vector can be
shorter than the workspaces vector.

This caused an index-out-of-bounds panic when saving a file in a JSON
project without flycheck configured:

  thread 'Worker' panicked at notification.rs:
  index out of bounds: the len is 0 but the index is 0

Fix by looking up the flycheck handle by its ID (which is the workspace
index set during spawn) rather than using the workspace index directly
as a vector index.
---
 .../src/handlers/notification.rs              | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
index d95601043330..6cc40677fb51 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
@@ -387,14 +387,21 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool {
                                         } => false,
                                     });
                                 if let Some(idx) = package_workspace_idx {
-                                    let workspace_deps =
-                                        world.all_workspace_dependencies_for_package(&package);
-                                    world.flycheck[idx].restart_for_package(
-                                        package,
-                                        target,
-                                        workspace_deps,
-                                        saved_file.clone(),
-                                    );
+                                    // flycheck handles are indexed by their ID (which is the workspace index),
+                                    // but not all workspaces have flycheck enabled (e.g., JSON projects without
+                                    // a flycheck template). Find the flycheck handle by its ID.
+                                    if let Some(flycheck) =
+                                        world.flycheck.iter().find(|fc| fc.id() == idx)
+                                    {
+                                        let workspace_deps =
+                                            world.all_workspace_dependencies_for_package(&package);
+                                        flycheck.restart_for_package(
+                                            package,
+                                            target,
+                                            workspace_deps,
+                                            saved_file.clone(),
+                                        );
+                                    }
                                 }
                             }
                         }

From 628bacac3312357af71816b0875174062b310a45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maja=20K=C4=85dzio=C5=82ka?= 
Date: Sun, 4 Jan 2026 04:22:55 +0100
Subject: [PATCH 0266/1843] add tests for emergent behavior of partial captures

---
 .../auxiliary/partial_move_drop_order_lib.rs  |  11 ++
 .../match/auxiliary/partial_move_lib.rs       |   4 +
 .../match/partial-move-drop-order.rs          |  83 ++++++++++++
 .../match/partial-move-drop-order.run.stdout  |  23 ++++
 .../match/partial-move.rs                     | 118 +++++++++++++++++
 .../match/partial-move.stderr                 | 123 ++++++++++++++++++
 6 files changed, 362 insertions(+)
 create mode 100644 tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_drop_order_lib.rs
 create mode 100644 tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_lib.rs
 create mode 100644 tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs
 create mode 100644 tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout
 create mode 100644 tests/ui/closures/2229_closure_analysis/match/partial-move.rs
 create mode 100644 tests/ui/closures/2229_closure_analysis/match/partial-move.stderr

diff --git a/tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_drop_order_lib.rs b/tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_drop_order_lib.rs
new file mode 100644
index 000000000000..8cf25c6ba7b2
--- /dev/null
+++ b/tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_drop_order_lib.rs
@@ -0,0 +1,11 @@
+pub struct LoudDrop(pub &'static str);
+impl Drop for LoudDrop {
+    fn drop(&mut self) {
+        println!("dropping {}", self.0);
+    }
+}
+
+#[non_exhaustive]
+pub enum ExtNonExhaustive {
+    One(i32, LoudDrop),
+}
diff --git a/tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_lib.rs b/tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_lib.rs
new file mode 100644
index 000000000000..b0646f1d0032
--- /dev/null
+++ b/tests/ui/closures/2229_closure_analysis/match/auxiliary/partial_move_lib.rs
@@ -0,0 +1,4 @@
+#[non_exhaustive]
+pub enum ExtNonExhaustive {
+    A(u32, String),
+}
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs
new file mode 100644
index 000000000000..1c178bd3f684
--- /dev/null
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs
@@ -0,0 +1,83 @@
+//@ edition:2021
+//@ run-pass
+//@ check-run-results
+//@ aux-build:partial_move_drop_order_lib.rs
+
+extern crate partial_move_drop_order_lib;
+use partial_move_drop_order_lib::{LoudDrop, ExtNonExhaustive};
+
+pub enum OneVariant {
+    One(i32, LoudDrop),
+}
+
+pub enum TwoVariants {
+    One(i32, LoudDrop),
+    Two,
+}
+
+#[non_exhaustive]
+pub enum NonExhaustive {
+    One(i32, LoudDrop),
+}
+
+#[allow(unused)]
+fn one_variant() {
+    println!("one variant:");
+    let mut thing = OneVariant::One(0, LoudDrop("a"));
+    let closure = move || match thing {
+        OneVariant::One(x, _) => {}
+        _ => unreachable!(),
+    };
+    println!("before assign");
+    thing = OneVariant::One(1, LoudDrop("b"));
+    println!("after assign");
+}
+
+#[allow(unused)]
+fn two_variants() {
+    println!("two variants:");
+    let mut thing = TwoVariants::One(0, LoudDrop("a"));
+    let closure = move || match thing {
+        TwoVariants::One(x, _) => {}
+        _ => unreachable!(),
+    };
+    println!("before assign");
+    thing = TwoVariants::One(1, LoudDrop("b"));
+    println!("after assign");
+}
+
+#[allow(unused)]
+fn non_exhaustive() {
+    println!("non exhaustive:");
+    let mut thing = NonExhaustive::One(0, LoudDrop("a"));
+    let closure = move || match thing {
+        NonExhaustive::One(x, _) => {}
+        _ => unreachable!(),
+    };
+    println!("before assign");
+    thing = NonExhaustive::One(1, LoudDrop("b"));
+    println!("after assign");
+}
+
+#[allow(unused)]
+fn ext_non_exhaustive() {
+    println!("external non exhaustive:");
+    let mut thing = ExtNonExhaustive::One(0, LoudDrop("a"));
+    let closure = move || match thing {
+        ExtNonExhaustive::One(x, _) => {}
+        _ => unreachable!(),
+    };
+    println!("before assign");
+    thing = ExtNonExhaustive::One(1, LoudDrop("b"));
+    println!("after assign");
+}
+
+fn main() {
+    one_variant();
+    println!();
+    two_variants();
+    println!();
+    non_exhaustive();
+    println!();
+    ext_non_exhaustive();
+}
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout
new file mode 100644
index 000000000000..e0d83dd99549
--- /dev/null
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout
@@ -0,0 +1,23 @@
+one variant:
+before assign
+dropping a
+after assign
+dropping b
+
+two variants:
+before assign
+after assign
+dropping a
+dropping b
+
+non exhaustive:
+before assign
+dropping a
+after assign
+dropping b
+
+external non exhaustive:
+before assign
+after assign
+dropping a
+dropping b
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move.rs b/tests/ui/closures/2229_closure_analysis/match/partial-move.rs
new file mode 100644
index 000000000000..62d1b362cb83
--- /dev/null
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move.rs
@@ -0,0 +1,118 @@
+// This test measures the effect of matching-induced partial captures on the borrow checker.
+// In particular, in each of the cases below, the closure either captures the entire enum/struct,
+// or each field separately.
+//
+// If the entire ADT gets captured, it'll happen by move, and the closure will live for 'static.
+// On the other hand, if each field gets captured separately, the u32 field, being Copy, will only
+// get captured by an immutable borrow, resulting in a borrow checker error.
+//
+// See rust-lang/rust#147722
+//
+//@ edition:2021
+//@ aux-build:partial_move_lib.rs
+pub struct Struct(u32, String);
+
+pub enum Enum {
+    A(u32, String),
+}
+
+pub enum TwoVariants {
+    A(u32, String),
+    B,
+}
+
+#[non_exhaustive]
+pub enum NonExhaustive {
+    A(u32, String),
+}
+
+extern crate partial_move_lib;
+use partial_move_lib::ExtNonExhaustive;
+
+// First, let's assert that the additional wildcard arm is not a source of any behavior
+// differences:
+pub fn test_enum1(x: Enum) -> impl FnOnce() {
+    || {
+    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
+        match x {
+            Enum::A(a, b) => {
+                drop((a, b));
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+pub fn test_enum2(x: Enum) -> impl FnOnce() {
+    || {
+    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
+        match x {
+            Enum::A(a, b) => {
+                drop((a, b));
+            }
+        }
+    }
+}
+
+// The behavior for single-variant enums matches what happens for a struct
+pub fn test_struct(x: Struct) -> impl FnOnce() {
+    || {
+    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
+        match x {
+            Struct(a, b) => {
+                drop((a, b));
+            }
+        }
+    }
+}
+
+// If we have two variants, the entire enum gets moved into the closure
+pub fn test_two_variants(x: TwoVariants) -> impl FnOnce() {
+    || {
+        match x {
+            TwoVariants::A(a, b) => {
+                drop((a, b));
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+// ...and single-variant, non-exhaustive enums *should* behave as if they had multiple variants
+pub fn test_non_exhaustive1(x: NonExhaustive) -> impl FnOnce() {
+    || {
+    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
+        match x {
+            NonExhaustive::A(a, b) => {
+                drop((a, b));
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+// (again, wildcard branch or not)
+pub fn test_non_exhaustive2(x: NonExhaustive) -> impl FnOnce() {
+    || {
+    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
+        match x {
+            NonExhaustive::A(a, b) => {
+                drop((a, b));
+            }
+        }
+    }
+}
+
+// ...regardless of whether the enum is defined in the current, or in another crate
+pub fn test_ext(x: ExtNonExhaustive) -> impl FnOnce() {
+    || {
+        match x {
+            ExtNonExhaustive::A(a, b) => {
+                drop((a, b));
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+fn main() {}
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr b/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr
new file mode 100644
index 000000000000..36adcc6717ed
--- /dev/null
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr
@@ -0,0 +1,123 @@
+error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
+  --> $DIR/partial-move.rs:35:5
+   |
+LL |     || {
+   |     ^^ may outlive borrowed value `x.0`
+LL |
+LL |         match x {
+   |               - `x.0` is borrowed here
+   |
+note: closure is returned here
+  --> $DIR/partial-move.rs:35:5
+   |
+LL | /     || {
+LL | |
+LL | |         match x {
+LL | |             Enum::A(a, b) => {
+...  |
+LL | |     }
+   | |_____^
+help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
+   |
+LL |     move || {
+   |     ++++
+
+error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
+  --> $DIR/partial-move.rs:47:5
+   |
+LL |     || {
+   |     ^^ may outlive borrowed value `x.0`
+LL |
+LL |         match x {
+   |               - `x.0` is borrowed here
+   |
+note: closure is returned here
+  --> $DIR/partial-move.rs:47:5
+   |
+LL | /     || {
+LL | |
+LL | |         match x {
+LL | |             Enum::A(a, b) => {
+...  |
+LL | |     }
+   | |_____^
+help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
+   |
+LL |     move || {
+   |     ++++
+
+error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
+  --> $DIR/partial-move.rs:59:5
+   |
+LL |     || {
+   |     ^^ may outlive borrowed value `x.0`
+LL |
+LL |         match x {
+   |               - `x.0` is borrowed here
+   |
+note: closure is returned here
+  --> $DIR/partial-move.rs:59:5
+   |
+LL | /     || {
+LL | |
+LL | |         match x {
+LL | |             Struct(a, b) => {
+...  |
+LL | |     }
+   | |_____^
+help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
+   |
+LL |     move || {
+   |     ++++
+
+error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
+  --> $DIR/partial-move.rs:83:5
+   |
+LL |     || {
+   |     ^^ may outlive borrowed value `x.0`
+LL |
+LL |         match x {
+   |               - `x.0` is borrowed here
+   |
+note: closure is returned here
+  --> $DIR/partial-move.rs:83:5
+   |
+LL | /     || {
+LL | |
+LL | |         match x {
+LL | |             NonExhaustive::A(a, b) => {
+...  |
+LL | |     }
+   | |_____^
+help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
+   |
+LL |     move || {
+   |     ++++
+
+error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
+  --> $DIR/partial-move.rs:96:5
+   |
+LL |     || {
+   |     ^^ may outlive borrowed value `x.0`
+LL |
+LL |         match x {
+   |               - `x.0` is borrowed here
+   |
+note: closure is returned here
+  --> $DIR/partial-move.rs:96:5
+   |
+LL | /     || {
+LL | |
+LL | |         match x {
+LL | |             NonExhaustive::A(a, b) => {
+...  |
+LL | |     }
+   | |_____^
+help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
+   |
+LL |     move || {
+   |     ++++
+
+error: aborting due to 5 previous errors
+
+For more information about this error, try `rustc --explain E0373`.

From 3c175080dca2a60eb42a28d46ecc416d3a6f8879 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maja=20K=C4=85dzio=C5=82ka?= 
Date: Sun, 4 Jan 2026 19:34:37 +0100
Subject: [PATCH 0267/1843] add tests for discriminant read borrowck

---
 tests/ui/match/borrowck-uninhabited.rs        | 53 +++++++++++++++++++
 tests/ui/match/borrowck-uninhabited.stderr    | 14 +++++
 tests/ui/match/uninhabited-granular-moves.rs  | 46 ++++++++++++++++
 .../match/uninhabited-granular-moves.stderr   | 18 +++++++
 4 files changed, 131 insertions(+)
 create mode 100644 tests/ui/match/borrowck-uninhabited.rs
 create mode 100644 tests/ui/match/borrowck-uninhabited.stderr
 create mode 100644 tests/ui/match/uninhabited-granular-moves.rs
 create mode 100644 tests/ui/match/uninhabited-granular-moves.stderr

diff --git a/tests/ui/match/borrowck-uninhabited.rs b/tests/ui/match/borrowck-uninhabited.rs
new file mode 100644
index 000000000000..93838ede2793
--- /dev/null
+++ b/tests/ui/match/borrowck-uninhabited.rs
@@ -0,0 +1,53 @@
+// See: rust-lang/rust#146590
+
+enum Never {}
+
+// baseline
+fn both_inhabited(x: &mut Result) {
+    match x {
+        &mut Ok(ref mut y) => match x {
+        //~^ ERROR: cannot use `*x` because it was mutably borrowed
+            &mut Err(ref mut z) => {
+                let _y = y;
+                let _z = z;
+            }
+            _ => {}
+        },
+        _ => {}
+    };
+}
+
+fn ref_uninhabited(x: &mut Result) {
+    match x {
+        &mut Ok(ref mut y) => match x {
+            &mut Err(ref mut z) => {
+                let _y = y;
+                let _z = z;
+            }
+            _ => {}
+        },
+        _ => {}
+    };
+}
+
+enum Single {
+    V(String, String),
+}
+
+// arguably this should be rejected as well, but currently it is still accepted
+fn single_variant(x: &mut Single) {
+    match x {
+        &mut Single::V(ref mut y, _) => {
+            match x {
+                &mut Single::V(_, ref mut z) => {
+                    let _y = y;
+                    let _z = z;
+                }
+                _ => {}
+            }
+        },
+        _ => {}
+    };
+}
+
+fn main() {}
diff --git a/tests/ui/match/borrowck-uninhabited.stderr b/tests/ui/match/borrowck-uninhabited.stderr
new file mode 100644
index 000000000000..ee3b814da9f4
--- /dev/null
+++ b/tests/ui/match/borrowck-uninhabited.stderr
@@ -0,0 +1,14 @@
+error[E0503]: cannot use `*x` because it was mutably borrowed
+  --> $DIR/borrowck-uninhabited.rs:8:37
+   |
+LL |         &mut Ok(ref mut y) => match x {
+   |                 ---------           ^ use of borrowed `x.0`
+   |                 |
+   |                 `x.0` is borrowed here
+...
+LL |                 let _y = y;
+   |                          - borrow later used here
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0503`.
diff --git a/tests/ui/match/uninhabited-granular-moves.rs b/tests/ui/match/uninhabited-granular-moves.rs
new file mode 100644
index 000000000000..9d04fcb3259e
--- /dev/null
+++ b/tests/ui/match/uninhabited-granular-moves.rs
@@ -0,0 +1,46 @@
+// See rust-lang/rust#146590, as well as Zulip discussion:
+//
+// https://rust-lang.zulipchat.com/#narrow/channel/513289-t-patterns/topic/Question.20about.20patterns.20and.20moves/with/558638455
+//
+// Whether pattern matching performs a discriminant read shouldn't depend on whether
+// you explicitly write down an uninhabited branch, or leave it implicit.
+
+enum Emp { }
+
+enum Foo {
+    Bar(A),
+    Qux(Emp),
+}
+
+fn test1(thefoo: Foo<(Box, Box)>) {
+    match thefoo {
+        Foo::Bar((a, _)) => { }
+    }
+
+    match thefoo {
+        Foo::Bar((_, a)) => { }
+    }
+}
+
+fn test2(thefoo: Foo<(Box, Box)>) {
+    match thefoo {
+        Foo::Bar((a, _)) => { }
+        Foo::Qux(_) => { }
+    }
+    match thefoo { //~ ERROR: use of partially moved value: `thefoo`
+        Foo::Bar((_, a)) => { }
+        Foo::Qux(_) => { }
+    }
+}
+
+fn test3(thefoo: Foo<(Box, Box)>) {
+    match thefoo {
+        Foo::Bar((a, _)) => { }
+        Foo::Qux(_) => { }
+    }
+    match thefoo {
+        Foo::Bar((_, a)) => { }
+    }
+}
+
+fn main() {}
diff --git a/tests/ui/match/uninhabited-granular-moves.stderr b/tests/ui/match/uninhabited-granular-moves.stderr
new file mode 100644
index 000000000000..d32f935d3d55
--- /dev/null
+++ b/tests/ui/match/uninhabited-granular-moves.stderr
@@ -0,0 +1,18 @@
+error[E0382]: use of partially moved value: `thefoo`
+  --> $DIR/uninhabited-granular-moves.rs:30:11
+   |
+LL |         Foo::Bar((a, _)) => { }
+   |                   - value partially moved here
+...
+LL |     match thefoo {
+   |           ^^^^^^ value used here after partial move
+   |
+   = note: partial move occurs because value has type `Box`, which does not implement the `Copy` trait
+help: borrow this binding in the pattern to avoid moving the value
+   |
+LL |         Foo::Bar((ref a, _)) => { }
+   |                   +++
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0382`.

From ee1a6f4e88bb71d964864de1d7c935d53a3d0cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maja=20K=C4=85dzio=C5=82ka?= 
Date: Sun, 4 Jan 2026 04:22:55 +0100
Subject: [PATCH 0268/1843] match in closure: capture non_exhaustive even if
 defined in current crate

---
 .../rustc_hir_typeck/src/expr_use_visitor.rs  | 29 +++--------
 .../match/non-exhaustive-match.rs             | 16 +++---
 .../match/non-exhaustive-match.stderr         | 19 +++++--
 .../match/partial-move-drop-order.rs          |  5 ++
 .../match/partial-move-drop-order.run.stdout  |  2 +-
 .../match/partial-move.rs                     |  4 +-
 .../match/partial-move.stderr                 | 50 +------------------
 7 files changed, 39 insertions(+), 86 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index ad34994526ed..9e9b511b9b23 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -818,14 +818,12 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
     /// The core driver for walking a pattern
     ///
     /// This should mirror how pattern-matching gets lowered to MIR, as
-    /// otherwise lowering will ICE when trying to resolve the upvars.
+    /// otherwise said lowering will ICE when trying to resolve the upvars.
     ///
     /// However, it is okay to approximate it here by doing *more* accesses than
     /// the actual MIR builder will, which is useful when some checks are too
-    /// cumbersome to perform here. For example, if after typeck it becomes
-    /// clear that only one variant of an enum is inhabited, and therefore a
-    /// read of the discriminant is not necessary, `walk_pat` will have
-    /// over-approximated the necessary upvar capture granularity.
+    /// cumbersome to perform here, because e.g. they require more typeck results
+    /// than available.
     ///
     /// Do note that discrepancies like these do still create obscure corners
     /// in the semantics of the language, and should be avoided if possible.
@@ -1852,26 +1850,13 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
     }
 
     /// Checks whether a type has multiple variants, and therefore, whether a
-    /// read of the discriminant might be necessary. Note that the actual MIR
-    /// builder code does a more specific check, filtering out variants that
-    /// happen to be uninhabited.
-    ///
-    /// Here, it is not practical to perform such a check, because inhabitedness
-    /// queries require typeck results, and typeck requires closure capture analysis.
-    ///
-    /// Moreover, the language is moving towards uninhabited variants still semantically
-    /// causing a discriminant read, so we *shouldn't* perform any such check.
-    ///
-    /// FIXME(never_patterns): update this comment once the aforementioned MIR builder
-    /// code is changed to be insensitive to inhhabitedness.
+    /// read of the discriminant might be necessary.
     #[instrument(skip(self, span), level = "debug")]
     fn is_multivariant_adt(&self, ty: Ty<'tcx>, span: Span) -> bool {
         if let ty::Adt(def, _) = self.cx.structurally_resolve_type(span, ty).kind() {
-            // Note that if a non-exhaustive SingleVariant is defined in another crate, we need
-            // to assume that more cases will be added to the variant in the future. This mean
-            // that we should handle non-exhaustive SingleVariant the same way we would handle
-            // a MultiVariant.
-            def.variants().len() > 1 || def.variant_list_has_applicable_non_exhaustive()
+            // We treat non-exhaustive enums the same independent of the crate they are
+            // defined in, to avoid differences in the operational semantics between crates.
+            def.variants().len() > 1 || def.is_variant_list_non_exhaustive()
         } else {
             false
         }
diff --git a/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.rs b/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.rs
index 5b7259c6c2cc..f47d70b52f20 100644
--- a/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.rs
+++ b/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.rs
@@ -28,12 +28,6 @@ fn main() {
     let _b = || { match l1 { L1::A => () } };
     //~^ ERROR: non-exhaustive patterns: `L1::B` not covered [E0004]
 
-    // l2 should not be captured as it is a non-exhaustive SingleVariant
-    // defined in this crate
-    let _c = || { match l2 { L2::C => (), _ => () }  };
-    let mut mut_l2 = l2;
-    _c();
-
     // E1 is not visibly uninhabited from here
     let (e1, e2, e3, e4) = bar();
     let _d = || { match e1 {} };
@@ -42,8 +36,14 @@ fn main() {
     //~^ ERROR: non-exhaustive patterns: `_` not covered [E0004]
     let _f = || { match e2 { E2::A => (), E2::B => (), _ => () }  };
 
-    // e3 should be captured as it is a non-exhaustive SingleVariant
-    // defined in another crate
+    // non-exhaustive enums should always be captured, regardless if they
+    // are defined in the current crate:
+    let _c = || { match l2 { L2::C => (), _ => () }  };
+    let mut mut_l2 = l2;
+    //~^ ERROR: cannot move out of `l2` because it is borrowed
+    _c();
+
+    // ...or in another crate:
     let _g = || { match e3 { E3::C => (), _ => () }  };
     let mut mut_e3 = e3;
     //~^ ERROR: cannot move out of `e3` because it is borrowed
diff --git a/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.stderr b/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.stderr
index 99d33b05429e..e34d1889803a 100644
--- a/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.stderr
+++ b/tests/ui/closures/2229_closure_analysis/match/non-exhaustive-match.stderr
@@ -16,7 +16,7 @@ LL |     let _b = || { match l1 { L1::A => (), L1::B => todo!() } };
    |                                         ++++++++++++++++++
 
 error[E0004]: non-exhaustive patterns: type `E1` is non-empty
-  --> $DIR/non-exhaustive-match.rs:39:25
+  --> $DIR/non-exhaustive-match.rs:33:25
    |
 LL |     let _d = || { match e1 {} };
    |                         ^^
@@ -35,7 +35,7 @@ LL ~     } };
    |
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/non-exhaustive-match.rs:41:25
+  --> $DIR/non-exhaustive-match.rs:35:25
    |
 LL |     let _e = || { match e2 { E2::A => (), E2::B => () } };
    |                         ^^ pattern `_` not covered
@@ -52,6 +52,19 @@ help: ensure that all possible cases are being handled by adding a match arm wit
 LL |     let _e = || { match e2 { E2::A => (), E2::B => (), _ => todo!() } };
    |                                                      ++++++++++++++
 
+error[E0505]: cannot move out of `l2` because it is borrowed
+  --> $DIR/non-exhaustive-match.rs:42:22
+   |
+LL |     let _c = || { match l2 { L2::C => (), _ => () }  };
+   |              --         -- borrow occurs due to use in closure
+   |              |
+   |              borrow of `l2` occurs here
+LL |     let mut mut_l2 = l2;
+   |                      ^^ move out of `l2` occurs here
+LL |
+LL |     _c();
+   |     -- borrow later used here
+
 error[E0505]: cannot move out of `e3` because it is borrowed
   --> $DIR/non-exhaustive-match.rs:48:22
    |
@@ -65,7 +78,7 @@ LL |
 LL |     _g();
    |     -- borrow later used here
 
-error: aborting due to 4 previous errors
+error: aborting due to 5 previous errors
 
 Some errors have detailed explanations: E0004, E0505.
 For more information about an error, try `rustc --explain E0004`.
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs
index 1c178bd3f684..348d558dd980 100644
--- a/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.rs
@@ -1,3 +1,8 @@
+// Make sure that #[non_exhaustive] cannot cause drop order to depend on which
+// crate the code is in.
+//
+// See rust-lang/rust#147722
+//
 //@ edition:2021
 //@ run-pass
 //@ check-run-results
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout
index e0d83dd99549..3fa346d87b07 100644
--- a/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move-drop-order.run.stdout
@@ -12,8 +12,8 @@ dropping b
 
 non exhaustive:
 before assign
-dropping a
 after assign
+dropping a
 dropping b
 
 external non exhaustive:
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move.rs b/tests/ui/closures/2229_closure_analysis/match/partial-move.rs
index 62d1b362cb83..7bd435bd1856 100644
--- a/tests/ui/closures/2229_closure_analysis/match/partial-move.rs
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move.rs
@@ -78,10 +78,9 @@ pub fn test_two_variants(x: TwoVariants) -> impl FnOnce() {
     }
 }
 
-// ...and single-variant, non-exhaustive enums *should* behave as if they had multiple variants
+// ...and single-variant, non-exhaustive enums behave as if they had multiple variants
 pub fn test_non_exhaustive1(x: NonExhaustive) -> impl FnOnce() {
     || {
-    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
         match x {
             NonExhaustive::A(a, b) => {
                 drop((a, b));
@@ -94,7 +93,6 @@ pub fn test_non_exhaustive1(x: NonExhaustive) -> impl FnOnce() {
 // (again, wildcard branch or not)
 pub fn test_non_exhaustive2(x: NonExhaustive) -> impl FnOnce() {
     || {
-    //~^ ERROR: closure may outlive the current function, but it borrows `x.0`
         match x {
             NonExhaustive::A(a, b) => {
                 drop((a, b));
diff --git a/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr b/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr
index 36adcc6717ed..09f9adf95d5b 100644
--- a/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr
+++ b/tests/ui/closures/2229_closure_analysis/match/partial-move.stderr
@@ -70,54 +70,6 @@ help: to force the closure to take ownership of `x.0` (and any other referenced
 LL |     move || {
    |     ++++
 
-error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
-  --> $DIR/partial-move.rs:83:5
-   |
-LL |     || {
-   |     ^^ may outlive borrowed value `x.0`
-LL |
-LL |         match x {
-   |               - `x.0` is borrowed here
-   |
-note: closure is returned here
-  --> $DIR/partial-move.rs:83:5
-   |
-LL | /     || {
-LL | |
-LL | |         match x {
-LL | |             NonExhaustive::A(a, b) => {
-...  |
-LL | |     }
-   | |_____^
-help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
-   |
-LL |     move || {
-   |     ++++
-
-error[E0373]: closure may outlive the current function, but it borrows `x.0`, which is owned by the current function
-  --> $DIR/partial-move.rs:96:5
-   |
-LL |     || {
-   |     ^^ may outlive borrowed value `x.0`
-LL |
-LL |         match x {
-   |               - `x.0` is borrowed here
-   |
-note: closure is returned here
-  --> $DIR/partial-move.rs:96:5
-   |
-LL | /     || {
-LL | |
-LL | |         match x {
-LL | |             NonExhaustive::A(a, b) => {
-...  |
-LL | |     }
-   | |_____^
-help: to force the closure to take ownership of `x.0` (and any other referenced variables), use the `move` keyword
-   |
-LL |     move || {
-   |     ++++
-
-error: aborting due to 5 previous errors
+error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0373`.

From 3387a58bc777d31493f37160cdd1f57223f0bc0c Mon Sep 17 00:00:00 2001
From: James Hendry 
Date: Wed, 14 Jan 2026 10:48:24 +0000
Subject: [PATCH 0269/1843] fN::BITS constants for feature float_bits_const

Also enables the feature for compiler_builtins as otherwise
this causes a warning and conflicts with the Float extension trait

Explicitly use Float trait BITS in miri tests to
prevent warnings against upcoming BITS field for floats
---
 library/compiler-builtins/compiler-builtins/src/lib.rs | 1 +
 library/core/src/num/f128.rs                           | 5 +++++
 library/core/src/num/f16.rs                            | 5 +++++
 library/core/src/num/f32.rs                            | 4 ++++
 library/core/src/num/f64.rs                            | 4 ++++
 src/tools/miri/tests/pass/float.rs                     | 6 +++---
 src/tools/test-float-parse/src/traits.rs               | 2 +-
 7 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/library/compiler-builtins/compiler-builtins/src/lib.rs b/library/compiler-builtins/compiler-builtins/src/lib.rs
index c993209699be..342427aca9c7 100644
--- a/library/compiler-builtins/compiler-builtins/src/lib.rs
+++ b/library/compiler-builtins/compiler-builtins/src/lib.rs
@@ -11,6 +11,7 @@
 #![feature(repr_simd)]
 #![feature(macro_metavar_expr_concat)]
 #![feature(rustc_attrs)]
+#![feature(float_bits_const)]
 #![cfg_attr(f16_enabled, feature(f16))]
 #![cfg_attr(f128_enabled, feature(f128))]
 #![no_builtins]
diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs
index bf99fee4fc78..8bb919de729e 100644
--- a/library/core/src/num/f128.rs
+++ b/library/core/src/num/f128.rs
@@ -145,6 +145,11 @@ impl f128 {
     #[unstable(feature = "f128", issue = "116909")]
     pub const RADIX: u32 = 2;
 
+    /// The size of this float type in bits.
+    // #[unstable(feature = "f128", issue = "116909")]
+    #[unstable(feature = "float_bits_const", issue = "151073")]
+    pub const BITS: u32 = 128;
+
     /// Number of significant digits in base 2.
     ///
     /// Note that the size of the mantissa in the bitwise representation is one
diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs
index f39ee22871d5..699b2e3c25c4 100644
--- a/library/core/src/num/f16.rs
+++ b/library/core/src/num/f16.rs
@@ -142,6 +142,11 @@ impl f16 {
     #[unstable(feature = "f16", issue = "116909")]
     pub const RADIX: u32 = 2;
 
+    /// The size of this float type in bits.
+    // #[unstable(feature = "f16", issue = "116909")]
+    #[unstable(feature = "float_bits_const", issue = "151073")]
+    pub const BITS: u32 = 16;
+
     /// Number of significant digits in base 2.
     ///
     /// Note that the size of the mantissa in the bitwise representation is one
diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs
index 6fe4285374b2..118ef4ff3d43 100644
--- a/library/core/src/num/f32.rs
+++ b/library/core/src/num/f32.rs
@@ -390,6 +390,10 @@ impl f32 {
     #[stable(feature = "assoc_int_consts", since = "1.43.0")]
     pub const RADIX: u32 = 2;
 
+    /// The size of this float type in bits.
+    #[unstable(feature = "float_bits_const", issue = "151073")]
+    pub const BITS: u32 = 32;
+
     /// Number of significant digits in base 2.
     ///
     /// Note that the size of the mantissa in the bitwise representation is one
diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs
index d0aca152415e..eacbd3ff20bf 100644
--- a/library/core/src/num/f64.rs
+++ b/library/core/src/num/f64.rs
@@ -390,6 +390,10 @@ impl f64 {
     #[stable(feature = "assoc_int_consts", since = "1.43.0")]
     pub const RADIX: u32 = 2;
 
+    /// The size of this float type in bits.
+    #[unstable(feature = "float_bits_const", issue = "151073")]
+    pub const BITS: u32 = 64;
+
     /// Number of significant digits in base 2.
     ///
     /// Note that the size of the mantissa in the bitwise representation is one
diff --git a/src/tools/miri/tests/pass/float.rs b/src/tools/miri/tests/pass/float.rs
index a74a66d5455a..052c83d1aa31 100644
--- a/src/tools/miri/tests/pass/float.rs
+++ b/src/tools/miri/tests/pass/float.rs
@@ -353,7 +353,7 @@ macro_rules! test_ftoi_itof {
             assert_itof(i, f, msg);
         }
 
-        let fbits = <$fty>::BITS;
+        let fbits = <$fty as Float>::BITS;
         let fsig_bits = <$fty>::SIGNIFICAND_BITS;
         let ibits = <$ity>::BITS;
         let imax: $ity = <$ity>::MAX;
@@ -528,9 +528,9 @@ macro_rules! test_ftof {
         assert!((<$f1>::NAN as $f2).is_nan(), "{} -> {} nan", stringify!($f1), stringify!($f2));
 
         let min_sub_casted = <$f1>::from_bits(0x1) as $f2;
-        let min_neg_sub_casted = <$f1>::from_bits(0x1 | 1 << (<$f1>::BITS - 1)) as $f2;
+        let min_neg_sub_casted = <$f1>::from_bits(0x1 | 1 << (<$f1 as Float>::BITS - 1)) as $f2;
 
-        if <$f1>::BITS > <$f2>::BITS {
+        if <$f1 as Float>::BITS > <$f2 as Float>::BITS {
             assert_feq(<$f1>::MAX as $f2, <$f2>::INFINITY, "max -> inf");
             assert_feq(<$f1>::MIN as $f2, <$f2>::NEG_INFINITY, "max -> inf");
             assert_biteq(min_sub_casted, f2zero, "min subnormal -> 0.0");
diff --git a/src/tools/test-float-parse/src/traits.rs b/src/tools/test-float-parse/src/traits.rs
index 65a8721bfa5c..0cb44d297bfb 100644
--- a/src/tools/test-float-parse/src/traits.rs
+++ b/src/tools/test-float-parse/src/traits.rs
@@ -155,7 +155,7 @@ macro_rules! impl_float {
                 const BITS: u32 = <$ity>::BITS;
                 const MAN_BITS: u32 = Self::MANTISSA_DIGITS - 1;
                 const MAN_MASK: Self::Int = (Self::Int::ONE << Self::MAN_BITS) - Self::Int::ONE;
-                const SIGN_MASK: Self::Int = Self::Int::ONE << (Self::BITS-1);
+                const SIGN_MASK: Self::Int = Self::Int::ONE << (::BITS-1);
                 fn from_bits(i: Self::Int) -> Self { Self::from_bits(i) }
                 fn to_bits(self) -> Self::Int { self.to_bits() }
                 fn constants() -> &'static Constants {

From af302a67fdc508cfd08ee22facb96bcf0e5bf831 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maja=20K=C4=85dzio=C5=82ka?= 
Date: Sun, 4 Jan 2026 19:34:37 +0100
Subject: [PATCH 0270/1843] discriminant reads: make semantics independent of
 module/crate

---
 .../src/builder/matches/match_pair.rs         | 20 +++-----
 .../fail/match/all_variants_uninhabited.rs    | 25 ++++++++++
 .../match/all_variants_uninhabited.stderr     | 13 +++++
 .../{ => match}/closures/deref-in-pattern.rs  |  0
 .../closures/deref-in-pattern.stderr          |  2 +-
 .../{ => match}/closures/partial-pattern.rs   |  0
 .../closures/partial-pattern.stderr           |  2 +-
 .../closures/uninhabited-variant1.rs}         |  0
 .../closures/uninhabited-variant1.stderr}     |  6 +--
 .../match/closures/uninhabited-variant2.rs    | 32 ++++++++++++
 .../closures/uninhabited-variant2.stderr      | 18 +++++++
 .../fail/match/only_inhabited_variant.rs      | 21 ++++++++
 .../fail/match/only_inhabited_variant.stderr  | 13 +++++
 .../miri/tests/fail/match/single_variant.rs   | 31 ++++++++++++
 .../tests/fail/match/single_variant.stderr    | 13 +++++
 .../tests/fail/match/single_variant_uninit.rs | 36 +++++++++++++
 .../fail/match/single_variant_uninit.stderr   | 18 +++++++
 .../enum/enum-transparent-extract.rs          |  4 ++
 ...atterns.opt1.SimplifyCfg-initial.after.mir | 10 ++--
 ...atterns.opt2.SimplifyCfg-initial.after.mir | 15 ++++++
 ...atterns.opt3.SimplifyCfg-initial.after.mir | 17 ++++---
 ...ng.identity.JumpThreading.panic-abort.diff | 19 +++++--
 ...g.identity.JumpThreading.panic-unwind.diff | 19 +++++--
 ...map_via_question_mark.PreCodegen.after.mir | 50 +++++++++++--------
 ...e_const_switch.identity.JumpThreading.diff | 15 ++++--
 ...ch.UnreachablePropagation.panic-abort.diff | 12 +++--
 ...h.UnreachablePropagation.panic-unwind.diff | 12 +++--
 tests/mir-opt/unreachable.rs                  |  2 +-
 ....UnreachableEnumBranching.panic-abort.diff | 28 +++++------
 ...UnreachableEnumBranching.panic-unwind.diff | 28 +++++------
 tests/ui/match/borrowck-uninhabited.rs        |  2 +
 tests/ui/match/borrowck-uninhabited.stderr    | 13 ++++-
 tests/ui/match/uninhabited-granular-moves.rs  |  4 +-
 .../match/uninhabited-granular-moves.stderr   | 32 +++++++++++-
 .../borrowck-exhaustive.rs                    | 12 -----
 .../borrowck-non-exhaustive.rs                | 12 +++++
 .../borrowck-non-exhaustive.stderr            | 15 +++++-
 37 files changed, 455 insertions(+), 116 deletions(-)
 create mode 100644 src/tools/miri/tests/fail/match/all_variants_uninhabited.rs
 create mode 100644 src/tools/miri/tests/fail/match/all_variants_uninhabited.stderr
 rename src/tools/miri/tests/fail/{ => match}/closures/deref-in-pattern.rs (100%)
 rename src/tools/miri/tests/fail/{ => match}/closures/deref-in-pattern.stderr (91%)
 rename src/tools/miri/tests/fail/{ => match}/closures/partial-pattern.rs (100%)
 rename src/tools/miri/tests/fail/{ => match}/closures/partial-pattern.stderr (92%)
 rename src/tools/miri/tests/fail/{closures/uninhabited-variant.rs => match/closures/uninhabited-variant1.rs} (100%)
 rename src/tools/miri/tests/fail/{closures/uninhabited-variant.stderr => match/closures/uninhabited-variant1.stderr} (74%)
 create mode 100644 src/tools/miri/tests/fail/match/closures/uninhabited-variant2.rs
 create mode 100644 src/tools/miri/tests/fail/match/closures/uninhabited-variant2.stderr
 create mode 100644 src/tools/miri/tests/fail/match/only_inhabited_variant.rs
 create mode 100644 src/tools/miri/tests/fail/match/only_inhabited_variant.stderr
 create mode 100644 src/tools/miri/tests/fail/match/single_variant.rs
 create mode 100644 src/tools/miri/tests/fail/match/single_variant.stderr
 create mode 100644 src/tools/miri/tests/fail/match/single_variant_uninit.rs
 create mode 100644 src/tools/miri/tests/fail/match/single_variant_uninit.stderr

diff --git a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
index 8cee3ff27e8f..07af66127ef9 100644
--- a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
@@ -295,22 +295,18 @@ impl<'tcx> MatchPairTree<'tcx> {
                 }
             }
 
-            PatKind::Variant { adt_def, variant_index, args, ref subpatterns } => {
+            PatKind::Variant { adt_def, variant_index, args: _, ref subpatterns } => {
                 let downcast_place = place_builder.downcast(adt_def, variant_index); // `(x as Variant)`
                 cx.field_match_pairs(&mut subpairs, extra_data, downcast_place, subpatterns);
 
-                let irrefutable = adt_def.variants().iter_enumerated().all(|(i, v)| {
-                    i == variant_index
-                        || !v.inhabited_predicate(cx.tcx, adt_def).instantiate(cx.tcx, args).apply(
-                            cx.tcx,
-                            cx.infcx.typing_env(cx.param_env),
-                            cx.def_id.into(),
-                        )
-                }) && !adt_def.variant_list_has_applicable_non_exhaustive();
-                if irrefutable {
-                    None
-                } else {
+                // We treat non-exhaustive enums the same independent of the crate they are
+                // defined in, to avoid differences in the operational semantics between crates.
+                let refutable =
+                    adt_def.variants().len() > 1 || adt_def.is_variant_list_non_exhaustive();
+                if refutable {
                     Some(TestableCase::Variant { adt_def, variant_index })
+                } else {
+                    None
                 }
             }
 
diff --git a/src/tools/miri/tests/fail/match/all_variants_uninhabited.rs b/src/tools/miri/tests/fail/match/all_variants_uninhabited.rs
new file mode 100644
index 000000000000..e7ca45579d84
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/all_variants_uninhabited.rs
@@ -0,0 +1,25 @@
+#![allow(deref_nullptr)]
+
+enum Never {}
+
+fn main() {
+    unsafe {
+        match *std::ptr::null::>() {
+        //~^ ERROR: read discriminant of an uninhabited enum variant
+            Ok(_) => {
+                lol();
+            }
+            Err(_) => {
+                wut();
+            }
+        }
+    }
+}
+
+fn lol() {
+    println!("lol");
+}
+
+fn wut() {
+    println!("wut");
+}
diff --git a/src/tools/miri/tests/fail/match/all_variants_uninhabited.stderr b/src/tools/miri/tests/fail/match/all_variants_uninhabited.stderr
new file mode 100644
index 000000000000..90809219340b
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/all_variants_uninhabited.stderr
@@ -0,0 +1,13 @@
+error: Undefined Behavior: read discriminant of an uninhabited enum variant
+  --> tests/fail/match/all_variants_uninhabited.rs:LL:CC
+   |
+LL |         match *std::ptr::null::>() {
+   |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/fail/closures/deref-in-pattern.rs b/src/tools/miri/tests/fail/match/closures/deref-in-pattern.rs
similarity index 100%
rename from src/tools/miri/tests/fail/closures/deref-in-pattern.rs
rename to src/tools/miri/tests/fail/match/closures/deref-in-pattern.rs
diff --git a/src/tools/miri/tests/fail/closures/deref-in-pattern.stderr b/src/tools/miri/tests/fail/match/closures/deref-in-pattern.stderr
similarity index 91%
rename from src/tools/miri/tests/fail/closures/deref-in-pattern.stderr
rename to src/tools/miri/tests/fail/match/closures/deref-in-pattern.stderr
index fae2d286c487..cbb74df5de77 100644
--- a/src/tools/miri/tests/fail/closures/deref-in-pattern.stderr
+++ b/src/tools/miri/tests/fail/match/closures/deref-in-pattern.stderr
@@ -1,5 +1,5 @@
 error: Undefined Behavior: constructing invalid value: encountered a dangling reference (use-after-free)
-  --> tests/fail/closures/deref-in-pattern.rs:LL:CC
+  --> tests/fail/match/closures/deref-in-pattern.rs:LL:CC
    |
 LL |       let _ = || {
    |  _____________^
diff --git a/src/tools/miri/tests/fail/closures/partial-pattern.rs b/src/tools/miri/tests/fail/match/closures/partial-pattern.rs
similarity index 100%
rename from src/tools/miri/tests/fail/closures/partial-pattern.rs
rename to src/tools/miri/tests/fail/match/closures/partial-pattern.rs
diff --git a/src/tools/miri/tests/fail/closures/partial-pattern.stderr b/src/tools/miri/tests/fail/match/closures/partial-pattern.stderr
similarity index 92%
rename from src/tools/miri/tests/fail/closures/partial-pattern.stderr
rename to src/tools/miri/tests/fail/match/closures/partial-pattern.stderr
index 8dea4d4d8c64..b8ca04559e23 100644
--- a/src/tools/miri/tests/fail/closures/partial-pattern.stderr
+++ b/src/tools/miri/tests/fail/match/closures/partial-pattern.stderr
@@ -1,5 +1,5 @@
 error: Undefined Behavior: constructing invalid value: encountered a dangling reference (use-after-free)
-  --> tests/fail/closures/partial-pattern.rs:LL:CC
+  --> tests/fail/match/closures/partial-pattern.rs:LL:CC
    |
 LL |       let _ = || {
    |  _____________^
diff --git a/src/tools/miri/tests/fail/closures/uninhabited-variant.rs b/src/tools/miri/tests/fail/match/closures/uninhabited-variant1.rs
similarity index 100%
rename from src/tools/miri/tests/fail/closures/uninhabited-variant.rs
rename to src/tools/miri/tests/fail/match/closures/uninhabited-variant1.rs
diff --git a/src/tools/miri/tests/fail/closures/uninhabited-variant.stderr b/src/tools/miri/tests/fail/match/closures/uninhabited-variant1.stderr
similarity index 74%
rename from src/tools/miri/tests/fail/closures/uninhabited-variant.stderr
rename to src/tools/miri/tests/fail/match/closures/uninhabited-variant1.stderr
index aea878660115..1a10a5b24d25 100644
--- a/src/tools/miri/tests/fail/closures/uninhabited-variant.stderr
+++ b/src/tools/miri/tests/fail/match/closures/uninhabited-variant1.stderr
@@ -1,5 +1,5 @@
 error: Undefined Behavior: read discriminant of an uninhabited enum variant
-  --> tests/fail/closures/uninhabited-variant.rs:LL:CC
+  --> tests/fail/match/closures/uninhabited-variant1.rs:LL:CC
    |
 LL |         match r {
    |               ^ Undefined Behavior occurred here
@@ -8,9 +8,9 @@ LL |         match r {
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
    = note: stack backtrace:
            0: main::{closure#0}
-               at tests/fail/closures/uninhabited-variant.rs:LL:CC
+               at tests/fail/match/closures/uninhabited-variant1.rs:LL:CC
            1: main
-               at tests/fail/closures/uninhabited-variant.rs:LL:CC
+               at tests/fail/match/closures/uninhabited-variant1.rs:LL:CC
 
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
 
diff --git a/src/tools/miri/tests/fail/match/closures/uninhabited-variant2.rs b/src/tools/miri/tests/fail/match/closures/uninhabited-variant2.rs
new file mode 100644
index 000000000000..ed68e357fbd5
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/closures/uninhabited-variant2.rs
@@ -0,0 +1,32 @@
+// Motivated by rust-lang/rust#138961, this shows how invalid discriminants interact with
+// closure captures.
+//
+// Test case with only one inhabited variant, for which rustc used to not emit
+// a discriminant read in the first place. See: rust-lang/miri#4778
+#![feature(never_type)]
+
+#[repr(C)]
+#[allow(dead_code)]
+enum E {
+    V0,    // discriminant: 0
+    V1(!), // 1
+}
+
+fn main() {
+    assert_eq!(std::mem::size_of::(), 4);
+
+    let val = 1u32;
+    let ptr = (&raw const val).cast::();
+    let r = unsafe { &*ptr };
+    let f = || {
+        // After rust-lang/rust#138961, constructing the closure performs a reborrow of r.
+        // Nevertheless, the discriminant is only actually inspected when the closure
+        // is called.
+        match r { //~ ERROR: read discriminant of an uninhabited enum variant
+            E::V0 => {}
+            E::V1(_) => {}
+        }
+    };
+
+    f();
+}
diff --git a/src/tools/miri/tests/fail/match/closures/uninhabited-variant2.stderr b/src/tools/miri/tests/fail/match/closures/uninhabited-variant2.stderr
new file mode 100644
index 000000000000..611e0435ef0e
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/closures/uninhabited-variant2.stderr
@@ -0,0 +1,18 @@
+error: Undefined Behavior: read discriminant of an uninhabited enum variant
+  --> tests/fail/match/closures/uninhabited-variant2.rs:LL:CC
+   |
+LL |         match r {
+   |               ^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: stack backtrace:
+           0: main::{closure#0}
+               at tests/fail/match/closures/uninhabited-variant2.rs:LL:CC
+           1: main
+               at tests/fail/match/closures/uninhabited-variant2.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/fail/match/only_inhabited_variant.rs b/src/tools/miri/tests/fail/match/only_inhabited_variant.rs
new file mode 100644
index 000000000000..30a7350d2b66
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/only_inhabited_variant.rs
@@ -0,0 +1,21 @@
+// rust-lang/miri#4778
+#![feature(never_type)]
+
+#[repr(C)]
+#[allow(dead_code)]
+enum E {
+  V0, // discriminant: 0
+  V1(!), // 1
+}
+
+fn main() {
+    assert_eq!(std::mem::size_of::(), 4);
+
+    let val = 1u32;
+    let ptr = (&raw const val).cast::();
+    let r = unsafe { &*ptr };
+    match r { //~ ERROR: read discriminant of an uninhabited enum variant
+        E::V0 => {}
+        E::V1(_) => {}
+    }
+}
diff --git a/src/tools/miri/tests/fail/match/only_inhabited_variant.stderr b/src/tools/miri/tests/fail/match/only_inhabited_variant.stderr
new file mode 100644
index 000000000000..8ca362f902d3
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/only_inhabited_variant.stderr
@@ -0,0 +1,13 @@
+error: Undefined Behavior: read discriminant of an uninhabited enum variant
+  --> tests/fail/match/only_inhabited_variant.rs:LL:CC
+   |
+LL |     match r {
+   |           ^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/fail/match/single_variant.rs b/src/tools/miri/tests/fail/match/single_variant.rs
new file mode 100644
index 000000000000..dcef6d461a2c
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/single_variant.rs
@@ -0,0 +1,31 @@
+// Ideally, this would be UB regardless of #[non_exhaustive]. For now,
+// at least the semantics don't depend on the crate you're in.
+//
+// See: rust-lang/rust#147722
+#![allow(dead_code)]
+
+#[repr(u8)]
+enum Exhaustive {
+    A(u8) = 42,
+}
+
+#[repr(u8)]
+#[non_exhaustive]
+enum NonExhaustive {
+    A(u8) = 42,
+}
+
+fn main() {
+    unsafe {
+        let x: &[u8; 2] = &[21, 37];
+        let y: &Exhaustive = std::mem::transmute(x);
+        match y {
+            Exhaustive::A(_) => {},
+        }
+
+        let y: &NonExhaustive = std::mem::transmute(x);
+        match y { //~ ERROR: enum value has invalid tag
+            NonExhaustive::A(_) => {},
+        }
+    }
+}
diff --git a/src/tools/miri/tests/fail/match/single_variant.stderr b/src/tools/miri/tests/fail/match/single_variant.stderr
new file mode 100644
index 000000000000..7467e27ff791
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/single_variant.stderr
@@ -0,0 +1,13 @@
+error: Undefined Behavior: enum value has invalid tag: 0x15
+  --> tests/fail/match/single_variant.rs:LL:CC
+   |
+LL |         match y {
+   |               ^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/src/tools/miri/tests/fail/match/single_variant_uninit.rs b/src/tools/miri/tests/fail/match/single_variant_uninit.rs
new file mode 100644
index 000000000000..51e8bc57c837
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/single_variant_uninit.rs
@@ -0,0 +1,36 @@
+// Ideally, this would be UB regardless of #[non_exhaustive]. For now,
+// at least the semantics don't depend on the crate you're in.
+//
+// See: rust-lang/rust#147722
+#![allow(dead_code)]
+#![allow(unreachable_patterns)]
+
+#[repr(u8)]
+enum Exhaustive {
+    A(u8) = 0,
+}
+
+#[repr(u8)]
+#[non_exhaustive]
+enum NonExhaustive {
+    A(u8) = 0,
+}
+
+use std::mem::MaybeUninit;
+
+fn main() {
+    let buffer: [MaybeUninit; 2] = [MaybeUninit::uninit(), MaybeUninit::new(0u8)];
+    let exh: *const Exhaustive = (&raw const buffer).cast();
+    let nexh: *const NonExhaustive = (&raw const buffer).cast();
+    unsafe {
+        match *exh {
+            Exhaustive::A(ref _val) => {}
+            _ => {}
+        }
+
+        match *nexh { //~ ERROR: memory is uninitialized
+            NonExhaustive::A(ref _val) => {}
+            _ => {}
+        }
+    }
+}
diff --git a/src/tools/miri/tests/fail/match/single_variant_uninit.stderr b/src/tools/miri/tests/fail/match/single_variant_uninit.stderr
new file mode 100644
index 000000000000..fa8e9babae60
--- /dev/null
+++ b/src/tools/miri/tests/fail/match/single_variant_uninit.stderr
@@ -0,0 +1,18 @@
+error: Undefined Behavior: reading memory at ALLOC[0x0..0x1], but memory is uninitialized at [0x0..0x1], and this operation requires initialized memory
+  --> tests/fail/match/single_variant_uninit.rs:LL:CC
+   |
+LL |         match *nexh {
+   |               ^^^^^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+
+Uninitialized memory occurred at ALLOC[0x0..0x1], in this allocation:
+ALLOC (stack variable, size: 2, align: 1) {
+    __ 00                                           │ ░.
+}
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+
diff --git a/tests/codegen-llvm/enum/enum-transparent-extract.rs b/tests/codegen-llvm/enum/enum-transparent-extract.rs
index 1435e6ec8022..1a05b236abfb 100644
--- a/tests/codegen-llvm/enum/enum-transparent-extract.rs
+++ b/tests/codegen-llvm/enum/enum-transparent-extract.rs
@@ -11,6 +11,8 @@ pub enum Never {}
 pub fn make_unmake_result_never(x: i32) -> i32 {
     // CHECK-LABEL: define i32 @make_unmake_result_never(i32{{( signext)?}} %x)
     // CHECK: start:
+    // CHECK-NEXT: br label %[[next:bb.*]]
+    // CHECK: [[next]]:
     // CHECK-NEXT: ret i32 %x
 
     let y: Result = Ok(x);
@@ -22,6 +24,8 @@ pub fn make_unmake_result_never(x: i32) -> i32 {
 pub fn extract_control_flow_never(x: ControlFlow<&str, Never>) -> &str {
     // CHECK-LABEL: define { ptr, i64 } @extract_control_flow_never(ptr align 1 %x.0, i64 %x.1)
     // CHECK: start:
+    // CHECK-NEXT: br label %[[next:bb.*]]
+    // CHECK: [[next]]:
     // CHECK-NEXT: %[[P0:.+]] = insertvalue { ptr, i64 } poison, ptr %x.0, 0
     // CHECK-NEXT: %[[P1:.+]] = insertvalue { ptr, i64 } %[[P0]], i64 %x.1, 1
     // CHECK-NEXT: ret { ptr, i64 } %[[P1]]
diff --git a/tests/mir-opt/building/match/never_patterns.opt1.SimplifyCfg-initial.after.mir b/tests/mir-opt/building/match/never_patterns.opt1.SimplifyCfg-initial.after.mir
index bba4d9c0149a..78356a90743a 100644
--- a/tests/mir-opt/building/match/never_patterns.opt1.SimplifyCfg-initial.after.mir
+++ b/tests/mir-opt/building/match/never_patterns.opt1.SimplifyCfg-initial.after.mir
@@ -13,17 +13,17 @@ fn opt1(_1: &Result) -> &u32 {
 
     bb0: {
         PlaceMention(_1);
-        falseEdge -> [real: bb4, imaginary: bb1];
+        _2 = discriminant((*_1));
+        switchInt(move _2) -> [0: bb2, 1: bb3, otherwise: bb1];
     }
 
     bb1: {
-        _2 = discriminant((*_1));
-        switchInt(move _2) -> [1: bb3, otherwise: bb2];
+        FakeRead(ForMatchedPlace(None), _1);
+        unreachable;
     }
 
     bb2: {
-        FakeRead(ForMatchedPlace(None), _1);
-        unreachable;
+        falseEdge -> [real: bb4, imaginary: bb3];
     }
 
     bb3: {
diff --git a/tests/mir-opt/building/match/never_patterns.opt2.SimplifyCfg-initial.after.mir b/tests/mir-opt/building/match/never_patterns.opt2.SimplifyCfg-initial.after.mir
index fc0769d6f7dc..979fbb2860dc 100644
--- a/tests/mir-opt/building/match/never_patterns.opt2.SimplifyCfg-initial.after.mir
+++ b/tests/mir-opt/building/match/never_patterns.opt2.SimplifyCfg-initial.after.mir
@@ -11,10 +11,25 @@ fn opt2(_1: &Result) -> &u32 {
 
     bb0: {
         PlaceMention(_1);
+        _2 = discriminant((*_1));
+        switchInt(move _2) -> [0: bb2, 1: bb3, otherwise: bb1];
+    }
+
+    bb1: {
+        FakeRead(ForMatchedPlace(None), _1);
+        unreachable;
+    }
+
+    bb2: {
         StorageLive(_3);
         _3 = &(((*_1) as Ok).0: u32);
         _0 = &(*_3);
         StorageDead(_3);
         return;
     }
+
+    bb3: {
+        FakeRead(ForMatchedPlace(None), (((*_1) as Err).0: Void));
+        unreachable;
+    }
 }
diff --git a/tests/mir-opt/building/match/never_patterns.opt3.SimplifyCfg-initial.after.mir b/tests/mir-opt/building/match/never_patterns.opt3.SimplifyCfg-initial.after.mir
index 86347db4d92e..93ebe600b3ff 100644
--- a/tests/mir-opt/building/match/never_patterns.opt3.SimplifyCfg-initial.after.mir
+++ b/tests/mir-opt/building/match/never_patterns.opt3.SimplifyCfg-initial.after.mir
@@ -12,19 +12,24 @@ fn opt3(_1: &Result) -> &u32 {
     bb0: {
         PlaceMention(_1);
         _2 = discriminant((*_1));
-        switchInt(move _2) -> [1: bb2, otherwise: bb1];
+        switchInt(move _2) -> [0: bb3, 1: bb2, otherwise: bb1];
     }
 
     bb1: {
-        StorageLive(_3);
-        _3 = &(((*_1) as Ok).0: u32);
-        _0 = &(*_3);
-        StorageDead(_3);
-        return;
+        FakeRead(ForMatchedPlace(None), _1);
+        unreachable;
     }
 
     bb2: {
         FakeRead(ForMatchedPlace(None), (((*_1) as Err).0: Void));
         unreachable;
     }
+
+    bb3: {
+        StorageLive(_3);
+        _3 = &(((*_1) as Ok).0: u32);
+        _0 = &(*_3);
+        StorageDead(_3);
+        return;
+    }
 }
diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff
index 97b8d484194f..9630f4001494 100644
--- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff
+++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff
@@ -16,8 +16,10 @@
           debug residual => _6;
           scope 2 {
               scope 8 (inlined #[track_caller]  as FromResidual>>::from_residual) {
-                  let _14: i32;
-                  let mut _15: i32;
+                  let mut _14: isize;
+                  let _15: i32;
+                  let mut _16: i32;
+                  let mut _17: bool;
                   scope 9 {
                       scope 10 (inlined >::from) {
                       }
@@ -74,10 +76,17 @@
           StorageLive(_8);
           _8 = copy _6;
           StorageLive(_14);
-          _14 = move ((_8 as Err).0: i32);
           StorageLive(_15);
-          _15 = move _14;
-          _0 = Result::::Err(move _15);
+          StorageLive(_17);
+          _14 = discriminant(_8);
+          _17 = Eq(copy _14, const 1_isize);
+          assume(move _17);
+          _15 = move ((_8 as Err).0: i32);
+          StorageLive(_16);
+          _16 = move _15;
+          _0 = Result::::Err(move _16);
+          StorageDead(_16);
+          StorageDead(_17);
           StorageDead(_15);
           StorageDead(_14);
           StorageDead(_8);
diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff
index 97b8d484194f..9630f4001494 100644
--- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff
+++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff
@@ -16,8 +16,10 @@
           debug residual => _6;
           scope 2 {
               scope 8 (inlined #[track_caller]  as FromResidual>>::from_residual) {
-                  let _14: i32;
-                  let mut _15: i32;
+                  let mut _14: isize;
+                  let _15: i32;
+                  let mut _16: i32;
+                  let mut _17: bool;
                   scope 9 {
                       scope 10 (inlined >::from) {
                       }
@@ -74,10 +76,17 @@
           StorageLive(_8);
           _8 = copy _6;
           StorageLive(_14);
-          _14 = move ((_8 as Err).0: i32);
           StorageLive(_15);
-          _15 = move _14;
-          _0 = Result::::Err(move _15);
+          StorageLive(_17);
+          _14 = discriminant(_8);
+          _17 = Eq(copy _14, const 1_isize);
+          assume(move _17);
+          _15 = move ((_8 as Err).0: i32);
+          StorageLive(_16);
+          _16 = move _15;
+          _0 = Result::::Err(move _16);
+          StorageDead(_16);
+          StorageDead(_17);
           StorageDead(_15);
           StorageDead(_14);
           StorageDead(_8);
diff --git a/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.PreCodegen.after.mir
index b921b96966b2..ef7ccfa5bddf 100644
--- a/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.PreCodegen.after.mir
@@ -3,56 +3,66 @@
 fn map_via_question_mark(_1: Option) -> Option {
     debug x => _1;
     let mut _0: std::option::Option;
-    let mut _4: std::ops::ControlFlow, i32>;
-    let _5: i32;
-    let mut _6: i32;
+    let mut _4: std::option::Option;
+    let mut _7: std::ops::ControlFlow, i32>;
+    let _8: i32;
+    let mut _9: i32;
     scope 1 {
         debug residual => const Option::::None;
         scope 2 {
             scope 7 (inlined  as FromResidual>>::from_residual) {
+                let mut _3: isize;
+                let mut _5: bool;
             }
         }
     }
     scope 3 {
-        debug val => _5;
+        debug val => _8;
         scope 4 {
         }
     }
     scope 5 (inlined  as Try>::branch) {
         let mut _2: isize;
-        let _3: i32;
+        let _6: i32;
         scope 6 {
         }
     }
 
     bb0: {
-        StorageLive(_6);
-        StorageLive(_4);
+        StorageLive(_9);
+        StorageLive(_7);
         StorageLive(_2);
-        StorageLive(_3);
+        StorageLive(_6);
         _2 = discriminant(_1);
         switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
     }
 
     bb1: {
-        StorageDead(_3);
-        StorageDead(_2);
-        _0 = const Option::::None;
         StorageDead(_6);
-        StorageDead(_4);
+        StorageDead(_2);
+        StorageLive(_3);
+        StorageLive(_5);
+        _3 = discriminant(_4);
+        _5 = Eq(copy _3, const 0_isize);
+        assume(move _5);
+        _0 = const Option::::None;
+        StorageDead(_5);
+        StorageDead(_3);
+        StorageDead(_9);
+        StorageDead(_7);
         goto -> bb3;
     }
 
     bb2: {
-        _3 = copy ((_1 as Some).0: i32);
-        _4 = ControlFlow::, i32>::Continue(copy _3);
-        StorageDead(_3);
-        StorageDead(_2);
-        _5 = copy ((_4 as Continue).0: i32);
-        _6 = Add(copy _5, const 1_i32);
-        _0 = Option::::Some(move _6);
+        _6 = copy ((_1 as Some).0: i32);
+        _7 = ControlFlow::, i32>::Continue(copy _6);
         StorageDead(_6);
-        StorageDead(_4);
+        StorageDead(_2);
+        _8 = copy ((_7 as Continue).0: i32);
+        _9 = Add(copy _8, const 1_i32);
+        _0 = Option::::Some(move _9);
+        StorageDead(_9);
+        StorageDead(_7);
         goto -> bb3;
     }
 
diff --git a/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff b/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff
index 34f451fc698c..10ad4ec75414 100644
--- a/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff
+++ b/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff
@@ -12,7 +12,9 @@
           debug residual => _4;
           scope 2 {
               scope 8 (inlined #[track_caller]  as FromResidual>>::from_residual) {
-                  let _10: i32;
+                  let mut _10: isize;
+                  let _11: i32;
+                  let mut _12: bool;
                   scope 9 {
                       scope 10 (inlined >::from) {
                       }
@@ -58,8 +60,15 @@
   
       bb3: {
           _4 = copy ((_2 as Break).0: std::result::Result);
-          _10 = copy ((_4 as Err).0: i32);
-          _0 = Result::::Err(copy _10);
+          StorageLive(_10);
+          StorageLive(_12);
+          _10 = discriminant(_4);
+          _12 = Eq(copy _10, const 1_isize);
+          assume(move _12);
+          _11 = copy ((_4 as Err).0: i32);
+          _0 = Result::::Err(copy _11);
+          StorageDead(_12);
+          StorageDead(_10);
           StorageDead(_2);
           return;
       }
diff --git a/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff
index 17ddce0cdf8d..8c3b8ad9b66a 100644
--- a/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff
+++ b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff
@@ -19,19 +19,23 @@
   
       bb1: {
           _2 = discriminant(_1);
--         switchInt(move _2) -> [1: bb3, otherwise: bb2];
-+         _5 = Ne(copy _2, const 1_isize);
+-         switchInt(move _2) -> [0: bb3, 1: bb4, otherwise: bb2];
++         _5 = Eq(copy _2, const 0_isize);
 +         assume(move _5);
-+         goto -> bb2;
++         goto -> bb3;
       }
   
       bb2: {
+          unreachable;
+      }
+  
+      bb3: {
           _0 = const ();
           StorageDead(_1);
           return;
       }
   
-      bb3: {
+      bb4: {
 -         StorageLive(_3);
 -         _3 = move ((_1 as Some).0: Empty);
 -         StorageLive(_4);
diff --git a/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff
index 2f78092f5bd2..98f2a0a692ff 100644
--- a/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff
+++ b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff
@@ -19,19 +19,23 @@
   
       bb1: {
           _2 = discriminant(_1);
--         switchInt(move _2) -> [1: bb3, otherwise: bb2];
-+         _5 = Ne(copy _2, const 1_isize);
+-         switchInt(move _2) -> [0: bb3, 1: bb4, otherwise: bb2];
++         _5 = Eq(copy _2, const 0_isize);
 +         assume(move _5);
-+         goto -> bb2;
++         goto -> bb3;
       }
   
       bb2: {
+          unreachable;
+      }
+  
+      bb3: {
           _0 = const ();
           StorageDead(_1);
           return;
       }
   
-      bb3: {
+      bb4: {
 -         StorageLive(_3);
 -         _3 = move ((_1 as Some).0: Empty);
 -         StorageLive(_4);
diff --git a/tests/mir-opt/unreachable.rs b/tests/mir-opt/unreachable.rs
index afab1291fc3d..97cd15b107c1 100644
--- a/tests/mir-opt/unreachable.rs
+++ b/tests/mir-opt/unreachable.rs
@@ -45,7 +45,7 @@ fn as_match() {
     // CHECK: bb0: {
     // CHECK: {{_.*}} = empty()
     // CHECK: bb1: {
-    // CHECK: [[eq:_.*]] = Ne({{.*}}, const 1_isize);
+    // CHECK: [[eq:_.*]] = Eq({{.*}}, const 0_isize);
     // CHECK-NEXT: assume(move [[eq]]);
     // CHECK-NEXT: goto -> [[return:bb.*]];
     // CHECK: [[return]]: {
diff --git a/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-abort.diff b/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-abort.diff
index c24bd7e7446d..befee026d6cb 100644
--- a/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-abort.diff
+++ b/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-abort.diff
@@ -14,40 +14,40 @@
           StorageLive(_2);
           _2 = Test1::C;
           _3 = discriminant(_2);
--         switchInt(move _3) -> [0: bb3, 1: bb2, otherwise: bb1];
-+         switchInt(move _3) -> [0: bb5, 1: bb5, 2: bb1, otherwise: bb5];
+-         switchInt(move _3) -> [0: bb4, 1: bb3, 2: bb2, otherwise: bb1];
++         switchInt(move _3) -> [0: bb1, 1: bb1, 2: bb2, otherwise: bb1];
       }
   
       bb1: {
+          unreachable;
+      }
+  
+      bb2: {
           StorageLive(_5);
           _5 = const "C";
           _1 = &(*_5);
           StorageDead(_5);
-          goto -> bb4;
+          goto -> bb5;
       }
   
-      bb2: {
+      bb3: {
           StorageLive(_4);
           _4 = const "B(Empty)";
           _1 = &(*_4);
           StorageDead(_4);
-          goto -> bb4;
-      }
-  
-      bb3: {
-          _1 = const "A(Empty)";
-          goto -> bb4;
+          goto -> bb5;
       }
   
       bb4: {
+          _1 = const "A(Empty)";
+          goto -> bb5;
+      }
+  
+      bb5: {
           StorageDead(_2);
           StorageDead(_1);
           _0 = const ();
           return;
-+     }
-+ 
-+     bb5: {
-+         unreachable;
       }
   }
   
diff --git a/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-unwind.diff b/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-unwind.diff
index c24bd7e7446d..befee026d6cb 100644
--- a/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-unwind.diff
+++ b/tests/mir-opt/unreachable_enum_branching.simple.UnreachableEnumBranching.panic-unwind.diff
@@ -14,40 +14,40 @@
           StorageLive(_2);
           _2 = Test1::C;
           _3 = discriminant(_2);
--         switchInt(move _3) -> [0: bb3, 1: bb2, otherwise: bb1];
-+         switchInt(move _3) -> [0: bb5, 1: bb5, 2: bb1, otherwise: bb5];
+-         switchInt(move _3) -> [0: bb4, 1: bb3, 2: bb2, otherwise: bb1];
++         switchInt(move _3) -> [0: bb1, 1: bb1, 2: bb2, otherwise: bb1];
       }
   
       bb1: {
+          unreachable;
+      }
+  
+      bb2: {
           StorageLive(_5);
           _5 = const "C";
           _1 = &(*_5);
           StorageDead(_5);
-          goto -> bb4;
+          goto -> bb5;
       }
   
-      bb2: {
+      bb3: {
           StorageLive(_4);
           _4 = const "B(Empty)";
           _1 = &(*_4);
           StorageDead(_4);
-          goto -> bb4;
-      }
-  
-      bb3: {
-          _1 = const "A(Empty)";
-          goto -> bb4;
+          goto -> bb5;
       }
   
       bb4: {
+          _1 = const "A(Empty)";
+          goto -> bb5;
+      }
+  
+      bb5: {
           StorageDead(_2);
           StorageDead(_1);
           _0 = const ();
           return;
-+     }
-+ 
-+     bb5: {
-+         unreachable;
       }
   }
   
diff --git a/tests/ui/match/borrowck-uninhabited.rs b/tests/ui/match/borrowck-uninhabited.rs
index 93838ede2793..34f5e323a91e 100644
--- a/tests/ui/match/borrowck-uninhabited.rs
+++ b/tests/ui/match/borrowck-uninhabited.rs
@@ -17,9 +17,11 @@ fn both_inhabited(x: &mut Result) {
     };
 }
 
+// this used to be accepted, even though it shouldn't
 fn ref_uninhabited(x: &mut Result) {
     match x {
         &mut Ok(ref mut y) => match x {
+        //~^ ERROR: cannot use `*x` because it was mutably borrowed
             &mut Err(ref mut z) => {
                 let _y = y;
                 let _z = z;
diff --git a/tests/ui/match/borrowck-uninhabited.stderr b/tests/ui/match/borrowck-uninhabited.stderr
index ee3b814da9f4..4bbe6ecea16b 100644
--- a/tests/ui/match/borrowck-uninhabited.stderr
+++ b/tests/ui/match/borrowck-uninhabited.stderr
@@ -9,6 +9,17 @@ LL |         &mut Ok(ref mut y) => match x {
 LL |                 let _y = y;
    |                          - borrow later used here
 
-error: aborting due to 1 previous error
+error[E0503]: cannot use `*x` because it was mutably borrowed
+  --> $DIR/borrowck-uninhabited.rs:23:37
+   |
+LL |         &mut Ok(ref mut y) => match x {
+   |                 ---------           ^ use of borrowed `x.0`
+   |                 |
+   |                 `x.0` is borrowed here
+...
+LL |                 let _y = y;
+   |                          - borrow later used here
+
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0503`.
diff --git a/tests/ui/match/uninhabited-granular-moves.rs b/tests/ui/match/uninhabited-granular-moves.rs
index 9d04fcb3259e..f836aedbebc7 100644
--- a/tests/ui/match/uninhabited-granular-moves.rs
+++ b/tests/ui/match/uninhabited-granular-moves.rs
@@ -17,7 +17,7 @@ fn test1(thefoo: Foo<(Box, Box)>) {
         Foo::Bar((a, _)) => { }
     }
 
-    match thefoo {
+    match thefoo { //~ ERROR: use of partially moved value: `thefoo`
         Foo::Bar((_, a)) => { }
     }
 }
@@ -38,7 +38,7 @@ fn test3(thefoo: Foo<(Box, Box)>) {
         Foo::Bar((a, _)) => { }
         Foo::Qux(_) => { }
     }
-    match thefoo {
+    match thefoo { //~ ERROR: use of partially moved value: `thefoo`
         Foo::Bar((_, a)) => { }
     }
 }
diff --git a/tests/ui/match/uninhabited-granular-moves.stderr b/tests/ui/match/uninhabited-granular-moves.stderr
index d32f935d3d55..e3a66db526d4 100644
--- a/tests/ui/match/uninhabited-granular-moves.stderr
+++ b/tests/ui/match/uninhabited-granular-moves.stderr
@@ -1,3 +1,18 @@
+error[E0382]: use of partially moved value: `thefoo`
+  --> $DIR/uninhabited-granular-moves.rs:20:11
+   |
+LL |         Foo::Bar((a, _)) => { }
+   |                   - value partially moved here
+...
+LL |     match thefoo {
+   |           ^^^^^^ value used here after partial move
+   |
+   = note: partial move occurs because value has type `Box`, which does not implement the `Copy` trait
+help: borrow this binding in the pattern to avoid moving the value
+   |
+LL |         Foo::Bar((ref a, _)) => { }
+   |                   +++
+
 error[E0382]: use of partially moved value: `thefoo`
   --> $DIR/uninhabited-granular-moves.rs:30:11
    |
@@ -13,6 +28,21 @@ help: borrow this binding in the pattern to avoid moving the value
 LL |         Foo::Bar((ref a, _)) => { }
    |                   +++
 
-error: aborting due to 1 previous error
+error[E0382]: use of partially moved value: `thefoo`
+  --> $DIR/uninhabited-granular-moves.rs:41:11
+   |
+LL |         Foo::Bar((a, _)) => { }
+   |                   - value partially moved here
+...
+LL |     match thefoo {
+   |           ^^^^^^ value used here after partial move
+   |
+   = note: partial move occurs because value has type `Box`, which does not implement the `Copy` trait
+help: borrow this binding in the pattern to avoid moving the value
+   |
+LL |         Foo::Bar((ref a, _)) => { }
+   |                   +++
+
+error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0382`.
diff --git a/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-exhaustive.rs b/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-exhaustive.rs
index b2ebab382bd9..2e40819d69ad 100644
--- a/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-exhaustive.rs
+++ b/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-exhaustive.rs
@@ -14,11 +14,6 @@ enum Local {
     Variant(u32),
 }
 
-#[non_exhaustive]
-enum LocalNonExhaustive {
-    Variant(u32),
-}
-
 fn main() {
     let mut x = ExhaustiveMonovariant::Variant(1);
     let y = &mut x;
@@ -34,11 +29,4 @@ fn main() {
         _ => {},
     }
     drop(y);
-    let mut x = LocalNonExhaustive::Variant(1);
-    let y = &mut x;
-    match x {
-        LocalNonExhaustive::Variant(_) => {},
-        _ => {},
-    }
-    drop(y);
 }
diff --git a/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.rs b/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.rs
index d616f5e5e89a..a6a369e92a6c 100644
--- a/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.rs
+++ b/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.rs
@@ -6,6 +6,11 @@ extern crate monovariants;
 
 use monovariants::NonExhaustiveMonovariant;
 
+#[non_exhaustive]
+enum LocalNonExhaustive {
+    Variant(u32),
+}
+
 fn main() {
     let mut x = NonExhaustiveMonovariant::Variant(1);
     let y = &mut x;
@@ -15,4 +20,11 @@ fn main() {
         _ => {},
     }
     drop(y);
+    let mut x = LocalNonExhaustive::Variant(1);
+    let y = &mut x;
+    match x { //~ ERROR cannot use `x` because it was mutably borrowed
+        LocalNonExhaustive::Variant(_) => {},
+        _ => {},
+    }
+    drop(y);
 }
diff --git a/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.stderr b/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.stderr
index 70f5b2b84d86..d6225adc95cb 100644
--- a/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.stderr
+++ b/tests/ui/rfcs/rfc-2008-non-exhaustive/borrowck-non-exhaustive.stderr
@@ -1,5 +1,5 @@
 error[E0503]: cannot use `x` because it was mutably borrowed
-  --> $DIR/borrowck-non-exhaustive.rs:12:11
+  --> $DIR/borrowck-non-exhaustive.rs:17:11
    |
 LL |     let y = &mut x;
    |             ------ `x` is borrowed here
@@ -9,6 +9,17 @@ LL |     match x {
 LL |     drop(y);
    |          - borrow later used here
 
-error: aborting due to 1 previous error
+error[E0503]: cannot use `x` because it was mutably borrowed
+  --> $DIR/borrowck-non-exhaustive.rs:25:11
+   |
+LL |     let y = &mut x;
+   |             ------ `x` is borrowed here
+LL |     match x {
+   |           ^ use of borrowed `x`
+...
+LL |     drop(y);
+   |          - borrow later used here
+
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0503`.

From d194795f142639a7155ce82422f47fc56c53ed4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= 
Date: Thu, 15 Jan 2026 19:32:39 +0000
Subject: [PATCH 0271/1843] Do not recover from `Trait()` if generic list is
 unterminated

If we encounter `fn foo`), we bail from the recovery as more likely there could have been a missing closing `>` and the `(` corresponds to the start of the fn parameter list.
---
 compiler/rustc_parse/src/parser/generics.rs   | 13 ++++-
 compiler/rustc_parse/src/parser/mod.rs        |  3 ++
 compiler/rustc_parse/src/parser/ty.rs         | 47 +++++++++++++++++--
 .../missing-closing-generics-bracket.fixed    | 10 ++++
 .../missing-closing-generics-bracket.rs       | 10 ++++
 .../missing-closing-generics-bracket.stderr   | 13 +++++
 6 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 tests/ui/parser/missing-closing-generics-bracket.fixed
 create mode 100644 tests/ui/parser/missing-closing-generics-bracket.rs
 create mode 100644 tests/ui/parser/missing-closing-generics-bracket.stderr

diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs
index ef6c9cc344ce..8c02092fd678 100644
--- a/compiler/rustc_parse/src/parser/generics.rs
+++ b/compiler/rustc_parse/src/parser/generics.rs
@@ -204,9 +204,11 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_generic_params(&mut self) -> PResult<'a, ThinVec> {
         let mut params = ThinVec::new();
         let mut done = false;
+        let prev = self.parsing_generics;
+        self.parsing_generics = true;
         while !done {
             let attrs = self.parse_outer_attributes()?;
-            let param = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
+            let param = match self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
                 if this.eat_keyword_noexpect(kw::SelfUpper) {
                     // `Self` as a generic param is invalid. Here we emit the diagnostic and continue parsing
                     // as if `Self` never existed.
@@ -288,7 +290,13 @@ impl<'a> Parser<'a> {
                 }
                 // We just ate the comma, so no need to capture the trailing token.
                 Ok((param, Trailing::No, UsePreAttrPos::No))
-            })?;
+            }) {
+                Ok(param) => param,
+                Err(err) => {
+                    self.parsing_generics = prev;
+                    return Err(err);
+                }
+            };
 
             if let Some(param) = param {
                 params.push(param);
@@ -296,6 +304,7 @@ impl<'a> Parser<'a> {
                 break;
             }
         }
+        self.parsing_generics = prev;
         Ok(params)
     }
 
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index d6e99bc540f7..ca2048d07147 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -212,6 +212,8 @@ pub struct Parser<'a> {
     /// See the comments in the `parse_path_segment` function for more details.
     unmatched_angle_bracket_count: u16,
     angle_bracket_nesting: u16,
+    /// Keep track of when we're within `<...>` for proper error recovery.
+    parsing_generics: bool = false,
 
     last_unexpected_token_span: Option,
     /// If present, this `Parser` is not parsing Rust code but rather a macro call.
@@ -372,6 +374,7 @@ impl<'a> Parser<'a> {
             },
             current_closure: None,
             recovery: Recovery::Allowed,
+            ..
         };
 
         // Make parser point to the first token.
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 0185c51c5c56..380b6a214846 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -1488,14 +1488,44 @@ impl<'a> Parser<'a> {
             return Ok(());
         }
 
+        let snapshot = if self.parsing_generics {
+            // The snapshot is only relevant if we're parsing the generics of an `fn` to avoid
+            // incorrect recovery.
+            Some(self.create_snapshot_for_diagnostic())
+        } else {
+            None
+        };
         // Parse `(T, U) -> R`.
         let inputs_lo = self.token.span;
         let mode =
             FnParseMode { req_name: |_, _| false, context: FnContext::Free, req_body: false };
-        let inputs: ThinVec<_> =
-            self.parse_fn_params(&mode)?.into_iter().map(|input| input.ty).collect();
+        let params = match self.parse_fn_params(&mode) {
+            Ok(params) => params,
+            Err(err) => {
+                if let Some(snapshot) = snapshot {
+                    self.restore_snapshot(snapshot);
+                    err.cancel();
+                    return Ok(());
+                } else {
+                    return Err(err);
+                }
+            }
+        };
+        let inputs: ThinVec<_> = params.into_iter().map(|input| input.ty).collect();
         let inputs_span = inputs_lo.to(self.prev_token.span);
-        let output = self.parse_ret_ty(AllowPlus::No, RecoverQPath::No, RecoverReturnSign::No)?;
+        let output = match self.parse_ret_ty(AllowPlus::No, RecoverQPath::No, RecoverReturnSign::No)
+        {
+            Ok(output) => output,
+            Err(err) => {
+                if let Some(snapshot) = snapshot {
+                    self.restore_snapshot(snapshot);
+                    err.cancel();
+                    return Ok(());
+                } else {
+                    return Err(err);
+                }
+            }
+        };
         let args = ast::ParenthesizedArgs {
             span: fn_path_segment.span().to(self.prev_token.span),
             inputs,
@@ -1503,6 +1533,17 @@ impl<'a> Parser<'a> {
             output,
         }
         .into();
+
+        if let Some(snapshot) = snapshot
+            && ![token::Comma, token::Gt, token::Plus].contains(&self.token.kind)
+        {
+            // We would expect another bound or the end of type params by now. Most likely we've
+            // encountered a `(` *not* representing `Trait()`, but rather the start of the `fn`'s
+            // argument list where the generic param list wasn't properly closed.
+            self.restore_snapshot(snapshot);
+            return Ok(());
+        }
+
         *fn_path_segment = ast::PathSegment {
             ident: fn_path_segment.ident,
             args: Some(args),
diff --git a/tests/ui/parser/missing-closing-generics-bracket.fixed b/tests/ui/parser/missing-closing-generics-bracket.fixed
new file mode 100644
index 000000000000..3166887fa8c3
--- /dev/null
+++ b/tests/ui/parser/missing-closing-generics-bracket.fixed
@@ -0,0 +1,10 @@
+// Issue #141436
+//@ run-rustfix
+#![allow(dead_code)]
+
+trait Trait<'a> {}
+
+fn foo>() {}
+//~^ ERROR expected one of
+
+fn main() {}
diff --git a/tests/ui/parser/missing-closing-generics-bracket.rs b/tests/ui/parser/missing-closing-generics-bracket.rs
new file mode 100644
index 000000000000..9424e3467246
--- /dev/null
+++ b/tests/ui/parser/missing-closing-generics-bracket.rs
@@ -0,0 +1,10 @@
+// Issue #141436
+//@ run-rustfix
+#![allow(dead_code)]
+
+trait Trait<'a> {}
+
+fn foo() {}
+//~^ ERROR expected one of
+
+fn main() {}
diff --git a/tests/ui/parser/missing-closing-generics-bracket.stderr b/tests/ui/parser/missing-closing-generics-bracket.stderr
new file mode 100644
index 000000000000..c4287301c595
--- /dev/null
+++ b/tests/ui/parser/missing-closing-generics-bracket.stderr
@@ -0,0 +1,13 @@
+error: expected one of `+`, `,`, `::`, `=`, or `>`, found `(`
+  --> $DIR/missing-closing-generics-bracket.rs:7:25
+   |
+LL | fn foo() {}
+   |                         ^ expected one of `+`, `,`, `::`, `=`, or `>`
+   |
+help: you might have meant to end the type parameters here
+   |
+LL | fn foo>() {}
+   |                         +
+
+error: aborting due to 1 previous error
+

From 3df0dc880376ef16076851046c40f2ad7374b63c Mon Sep 17 00:00:00 2001
From: The 8472 
Date: Sat, 10 Jan 2026 14:21:33 +0100
Subject: [PATCH 0272/1843] mark rust_dealloc as captures(address)

Co-authored-by: Ralf Jung 
---
 compiler/rustc_codegen_llvm/src/attributes.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index a25ce9e5a90a..bf6bb81b53b0 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -517,7 +517,16 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free));
         // applies to argument place instead of function place
         let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx);
-        attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]);
+        let attrs: &[_] = if llvm_util::get_version() >= (21, 0, 0) {
+            // "Does not capture provenance" means "if the function call stashes the pointer somewhere,
+            // accessing that pointer after the function returns is UB". That is definitely the case here since
+            // freeing will destroy the provenance.
+            let captures_addr = AttributeKind::CapturesAddress.create_attr(cx.llcx);
+            &[allocated_pointer, captures_addr]
+        } else {
+            &[allocated_pointer]
+        };
+        attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), attrs);
     }
     if let Some(align) = codegen_fn_attrs.alignment {
         llvm::set_alignment(llfn, align);

From c6c4372f8231bab89007267c23a42e35b177578b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= 
Date: Thu, 15 Jan 2026 19:46:54 +0000
Subject: [PATCH 0273/1843] Use default field values in `Parser`

---
 compiler/rustc_ast/src/token.rs               |  4 +-
 compiler/rustc_parse/src/parser/mod.rs        | 39 +++++++------------
 compiler/rustc_parse/src/parser/token_type.rs |  2 +-
 3 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index accf4d181632..e0807dbceee4 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -625,12 +625,12 @@ impl TokenKind {
 }
 
 impl Token {
-    pub fn new(kind: TokenKind, span: Span) -> Self {
+    pub const fn new(kind: TokenKind, span: Span) -> Self {
         Token { kind, span }
     }
 
     /// Some token that will be thrown away later.
-    pub fn dummy() -> Self {
+    pub const fn dummy() -> Self {
         Token::new(TokenKind::Question, DUMMY_SP)
     }
 
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index ca2048d07147..4145cd5727dc 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -175,17 +175,17 @@ pub enum Recovery {
 pub struct Parser<'a> {
     pub psess: &'a ParseSess,
     /// The current token.
-    pub token: Token,
+    pub token: Token = Token::dummy(),
     /// The spacing for the current token.
-    token_spacing: Spacing,
+    token_spacing: Spacing = Spacing::Alone,
     /// The previous token.
-    pub prev_token: Token,
-    pub capture_cfg: bool,
-    restrictions: Restrictions,
-    expected_token_types: TokenTypeSet,
+    pub prev_token: Token = Token::dummy(),
+    pub capture_cfg: bool = false,
+    restrictions: Restrictions = Restrictions::empty(),
+    expected_token_types: TokenTypeSet = TokenTypeSet::new(),
     token_cursor: TokenCursor,
     // The number of calls to `bump`, i.e. the position in the token stream.
-    num_bump_calls: u32,
+    num_bump_calls: u32 = 0,
     // During parsing we may sometimes need to "unglue" a glued token into two
     // or three component tokens (e.g. `>>` into `>` and `>`, or `>>=` into `>`
     // and `>` and `=`), so the parser can consume them one at a time. This
@@ -204,27 +204,27 @@ pub struct Parser<'a> {
     //
     // This value is always 0, 1, or 2. It can only reach 2 when splitting
     // `>>=` or `<<=`.
-    break_last_token: u32,
+    break_last_token: u32 = 0,
     /// This field is used to keep track of how many left angle brackets we have seen. This is
     /// required in order to detect extra leading left angle brackets (`<` characters) and error
     /// appropriately.
     ///
     /// See the comments in the `parse_path_segment` function for more details.
-    unmatched_angle_bracket_count: u16,
-    angle_bracket_nesting: u16,
+    unmatched_angle_bracket_count: u16 = 0,
+    angle_bracket_nesting: u16 = 0,
     /// Keep track of when we're within `<...>` for proper error recovery.
     parsing_generics: bool = false,
 
-    last_unexpected_token_span: Option,
+    last_unexpected_token_span: Option = None,
     /// If present, this `Parser` is not parsing Rust code but rather a macro call.
     subparser_name: Option<&'static str>,
     capture_state: CaptureState,
     /// This allows us to recover when the user forget to add braces around
     /// multiple statements in the closure body.
-    current_closure: Option,
+    current_closure: Option = None,
     /// Whether the parser is allowed to do recovery.
     /// This is disabled when parsing macro arguments, see #103534
-    recovery: Recovery,
+    recovery: Recovery = Recovery::Allowed,
 }
 
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with
@@ -353,18 +353,7 @@ impl<'a> Parser<'a> {
     ) -> Self {
         let mut parser = Parser {
             psess,
-            token: Token::dummy(),
-            token_spacing: Spacing::Alone,
-            prev_token: Token::dummy(),
-            capture_cfg: false,
-            restrictions: Restrictions::empty(),
-            expected_token_types: TokenTypeSet::new(),
             token_cursor: TokenCursor { curr: TokenTreeCursor::new(stream), stack: Vec::new() },
-            num_bump_calls: 0,
-            break_last_token: 0,
-            unmatched_angle_bracket_count: 0,
-            angle_bracket_nesting: 0,
-            last_unexpected_token_span: None,
             subparser_name,
             capture_state: CaptureState {
                 capturing: Capturing::No,
@@ -372,8 +361,6 @@ impl<'a> Parser<'a> {
                 inner_attr_parser_ranges: Default::default(),
                 seen_attrs: IntervalSet::new(u32::MAX as usize),
             },
-            current_closure: None,
-            recovery: Recovery::Allowed,
             ..
         };
 
diff --git a/compiler/rustc_parse/src/parser/token_type.rs b/compiler/rustc_parse/src/parser/token_type.rs
index e5dda7cf9104..567b1be5e5d9 100644
--- a/compiler/rustc_parse/src/parser/token_type.rs
+++ b/compiler/rustc_parse/src/parser/token_type.rs
@@ -585,7 +585,7 @@ macro_rules! exp {
 pub(super) struct TokenTypeSet(u128);
 
 impl TokenTypeSet {
-    pub(super) fn new() -> TokenTypeSet {
+    pub(super) const fn new() -> TokenTypeSet {
         TokenTypeSet(0)
     }
 

From b4781c8fe9f97b582b44d615d2edf2bb114f1f2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= 
Date: Thu, 15 Jan 2026 20:39:14 +0000
Subject: [PATCH 0274/1843] Use default field values in a few more cases

Makes the use expressions significantly shorter.
---
 .../src/diagnostics/conflict_errors.rs        | 48 ++++++-------------
 compiler/rustc_borrowck/src/lib.rs            |  1 +
 .../rustc_hir_analysis/src/hir_wf_check.rs    | 18 ++-----
 compiler/rustc_hir_analysis/src/lib.rs        |  1 +
 4 files changed, 22 insertions(+), 46 deletions(-)

diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 5cfe9db009bf..1b1c72394e75 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -2312,12 +2312,12 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             tcx: TyCtxt<'hir>,
             issue_span: Span,
             expr_span: Span,
-            body_expr: Option<&'hir hir::Expr<'hir>>,
-            loop_bind: Option<&'hir Ident>,
-            loop_span: Option,
-            head_span: Option,
-            pat_span: Option,
-            head: Option<&'hir hir::Expr<'hir>>,
+            body_expr: Option<&'hir hir::Expr<'hir>> = None,
+            loop_bind: Option<&'hir Ident> = None,
+            loop_span: Option = None,
+            head_span: Option = None,
+            pat_span: Option = None,
+            head: Option<&'hir hir::Expr<'hir>> = None,
         }
         impl<'hir> Visitor<'hir> for ExprFinder<'hir> {
             fn visit_expr(&mut self, ex: &'hir hir::Expr<'hir>) {
@@ -2383,17 +2383,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 hir::intravisit::walk_expr(self, ex);
             }
         }
-        let mut finder = ExprFinder {
-            tcx,
-            expr_span: span,
-            issue_span,
-            loop_bind: None,
-            body_expr: None,
-            head_span: None,
-            loop_span: None,
-            pat_span: None,
-            head: None,
-        };
+        let mut finder = ExprFinder { tcx, expr_span: span, issue_span, .. };
         finder.visit_expr(tcx.hir_body(body_id).value);
 
         if let Some(body_expr) = finder.body_expr
@@ -2628,13 +2618,13 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
 
         struct ExpressionFinder<'tcx> {
             capture_span: Span,
-            closure_change_spans: Vec,
-            closure_arg_span: Option,
-            in_closure: bool,
-            suggest_arg: String,
+            closure_change_spans: Vec = vec![],
+            closure_arg_span: Option = None,
+            in_closure: bool = false,
+            suggest_arg: String = String::new(),
             tcx: TyCtxt<'tcx>,
-            closure_local_id: Option,
-            closure_call_changes: Vec<(Span, String)>,
+            closure_local_id: Option = None,
+            closure_call_changes: Vec<(Span, String)> = vec![],
         }
         impl<'hir> Visitor<'hir> for ExpressionFinder<'hir> {
             fn visit_expr(&mut self, e: &'hir hir::Expr<'hir>) {
@@ -2715,16 +2705,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         }) = self.infcx.tcx.hir_node(self.mir_hir_id())
             && let hir::Node::Expr(expr) = self.infcx.tcx.hir_node(body_id.hir_id)
         {
-            let mut finder = ExpressionFinder {
-                capture_span: *capture_kind_span,
-                closure_change_spans: vec![],
-                closure_arg_span: None,
-                in_closure: false,
-                suggest_arg: String::new(),
-                closure_local_id: None,
-                closure_call_changes: vec![],
-                tcx: self.infcx.tcx,
-            };
+            let mut finder =
+                ExpressionFinder { capture_span: *capture_kind_span, tcx: self.infcx.tcx, .. };
             finder.visit_expr(expr);
 
             if finder.closure_change_spans.is_empty() || finder.closure_call_changes.is_empty() {
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 91defbad0a0e..f2ea8cef192e 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -4,6 +4,7 @@
 #![allow(internal_features)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]
+#![feature(default_field_values)]
 #![feature(file_buffered)]
 #![feature(if_let_guard)]
 #![feature(negative_impls)]
diff --git a/compiler/rustc_hir_analysis/src/hir_wf_check.rs b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
index f879153c5765..d414f4dbcc24 100644
--- a/compiler/rustc_hir_analysis/src/hir_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
@@ -51,12 +51,12 @@ pub(super) fn diagnostic_hir_wf_check<'tcx>(
     struct HirWfCheck<'tcx> {
         tcx: TyCtxt<'tcx>,
         predicate: ty::Predicate<'tcx>,
-        cause: Option>,
-        cause_depth: usize,
+        cause: Option> = None,
+        cause_depth: usize = 0,
         icx: ItemCtxt<'tcx>,
         def_id: LocalDefId,
         param_env: ty::ParamEnv<'tcx>,
-        depth: usize,
+        depth: usize = 0,
     }
 
     impl<'tcx> Visitor<'tcx> for HirWfCheck<'tcx> {
@@ -124,16 +124,8 @@ pub(super) fn diagnostic_hir_wf_check<'tcx>(
         }
     }
 
-    let mut visitor = HirWfCheck {
-        tcx,
-        predicate,
-        cause: None,
-        cause_depth: 0,
-        icx,
-        def_id,
-        param_env: tcx.param_env(def_id.to_def_id()),
-        depth: 0,
-    };
+    let param_env = tcx.param_env(def_id.to_def_id());
+    let mut visitor = HirWfCheck { tcx, predicate, icx, def_id, param_env, .. };
 
     // Get the starting `hir::Ty` using our `WellFormedLoc`.
     // We will walk 'into' this type to try to find
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index a51355adf72f..95f0e641c611 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -59,6 +59,7 @@ This API is completely unstable and subject to change.
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
 #![feature(assert_matches)]
+#![feature(default_field_values)]
 #![feature(gen_blocks)]
 #![feature(if_let_guard)]
 #![feature(iter_intersperse)]

From 676d71a45b4fa9966e77e9d4fe6508d8bbea7d29 Mon Sep 17 00:00:00 2001
From: Hugh 
Date: Thu, 15 Jan 2026 16:22:23 -0800
Subject: [PATCH 0275/1843] Defer is_from_proc_macro check and add tests

Move the is_from_proc_macro check inside check_fn_inner to be called
only right before emitting lints, rather than at the entry points.
This avoids the expensive check when early returns would prevent any
lint emission anyway.

Add tests for proc-macro generated code covering all check locations:
- Standalone functions
- Methods in impl blocks
- Trait methods
- Impl blocks with extra lifetimes
---
 clippy_lints/src/lifetimes.rs          | 46 +++++++++++++++++---------
 tests/ui/extra_unused_lifetimes.rs     | 32 ++++++++++++++++++
 tests/ui/extra_unused_lifetimes.stderr | 14 ++++----
 tests/ui/needless_lifetimes.fixed      | 25 +++++++++++++-
 tests/ui/needless_lifetimes.rs         | 25 +++++++++++++-
 5 files changed, 118 insertions(+), 24 deletions(-)

diff --git a/clippy_lints/src/lifetimes.rs b/clippy_lints/src/lifetimes.rs
index 8917c90262a4..679fb983d532 100644
--- a/clippy_lints/src/lifetimes.rs
+++ b/clippy_lints/src/lifetimes.rs
@@ -149,10 +149,9 @@ impl<'tcx> LateLintPass<'tcx> for Lifetimes {
             ..
         } = item.kind
         {
-            if is_from_proc_macro(cx, item) {
-                return;
-            }
-            check_fn_inner(cx, sig, Some(id), None, generics, item.span, true, self.msrv);
+            check_fn_inner(cx, sig, Some(id), None, generics, item.span, true, self.msrv, || {
+                is_from_proc_macro(cx, item)
+            });
         } else if let ItemKind::Impl(impl_) = &item.kind
             && !item.span.from_expansion()
             && !is_from_proc_macro(cx, item)
@@ -162,9 +161,7 @@ impl<'tcx> LateLintPass<'tcx> for Lifetimes {
     }
 
     fn check_impl_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx ImplItem<'_>) {
-        if let ImplItemKind::Fn(ref sig, id) = item.kind
-            && !is_from_proc_macro(cx, item)
-        {
+        if let ImplItemKind::Fn(ref sig, id) = item.kind {
             let report_extra_lifetimes = trait_ref_of_method(cx, item.owner_id).is_none();
             check_fn_inner(
                 cx,
@@ -175,19 +172,28 @@ impl<'tcx> LateLintPass<'tcx> for Lifetimes {
                 item.span,
                 report_extra_lifetimes,
                 self.msrv,
+                || is_from_proc_macro(cx, item),
             );
         }
     }
 
     fn check_trait_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx TraitItem<'_>) {
-        if let TraitItemKind::Fn(ref sig, ref body) = item.kind
-            && !is_from_proc_macro(cx, item)
-        {
+        if let TraitItemKind::Fn(ref sig, ref body) = item.kind {
             let (body, trait_sig) = match *body {
                 TraitFn::Required(sig) => (None, Some(sig)),
                 TraitFn::Provided(id) => (Some(id), None),
             };
-            check_fn_inner(cx, sig, body, trait_sig, item.generics, item.span, true, self.msrv);
+            check_fn_inner(
+                cx,
+                sig,
+                body,
+                trait_sig,
+                item.generics,
+                item.span,
+                true,
+                self.msrv,
+                || is_from_proc_macro(cx, item),
+            );
         }
     }
 }
@@ -202,6 +208,7 @@ fn check_fn_inner<'tcx>(
     span: Span,
     report_extra_lifetimes: bool,
     msrv: Msrv,
+    is_from_proc_macro: impl FnOnce() -> bool,
 ) {
     if span.in_external_macro(cx.sess().source_map()) || has_where_lifetimes(cx, generics) {
         return;
@@ -253,10 +260,19 @@ fn check_fn_inner<'tcx>(
         }
     }
 
-    if let Some((elidable_lts, usages)) = could_use_elision(cx, sig.decl, body, trait_sig, generics.params, msrv) {
-        if usages.iter().any(|usage| !usage.ident.span.eq_ctxt(span)) {
-            return;
-        }
+    let elidable = could_use_elision(cx, sig.decl, body, trait_sig, generics.params, msrv);
+    let has_elidable_lts = elidable
+        .as_ref()
+        .is_some_and(|(_, usages)| !usages.iter().any(|usage| !usage.ident.span.eq_ctxt(span)));
+
+    // Only check is_from_proc_macro if we're about to emit a lint (it's an expensive check)
+    if (has_elidable_lts || report_extra_lifetimes) && is_from_proc_macro() {
+        return;
+    }
+
+    if let Some((elidable_lts, usages)) = elidable
+        && has_elidable_lts
+    {
         // async functions have usages whose spans point at the lifetime declaration which messes up
         // suggestions
         let include_suggestions = !sig.header.is_async();
diff --git a/tests/ui/extra_unused_lifetimes.rs b/tests/ui/extra_unused_lifetimes.rs
index 5fdcd5a7e078..677523489599 100644
--- a/tests/ui/extra_unused_lifetimes.rs
+++ b/tests/ui/extra_unused_lifetimes.rs
@@ -1,4 +1,5 @@
 //@aux-build:proc_macro_derive.rs
+//@aux-build:proc_macros.rs
 
 #![allow(
     unused,
@@ -11,6 +12,7 @@
 
 #[macro_use]
 extern crate proc_macro_derive;
+extern crate proc_macros;
 
 fn empty() {}
 
@@ -148,4 +150,34 @@ mod issue_13578 {
     impl<'a, T: 'a> Foo for Option where &'a T: Foo {}
 }
 
+// no lint on proc macro generated code
+mod proc_macro_generated {
+    use proc_macros::external;
+
+    // no lint on external macro (extra unused lifetimes in impl block)
+    external! {
+        struct ExternalImplStruct;
+
+        impl<'a> ExternalImplStruct {
+            fn foo() {}
+        }
+    }
+
+    // no lint on external macro (extra unused lifetimes in method)
+    external! {
+        struct ExternalMethodStruct;
+
+        impl ExternalMethodStruct {
+            fn bar<'a>(&self) {}
+        }
+    }
+
+    // no lint on external macro (extra unused lifetimes in trait method)
+    external! {
+        trait ExternalUnusedLifetimeTrait {
+            fn unused_lt<'a>(x: u8) {}
+        }
+    }
+}
+
 fn main() {}
diff --git a/tests/ui/extra_unused_lifetimes.stderr b/tests/ui/extra_unused_lifetimes.stderr
index 0cecbbe80f76..d748376d476c 100644
--- a/tests/ui/extra_unused_lifetimes.stderr
+++ b/tests/ui/extra_unused_lifetimes.stderr
@@ -1,5 +1,5 @@
 error: this lifetime isn't used in the function definition
-  --> tests/ui/extra_unused_lifetimes.rs:19:14
+  --> tests/ui/extra_unused_lifetimes.rs:21:14
    |
 LL | fn unused_lt<'a>(x: u8) {}
    |              ^^
@@ -8,37 +8,37 @@ LL | fn unused_lt<'a>(x: u8) {}
    = help: to override `-D warnings` add `#[allow(clippy::extra_unused_lifetimes)]`
 
 error: this lifetime isn't used in the function definition
-  --> tests/ui/extra_unused_lifetimes.rs:47:10
+  --> tests/ui/extra_unused_lifetimes.rs:49:10
    |
 LL |     fn x<'a>(&self) {}
    |          ^^
 
 error: this lifetime isn't used in the function definition
-  --> tests/ui/extra_unused_lifetimes.rs:74:22
+  --> tests/ui/extra_unused_lifetimes.rs:76:22
    |
 LL |         fn unused_lt<'a>(x: u8) {}
    |                      ^^
 
 error: this lifetime isn't used in the impl
-  --> tests/ui/extra_unused_lifetimes.rs:86:10
+  --> tests/ui/extra_unused_lifetimes.rs:88:10
    |
 LL |     impl<'a> std::ops::AddAssign<&Scalar> for &mut Scalar {
    |          ^^
 
 error: this lifetime isn't used in the impl
-  --> tests/ui/extra_unused_lifetimes.rs:93:10
+  --> tests/ui/extra_unused_lifetimes.rs:95:10
    |
 LL |     impl<'b> Scalar {
    |          ^^
 
 error: this lifetime isn't used in the function definition
-  --> tests/ui/extra_unused_lifetimes.rs:95:26
+  --> tests/ui/extra_unused_lifetimes.rs:97:26
    |
 LL |         pub fn something<'c>() -> Self {
    |                          ^^
 
 error: this lifetime isn't used in the impl
-  --> tests/ui/extra_unused_lifetimes.rs:125:10
+  --> tests/ui/extra_unused_lifetimes.rs:127:10
    |
 LL |     impl<'a, T: Source + ?Sized + 'a> Source for Box {
    |          ^^
diff --git a/tests/ui/needless_lifetimes.fixed b/tests/ui/needless_lifetimes.fixed
index 15ca409c95bd..90a07454b4d7 100644
--- a/tests/ui/needless_lifetimes.fixed
+++ b/tests/ui/needless_lifetimes.fixed
@@ -470,13 +470,36 @@ mod in_macro {
         }
     }
 
-    // no lint on external macro
+    // no lint on external macro (standalone function)
     external! {
         fn needless_lifetime<'a>(x: &'a u8) -> &'a u8 {
             unimplemented!()
         }
     }
 
+    // no lint on external macro (method in impl block)
+    external! {
+        struct ExternalStruct;
+
+        impl ExternalStruct {
+            fn needless_lifetime_method<'a>(x: &'a u8) -> &'a u8 {
+                unimplemented!()
+            }
+        }
+    }
+
+    // no lint on external macro (trait method)
+    external! {
+        trait ExternalTrait {
+            fn needless_lifetime_trait_method<'a>(x: &'a u8) -> &'a u8;
+        }
+    }
+
+    // no lint on external macro (extra unused lifetimes in function)
+    external! {
+        fn extra_unused_lifetime<'a>(x: u8) {}
+    }
+
     inline! {
         fn f<$'a>(arg: &$'a str) -> &$'a str {
             arg
diff --git a/tests/ui/needless_lifetimes.rs b/tests/ui/needless_lifetimes.rs
index af9649d72987..6df38897f42d 100644
--- a/tests/ui/needless_lifetimes.rs
+++ b/tests/ui/needless_lifetimes.rs
@@ -470,13 +470,36 @@ mod in_macro {
         }
     }
 
-    // no lint on external macro
+    // no lint on external macro (standalone function)
     external! {
         fn needless_lifetime<'a>(x: &'a u8) -> &'a u8 {
             unimplemented!()
         }
     }
 
+    // no lint on external macro (method in impl block)
+    external! {
+        struct ExternalStruct;
+
+        impl ExternalStruct {
+            fn needless_lifetime_method<'a>(x: &'a u8) -> &'a u8 {
+                unimplemented!()
+            }
+        }
+    }
+
+    // no lint on external macro (trait method)
+    external! {
+        trait ExternalTrait {
+            fn needless_lifetime_trait_method<'a>(x: &'a u8) -> &'a u8;
+        }
+    }
+
+    // no lint on external macro (extra unused lifetimes in function)
+    external! {
+        fn extra_unused_lifetime<'a>(x: u8) {}
+    }
+
     inline! {
         fn f<$'a>(arg: &$'a str) -> &$'a str {
             arg

From f19591ad27ba306ffe0d2665790894420e0f96bc Mon Sep 17 00:00:00 2001
From: Redddy 
Date: Fri, 16 Jan 2026 18:39:38 +0900
Subject: [PATCH 0276/1843] Add rust-analyzer setup for out-of-tree
 rustc_private crates

---
 .../remarks-on-perma-unstable-features.md     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md b/src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md
index b434cfc9cf14..3c2fe133be84 100644
--- a/src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md
+++ b/src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md
@@ -49,6 +49,29 @@ For custom-built toolchains or environments not using rustup, additional configu
    ```
 3. **Check version compatibility**: Ensure your LLVM version is compatible with your Rust toolchain
 
+### Configuring `rust-analyzer` for Out-of-Tree Projects
+
+When developing out-of-tree projects that use `rustc_private` crates, you can configure `rust-analyzer` to recognize these crates.
+
+#### Configuration Steps
+
+1. **Set rust-analyzer configuration**
+   Configure `rust-analyzer.rustc.source` to `"discover"` in your editor settings.  
+   For VS Code, add to `rust_analyzer_settings.json`:
+   ```json
+   {
+       "rust-analyzer.rustc.source": "discover"
+   }
+   ```
+2. **Add metadata to Cargo.toml**  
+   Add the following to the `Cargo.toml` of every crate that uses `rustc_private`:
+   ```toml
+   [package.metadata.rust-analyzer]
+   rustc_private = true
+   ```
+
+This configuration allows `rust-analyzer` to properly recognize and provide IDE support for `rustc_private` crates in out-of-tree projects. 
+
 ### Additional Resources
 
 - [GitHub Issue #137421](https://github.com/rust-lang/rust/issues/137421): Explains that `rustc_private` linker failures often occur because `llvm-tools` is not installed

From 21f269e72ed90f73abbd531243d6f563d2689650 Mon Sep 17 00:00:00 2001
From: lcnr 
Date: Fri, 16 Jan 2026 10:47:01 +0100
Subject: [PATCH 0277/1843] add RPITIT cycle issue

---
 .../src/solve/candidate-preference.md         | 41 ++++++++++++++++++-
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/src/doc/rustc-dev-guide/src/solve/candidate-preference.md b/src/doc/rustc-dev-guide/src/solve/candidate-preference.md
index 8b28f56760a7..d5a75b41ff1f 100644
--- a/src/doc/rustc-dev-guide/src/solve/candidate-preference.md
+++ b/src/doc/rustc-dev-guide/src/solve/candidate-preference.md
@@ -260,11 +260,13 @@ We prefer builtin trait object impls over user-written impls. This is **unsound*
 
 The candidate preference behavior during normalization is implemented in [`fn assemble_and_merge_candidates`].
 
-### Where-bounds shadow impls
+### Trait where-bounds shadow impls
 
 Normalization of associated items does not consider impls if the corresponding trait goal has been proven via a `ParamEnv` or `AliasBound` candidate.
 This means that for where-bounds which do not constrain associated types, the associated types remain *rigid*.
 
+#### Using impls results in different region constraints
+
 This is necessary to avoid unnecessary region constraints from applying impls.
 ```rust
 trait Trait<'a> {
@@ -286,6 +288,39 @@ where
 }
 ```
 
+#### RPITIT `type_of` cycles
+
+We currently have to avoid impl candidates if there are where-bounds to avoid query cycles for RPITIT, see [#139762]. It feels desirable to me to stop relying on auto-trait leakage of during RPITIT computation to remove this issue, see [#139788].
+
+```rust
+use std::future::Future;
+pub trait ReactiveFunction: Send {
+    type Output;
+
+    fn invoke(self) -> Self::Output;
+}
+
+trait AttributeValue {
+    fn resolve(self) -> impl Future + Send;
+}
+
+impl AttributeValue for F
+where
+    F: ReactiveFunction,
+    V: AttributeValue,
+{
+    async fn resolve(self) {
+        // We're awaiting `::{synthetic#0}` here.
+        // Normalizing that one via the the impl we're currently in
+        // relies on `collect_return_position_impl_trait_in_trait_tys` which
+        // ends up relying on auto-trait leakage when checking that the
+        // opaque return type of this function implements the `Send` item
+        // bound of the trait definition.
+        self.invoke().resolve().await
+    }
+}
+```
+
 ### We always consider `AliasBound` candidates
 
 In case the where-bound does not specify the associated item, we consider `AliasBound` candidates instead of treating the alias as rigid, even though the trait goal was proven via a `ParamEnv` candidate.
@@ -424,4 +459,6 @@ where
 [`fn assemble_and_merge_candidates`]: https://github.com/rust-lang/rust/blob/e3ee7f7aea5b45af3b42b5e4713da43876a65ac9/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs#L920-L1003
 [trait-system-refactor-initiative#76]: https://github.com/rust-lang/trait-system-refactor-initiative/issues/76
 [#24066]: https://github.com/rust-lang/rust/issues/24066
-[#133044]: https://github.com/rust-lang/rust/issues/133044
\ No newline at end of file
+[#133044]: https://github.com/rust-lang/rust/issues/133044
+[#139762]: https://github.com/rust-lang/rust/pull/139762
+[#139788]: https://github.com/rust-lang/rust/issues/139788
\ No newline at end of file

From 908ae5a86f5191bf316456dcc9257a942d8bce7f Mon Sep 17 00:00:00 2001
From: lcnr 
Date: Fri, 16 Jan 2026 10:48:01 +0100
Subject: [PATCH 0278/1843] move section

---
 .../src/solve/candidate-preference.md         | 121 +++++++++---------
 1 file changed, 61 insertions(+), 60 deletions(-)

diff --git a/src/doc/rustc-dev-guide/src/solve/candidate-preference.md b/src/doc/rustc-dev-guide/src/solve/candidate-preference.md
index d5a75b41ff1f..461523424b3c 100644
--- a/src/doc/rustc-dev-guide/src/solve/candidate-preference.md
+++ b/src/doc/rustc-dev-guide/src/solve/candidate-preference.md
@@ -260,66 +260,6 @@ We prefer builtin trait object impls over user-written impls. This is **unsound*
 
 The candidate preference behavior during normalization is implemented in [`fn assemble_and_merge_candidates`].
 
-### Trait where-bounds shadow impls
-
-Normalization of associated items does not consider impls if the corresponding trait goal has been proven via a `ParamEnv` or `AliasBound` candidate.
-This means that for where-bounds which do not constrain associated types, the associated types remain *rigid*.
-
-#### Using impls results in different region constraints
-
-This is necessary to avoid unnecessary region constraints from applying impls.
-```rust
-trait Trait<'a> {
-    type Assoc;
-}
-impl Trait<'static> for u32 {
-    type Assoc = u32;
-}
-
-fn bar<'b, T: Trait<'b>>() -> T::Assoc { todo!() }
-fn foo<'a>()
-where
-    u32: Trait<'a>,
-{
-    // Normalizing the return type would use the impl, proving
-    // the `T: Trait` where-bound would use the where-bound, resulting
-    // in different region constraints.
-    bar::<'_, u32>();
-}
-```
-
-#### RPITIT `type_of` cycles
-
-We currently have to avoid impl candidates if there are where-bounds to avoid query cycles for RPITIT, see [#139762]. It feels desirable to me to stop relying on auto-trait leakage of during RPITIT computation to remove this issue, see [#139788].
-
-```rust
-use std::future::Future;
-pub trait ReactiveFunction: Send {
-    type Output;
-
-    fn invoke(self) -> Self::Output;
-}
-
-trait AttributeValue {
-    fn resolve(self) -> impl Future + Send;
-}
-
-impl AttributeValue for F
-where
-    F: ReactiveFunction,
-    V: AttributeValue,
-{
-    async fn resolve(self) {
-        // We're awaiting `::{synthetic#0}` here.
-        // Normalizing that one via the the impl we're currently in
-        // relies on `collect_return_position_impl_trait_in_trait_tys` which
-        // ends up relying on auto-trait leakage when checking that the
-        // opaque return type of this function implements the `Send` item
-        // bound of the trait definition.
-        self.invoke().resolve().await
-    }
-}
-```
 
 ### We always consider `AliasBound` candidates
 
@@ -453,6 +393,67 @@ where
 }
 ```
 
+### Trait where-bounds shadow impls
+
+Normalization of associated items does not consider impls if the corresponding trait goal has been proven via a `ParamEnv` or `AliasBound` candidate.
+This means that for where-bounds which do not constrain associated types, the associated types remain *rigid*.
+
+#### Using impls results in different region constraints
+
+This is necessary to avoid unnecessary region constraints from applying impls.
+```rust
+trait Trait<'a> {
+    type Assoc;
+}
+impl Trait<'static> for u32 {
+    type Assoc = u32;
+}
+
+fn bar<'b, T: Trait<'b>>() -> T::Assoc { todo!() }
+fn foo<'a>()
+where
+    u32: Trait<'a>,
+{
+    // Normalizing the return type would use the impl, proving
+    // the `T: Trait` where-bound would use the where-bound, resulting
+    // in different region constraints.
+    bar::<'_, u32>();
+}
+```
+
+#### RPITIT `type_of` cycles
+
+We currently have to avoid impl candidates if there are where-bounds to avoid query cycles for RPITIT, see [#139762]. It feels desirable to me to stop relying on auto-trait leakage of during RPITIT computation to remove this issue, see [#139788].
+
+```rust
+use std::future::Future;
+pub trait ReactiveFunction: Send {
+    type Output;
+
+    fn invoke(self) -> Self::Output;
+}
+
+trait AttributeValue {
+    fn resolve(self) -> impl Future + Send;
+}
+
+impl AttributeValue for F
+where
+    F: ReactiveFunction,
+    V: AttributeValue,
+{
+    async fn resolve(self) {
+        // We're awaiting `::{synthetic#0}` here.
+        // Normalizing that one via the the impl we're currently in
+        // relies on `collect_return_position_impl_trait_in_trait_tys` which
+        // ends up relying on auto-trait leakage when checking that the
+        // opaque return type of this function implements the `Send` item
+        // bound of the trait definition.
+        self.invoke().resolve().await
+    }
+}
+```
+
 [`Candidate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/assembly/struct.Candidate.html
 [`CandidateSource`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/enum.CandidateSource.html
 [`fn merge_trait_candidates`]: https://github.com/rust-lang/rust/blob/e3ee7f7aea5b45af3b42b5e4713da43876a65ac9/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs#L1342-L1424

From 9e4c55ac8d9d1083983bdc2c35a3513cddb3fc69 Mon Sep 17 00:00:00 2001
From: reddevilmidzy 
Date: Fri, 16 Jan 2026 19:26:34 +0900
Subject: [PATCH 0279/1843] Rename "remarks on perma unstable features" to
 "external rustc_drivers"

---
 src/doc/rustc-dev-guide/src/SUMMARY.md                          | 2 +-
 ...-on-perma-unstable-features.md => external-rustc-drivers.md} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename src/doc/rustc-dev-guide/src/rustc-driver/{remarks-on-perma-unstable-features.md => external-rustc-drivers.md} (98%)

diff --git a/src/doc/rustc-dev-guide/src/SUMMARY.md b/src/doc/rustc-dev-guide/src/SUMMARY.md
index 2dd07144a78c..daaaef42d909 100644
--- a/src/doc/rustc-dev-guide/src/SUMMARY.md
+++ b/src/doc/rustc-dev-guide/src/SUMMARY.md
@@ -140,7 +140,7 @@
 
 - [Command-line arguments](./cli.md)
 - [rustc_driver and rustc_interface](./rustc-driver/intro.md)
-    - [Remarks on perma-unstable features](./rustc-driver/remarks-on-perma-unstable-features.md)
+    - [External rustc_drivers](./rustc-driver/external-rustc-drivers.md)
     - [Example: Type checking](./rustc-driver/interacting-with-the-ast.md)
     - [Example: Getting diagnostics](./rustc-driver/getting-diagnostics.md)
 - [Errors and lints](diagnostics.md)
diff --git a/src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md b/src/doc/rustc-dev-guide/src/rustc-driver/external-rustc-drivers.md
similarity index 98%
rename from src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md
rename to src/doc/rustc-dev-guide/src/rustc-driver/external-rustc-drivers.md
index 3c2fe133be84..1049d7a82ddd 100644
--- a/src/doc/rustc-dev-guide/src/rustc-driver/remarks-on-perma-unstable-features.md
+++ b/src/doc/rustc-dev-guide/src/rustc-driver/external-rustc-drivers.md
@@ -1,4 +1,4 @@
-# Remarks on perma unstable features
+# External `rustc_driver`s
 
 ## `rustc_private`
 

From 567b569e2bb131abcbf77eef7d98b7f50d47465a Mon Sep 17 00:00:00 2001
From: Clara Engler 
Date: Fri, 16 Jan 2026 10:39:35 +0100
Subject: [PATCH 0280/1843] time: Add saturating arithmetic for `SystemTime`

This commit implements the following methods:
* `SystemTime::saturating_add`
* `SystemTime::saturating_sub`
* `SystemTime::saturating_duration_since`

The implementation of these methods is rather trivial, as the main logic
lies behind the constants `SystemTime::MIN` and `SystemTime::MAX`.
---
 library/std/src/time.rs   | 50 +++++++++++++++++++++++++++++++++++++++
 library/std/tests/time.rs | 39 ++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)

diff --git a/library/std/src/time.rs b/library/std/src/time.rs
index 3d09824019e3..1805d8926098 100644
--- a/library/std/src/time.rs
+++ b/library/std/src/time.rs
@@ -682,6 +682,56 @@ impl SystemTime {
     pub fn checked_sub(&self, duration: Duration) -> Option {
         self.0.checked_sub_duration(&duration).map(SystemTime)
     }
+
+    /// Saturating [`SystemTime`] addition, computing `self + duration`,
+    /// returning [`SystemTime::MAX`] if overflow occurred.
+    ///
+    /// In the case that the `duration` is smaller than the time precision of
+    /// the operating system, `self` will be returned.
+    #[unstable(feature = "time_saturating_systemtime", issue = "151199")]
+    pub fn saturating_add(&self, duration: Duration) -> SystemTime {
+        self.checked_add(duration).unwrap_or(SystemTime::MAX)
+    }
+
+    /// Saturating [`SystemTime`] subtraction, computing `self - duration`,
+    /// returning [`SystemTime::MIN`] if overflow occurred.
+    ///
+    /// In the case that the `duration` is smaller than the time precision of
+    /// the operating system, `self` will be returned.
+    #[unstable(feature = "time_saturating_systemtime", issue = "151199")]
+    pub fn saturating_sub(&self, duration: Duration) -> SystemTime {
+        self.checked_sub(duration).unwrap_or(SystemTime::MIN)
+    }
+
+    /// Saturating computation of time elapsed from an earlier point in time,
+    /// returning [`Duration::ZERO`] in the case that `earlier` is later or
+    /// equal to `self`.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// #![feature(time_saturating_systemtime)]
+    /// use std::time::{Duration, SystemTime};
+    ///
+    /// let now = SystemTime::now();
+    /// let prev = now.saturating_sub(Duration::new(1, 0));
+    ///
+    /// // now - prev should return non-zero.
+    /// assert_eq!(now.saturating_duration_since(prev), Duration::new(1, 0));
+    /// assert!(now.duration_since(prev).is_ok());
+    ///
+    /// // prev - now should return zero (and fail with the non-saturating).
+    /// assert_eq!(prev.saturating_duration_since(now), Duration::ZERO);
+    /// assert!(prev.duration_since(now).is_err());
+    ///
+    /// // now - now should return zero (and work with the non-saturating).
+    /// assert_eq!(now.saturating_duration_since(now), Duration::ZERO);
+    /// assert!(now.duration_since(now).is_ok());
+    /// ```
+    #[unstable(feature = "time_saturating_systemtime", issue = "151199")]
+    pub fn saturating_duration_since(&self, earlier: SystemTime) -> Duration {
+        self.duration_since(earlier).unwrap_or(Duration::ZERO)
+    }
 }
 
 #[stable(feature = "time2", since = "1.8.0")]
diff --git a/library/std/tests/time.rs b/library/std/tests/time.rs
index 0ef89bb09c63..b73e7bc3962e 100644
--- a/library/std/tests/time.rs
+++ b/library/std/tests/time.rs
@@ -1,5 +1,6 @@
 #![feature(duration_constants)]
 #![feature(time_systemtime_limits)]
+#![feature(time_saturating_systemtime)]
 
 use std::fmt::Debug;
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
@@ -269,3 +270,41 @@ fn system_time_max_min() {
     assert!(SystemTime::MIN.checked_add(MIN_INTERVAL).is_some());
     assert!(SystemTime::MIN.checked_sub(MIN_INTERVAL).is_none());
 }
+
+#[test]
+fn system_time_saturating() {
+    // Perform saturating addition on SystemTime::MAX to see how it behaves.
+    assert_eq!(SystemTime::MAX.saturating_add(Duration::ZERO), SystemTime::MAX);
+    assert_eq!(SystemTime::MAX.saturating_add(Duration::new(1, 0)), SystemTime::MAX);
+    assert!(SystemTime::MAX.checked_add(Duration::new(1, 0)).is_none());
+    assert_eq!(
+        SystemTime::MAX.saturating_sub(Duration::new(1, 0)),
+        SystemTime::MAX.checked_sub(Duration::new(1, 0)).unwrap()
+    );
+
+    // Perform saturating subtraction on SystemTime::MIn to see how it behaves.
+    assert_eq!(SystemTime::MIN.saturating_sub(Duration::ZERO), SystemTime::MIN);
+    assert_eq!(SystemTime::MIN.saturating_sub(Duration::new(1, 0)), SystemTime::MIN);
+    assert!(SystemTime::MIN.checked_sub(Duration::new(1, 0)).is_none());
+    assert_eq!(
+        SystemTime::MIN.saturating_add(Duration::new(1, 0)),
+        SystemTime::MIN.checked_add(Duration::new(1, 0)).unwrap()
+    );
+
+    // Check saturating_duration_since with various constant values.
+    assert!(SystemTime::MAX.saturating_duration_since(SystemTime::MIN) >= Duration::ZERO);
+    assert_eq!(SystemTime::MAX.saturating_duration_since(SystemTime::MAX), Duration::ZERO);
+    assert!(SystemTime::MAX.duration_since(SystemTime::MAX).is_ok());
+    assert_eq!(SystemTime::MIN.saturating_duration_since(SystemTime::MAX), Duration::ZERO);
+    assert!(SystemTime::MIN.duration_since(SystemTime::MAX).is_err());
+    assert_eq!(
+        (SystemTime::UNIX_EPOCH + Duration::new(1, 0))
+            .saturating_duration_since(SystemTime::UNIX_EPOCH),
+        Duration::new(1, 0)
+    );
+    assert_eq!(
+        SystemTime::UNIX_EPOCH
+            .saturating_duration_since(SystemTime::UNIX_EPOCH + Duration::new(1, 0)),
+        Duration::ZERO
+    );
+}

From 19e0d7914f2865c707ace136fd33ab6f8c2931fb Mon Sep 17 00:00:00 2001
From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com>
Date: Fri, 16 Jan 2026 13:08:09 +0000
Subject: [PATCH 0281/1843] Mention `cast_signed` in docs of
 `cast_possible_wrap`

changelog: [`cast_possible_wrap`]: mention `cast_{un,}signed()` methods in doc

I was evaluating this lint recently, and accepted using it
because these methods exist.
But the docs on the lint don't mention it, so I thought
it would be prudent to include it in the docs.

See also https://github.com/rust-lang/rust-clippy/pull/15384

Co-authored-by: Kaur Kuut 
Co-authored-by: Samuel Tardieu 
---
 clippy_lints/src/casts/mod.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/clippy_lints/src/casts/mod.rs b/clippy_lints/src/casts/mod.rs
index 7220a8a80066..3c9ebef73f0d 100644
--- a/clippy_lints/src/casts/mod.rs
+++ b/clippy_lints/src/casts/mod.rs
@@ -145,6 +145,12 @@ declare_clippy_lint! {
     /// let _ = i32::try_from(u32::MAX).ok();
     /// ```
     ///
+    /// If the wrapping is intended, you can use:
+    /// ```no_run
+    /// let _ = u32::MAX.cast_signed();
+    /// let _ = (-1i32).cast_unsigned();
+    /// ```
+    ///
     #[clippy::version = "pre 1.29.0"]
     pub CAST_POSSIBLE_WRAP,
     pedantic,

From 2f3b9ce72c700766ad7e331b698afb38fd04f9b6 Mon Sep 17 00:00:00 2001
From: Dima Khort 
Date: Wed, 14 Jan 2026 00:31:54 +0100
Subject: [PATCH 0282/1843] feat(strlen_on_c_strings): suggest .count_bytes()

---
 clippy_lints/src/lib.rs                 |  2 +-
 clippy_lints/src/strlen_on_c_strings.rs | 31 +++++++++++++----
 clippy_utils/src/msrvs.rs               |  2 +-
 tests/ui/strlen_on_c_strings.fixed      | 42 ++++++++++++++++------
 tests/ui/strlen_on_c_strings.rs         | 22 ++++++++++++
 tests/ui/strlen_on_c_strings.stderr     | 46 +++++++++++++++++++------
 6 files changed, 116 insertions(+), 29 deletions(-)

diff --git a/clippy_lints/src/lib.rs b/clippy_lints/src/lib.rs
index ef2461f8b097..bd9db80d98ad 100644
--- a/clippy_lints/src/lib.rs
+++ b/clippy_lints/src/lib.rs
@@ -718,7 +718,7 @@ pub fn register_lint_passes(store: &mut rustc_lint::LintStore, conf: &'static Co
         Box::new(|_| Box::::default()),
         Box::new(move |tcx| Box::new(disallowed_types::DisallowedTypes::new(tcx, conf))),
         Box::new(move |tcx| Box::new(missing_enforced_import_rename::ImportRename::new(tcx, conf))),
-        Box::new(|_| Box::new(strlen_on_c_strings::StrlenOnCStrings)),
+        Box::new(move |_| Box::new(strlen_on_c_strings::StrlenOnCStrings::new(conf))),
         Box::new(move |_| Box::new(self_named_constructors::SelfNamedConstructors)),
         Box::new(move |_| Box::new(iter_not_returning_iterator::IterNotReturningIterator)),
         Box::new(move |_| Box::new(manual_assert::ManualAssert)),
diff --git a/clippy_lints/src/strlen_on_c_strings.rs b/clippy_lints/src/strlen_on_c_strings.rs
index 5eb160720c52..962ab9cce14c 100644
--- a/clippy_lints/src/strlen_on_c_strings.rs
+++ b/clippy_lints/src/strlen_on_c_strings.rs
@@ -1,4 +1,6 @@
+use clippy_config::Conf;
 use clippy_utils::diagnostics::span_lint_and_then;
+use clippy_utils::msrvs::{self, Msrv};
 use clippy_utils::res::MaybeDef;
 use clippy_utils::source::snippet_with_context;
 use clippy_utils::visitors::is_expr_unsafe;
@@ -6,12 +8,12 @@ use clippy_utils::{match_libc_symbol, sym};
 use rustc_errors::Applicability;
 use rustc_hir::{Block, BlockCheckMode, Expr, ExprKind, LangItem, Node, UnsafeSource};
 use rustc_lint::{LateContext, LateLintPass};
-use rustc_session::declare_lint_pass;
+use rustc_session::impl_lint_pass;
 
 declare_clippy_lint! {
     /// ### What it does
     /// Checks for usage of `libc::strlen` on a `CString` or `CStr` value,
-    /// and suggest calling `as_bytes().len()` or `to_bytes().len()` respectively instead.
+    /// and suggest calling `count_bytes()` instead.
     ///
     /// ### Why is this bad?
     /// libc::strlen is an unsafe function, which we don't need to call
@@ -27,15 +29,25 @@ declare_clippy_lint! {
     /// ```rust, no_run
     /// use std::ffi::CString;
     /// let cstring = CString::new("foo").expect("CString::new failed");
-    /// let len = cstring.as_bytes().len();
+    /// let len = cstring.count_bytes();
     /// ```
     #[clippy::version = "1.55.0"]
     pub STRLEN_ON_C_STRINGS,
     complexity,
-    "using `libc::strlen` on a `CString` or `CStr` value, while `as_bytes().len()` or `to_bytes().len()` respectively can be used instead"
+    "using `libc::strlen` on a `CString` or `CStr` value, while `count_bytes()` can be used instead"
 }
 
-declare_lint_pass!(StrlenOnCStrings => [STRLEN_ON_C_STRINGS]);
+pub struct StrlenOnCStrings {
+    msrv: Msrv,
+}
+
+impl StrlenOnCStrings {
+    pub fn new(conf: &Conf) -> Self {
+        Self { msrv: conf.msrv }
+    }
+}
+
+impl_lint_pass!(StrlenOnCStrings => [STRLEN_ON_C_STRINGS]);
 
 impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
     fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) {
@@ -80,7 +92,14 @@ impl<'tcx> LateLintPass<'tcx> for StrlenOnCStrings {
                 |diag| {
                     let mut app = Applicability::MachineApplicable;
                     let val_name = snippet_with_context(cx, self_arg.span, ctxt, "_", &mut app).0;
-                    diag.span_suggestion(span, "use", format!("{val_name}.to_bytes().len()"), app);
+
+                    let suggestion = if self.msrv.meets(cx, msrvs::CSTR_COUNT_BYTES) {
+                        format!("{val_name}.count_bytes()")
+                    } else {
+                        format!("{val_name}.to_bytes().len()")
+                    };
+
+                    diag.span_suggestion(span, "use", suggestion, app);
                 },
             );
         }
diff --git a/clippy_utils/src/msrvs.rs b/clippy_utils/src/msrvs.rs
index 39a2c2df1f81..18fab6035f28 100644
--- a/clippy_utils/src/msrvs.rs
+++ b/clippy_utils/src/msrvs.rs
@@ -32,7 +32,7 @@ msrv_aliases! {
     1,82,0 { IS_NONE_OR, REPEAT_N, RAW_REF_OP, SPECIALIZED_TO_STRING_FOR_REFS }
     1,81,0 { LINT_REASONS_STABILIZATION, ERROR_IN_CORE, EXPLICIT_SELF_TYPE_ELISION, DURATION_ABS_DIFF }
     1,80,0 { BOX_INTO_ITER, LAZY_CELL }
-    1,79,0 { CONST_BLOCKS }
+    1,79,0 { CONST_BLOCKS, CSTR_COUNT_BYTES }
     1,77,0 { C_STR_LITERALS }
     1,76,0 { PTR_FROM_REF, OPTION_RESULT_INSPECT }
     1,75,0 { OPTION_AS_SLICE }
diff --git a/tests/ui/strlen_on_c_strings.fixed b/tests/ui/strlen_on_c_strings.fixed
index 33a328af6df4..6604da70874d 100644
--- a/tests/ui/strlen_on_c_strings.fixed
+++ b/tests/ui/strlen_on_c_strings.fixed
@@ -7,40 +7,62 @@ use std::ffi::{CStr, CString};
 fn main() {
     // CString
     let cstring = CString::new("foo").expect("CString::new failed");
-    let _ = cstring.to_bytes().len();
+    let _ = cstring.count_bytes();
     //~^ ERROR: using `libc::strlen` on a `CString` value
 
     // CStr
     let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
-    let _ = cstr.to_bytes().len();
+    let _ = cstr.count_bytes();
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 
-    let _ = cstr.to_bytes().len();
+    let _ = cstr.count_bytes();
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let pcstr: *const &CStr = &cstr;
-    let _ = unsafe { (*pcstr).to_bytes().len() };
+    let _ = unsafe { (*pcstr).count_bytes() };
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     unsafe fn unsafe_identity(x: T) -> T {
         x
     }
-    let _ = unsafe { unsafe_identity(cstr).to_bytes().len() };
+    let _ = unsafe { unsafe_identity(cstr).count_bytes() };
     //~^ ERROR: using `libc::strlen` on a `CStr` value
-    let _ = unsafe { unsafe_identity(cstr) }.to_bytes().len();
+    let _ = unsafe { unsafe_identity(cstr) }.count_bytes();
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 
     let f: unsafe fn(_) -> _ = unsafe_identity;
-    let _ = unsafe { f(cstr).to_bytes().len() };
+    let _ = unsafe { f(cstr).count_bytes() };
     //~^ ERROR: using `libc::strlen` on a `CStr` value
 }
 
 // make sure we lint types that _adjust_ to `CStr`
 fn adjusted(box_cstring: Box, box_cstr: Box, arc_cstring: std::sync::Arc) {
-    let _ = box_cstring.to_bytes().len();
+    let _ = box_cstring.count_bytes();
     //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
-    let _ = box_cstr.to_bytes().len();
+    let _ = box_cstr.count_bytes();
     //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
-    let _ = arc_cstring.to_bytes().len();
+    let _ = arc_cstring.count_bytes();
     //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
 }
+
+#[clippy::msrv = "1.78"]
+fn msrv_1_78() {
+    let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
+    let _ = cstr.to_bytes().len();
+    //~^ strlen_on_c_strings
+
+    let cstring = CString::new("foo").expect("CString::new failed");
+    let _ = cstring.to_bytes().len();
+    //~^ strlen_on_c_strings
+}
+
+#[clippy::msrv = "1.79"]
+fn msrv_1_79() {
+    let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
+    let _ = cstr.count_bytes();
+    //~^ strlen_on_c_strings
+
+    let cstring = CString::new("foo").expect("CString::new failed");
+    let _ = cstring.count_bytes();
+    //~^ strlen_on_c_strings
+}
diff --git a/tests/ui/strlen_on_c_strings.rs b/tests/ui/strlen_on_c_strings.rs
index 3c11c3a05269..11fbdf585064 100644
--- a/tests/ui/strlen_on_c_strings.rs
+++ b/tests/ui/strlen_on_c_strings.rs
@@ -44,3 +44,25 @@ fn adjusted(box_cstring: Box, box_cstr: Box, arc_cstring: std::sy
     let _ = unsafe { libc::strlen(arc_cstring.as_ptr()) };
     //~^ ERROR: using `libc::strlen` on a type that dereferences to `CStr`
 }
+
+#[clippy::msrv = "1.78"]
+fn msrv_1_78() {
+    let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
+    let _ = unsafe { libc::strlen(cstr.as_ptr()) };
+    //~^ strlen_on_c_strings
+
+    let cstring = CString::new("foo").expect("CString::new failed");
+    let _ = unsafe { libc::strlen(cstring.as_ptr()) };
+    //~^ strlen_on_c_strings
+}
+
+#[clippy::msrv = "1.79"]
+fn msrv_1_79() {
+    let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
+    let _ = unsafe { libc::strlen(cstr.as_ptr()) };
+    //~^ strlen_on_c_strings
+
+    let cstring = CString::new("foo").expect("CString::new failed");
+    let _ = unsafe { libc::strlen(cstring.as_ptr()) };
+    //~^ strlen_on_c_strings
+}
diff --git a/tests/ui/strlen_on_c_strings.stderr b/tests/ui/strlen_on_c_strings.stderr
index 2b059872a2da..1f1b5ccdb0ef 100644
--- a/tests/ui/strlen_on_c_strings.stderr
+++ b/tests/ui/strlen_on_c_strings.stderr
@@ -2,7 +2,7 @@ error: using `libc::strlen` on a `CString` value
   --> tests/ui/strlen_on_c_strings.rs:10:13
    |
 LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.count_bytes()`
    |
    = note: `-D clippy::strlen-on-c-strings` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::strlen_on_c_strings)]`
@@ -11,55 +11,79 @@ error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:15:13
    |
 LL |     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.count_bytes()`
 
 error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:18:13
    |
 LL |     let _ = unsafe { strlen(cstr.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.count_bytes()`
 
 error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:22:22
    |
 LL |     let _ = unsafe { strlen((*pcstr).as_ptr()) };
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `(*pcstr).to_bytes().len()`
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `(*pcstr).count_bytes()`
 
 error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:28:22
    |
 LL |     let _ = unsafe { strlen(unsafe_identity(cstr).as_ptr()) };
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe_identity(cstr).to_bytes().len()`
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe_identity(cstr).count_bytes()`
 
 error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:30:13
    |
 LL |     let _ = unsafe { strlen(unsafe { unsafe_identity(cstr) }.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe { unsafe_identity(cstr) }.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `unsafe { unsafe_identity(cstr) }.count_bytes()`
 
 error: using `libc::strlen` on a `CStr` value
   --> tests/ui/strlen_on_c_strings.rs:34:22
    |
 LL |     let _ = unsafe { strlen(f(cstr).as_ptr()) };
-   |                      ^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `f(cstr).to_bytes().len()`
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `f(cstr).count_bytes()`
 
 error: using `libc::strlen` on a type that dereferences to `CStr`
   --> tests/ui/strlen_on_c_strings.rs:40:13
    |
 LL |     let _ = unsafe { libc::strlen(box_cstring.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `box_cstring.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `box_cstring.count_bytes()`
 
 error: using `libc::strlen` on a type that dereferences to `CStr`
   --> tests/ui/strlen_on_c_strings.rs:42:13
    |
 LL |     let _ = unsafe { libc::strlen(box_cstr.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `box_cstr.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `box_cstr.count_bytes()`
 
 error: using `libc::strlen` on a type that dereferences to `CStr`
   --> tests/ui/strlen_on_c_strings.rs:44:13
    |
 LL |     let _ = unsafe { libc::strlen(arc_cstring.as_ptr()) };
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `arc_cstring.to_bytes().len()`
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `arc_cstring.count_bytes()`
 
-error: aborting due to 10 previous errors
+error: using `libc::strlen` on a `CStr` value
+  --> tests/ui/strlen_on_c_strings.rs:51:13
+   |
+LL |     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.to_bytes().len()`
+
+error: using `libc::strlen` on a `CString` value
+  --> tests/ui/strlen_on_c_strings.rs:55:13
+   |
+LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.to_bytes().len()`
+
+error: using `libc::strlen` on a `CStr` value
+  --> tests/ui/strlen_on_c_strings.rs:62:13
+   |
+LL |     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstr.count_bytes()`
+
+error: using `libc::strlen` on a `CString` value
+  --> tests/ui/strlen_on_c_strings.rs:66:13
+   |
+LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `cstring.count_bytes()`
+
+error: aborting due to 14 previous errors
 

From 7d3bf37c4ddacf866129c90b58e0a4e7728368f2 Mon Sep 17 00:00:00 2001
From: Daedalus <16168171+RedDaedalus@users.noreply.github.com>
Date: Mon, 12 Jan 2026 19:02:03 -0700
Subject: [PATCH 0283/1843] fix fallback impl for select_unpredictable
 intrinsic

---
 library/core/src/intrinsics/mod.rs            | 15 +++++++++------
 .../intrinsics/select-unpredictable-drop.rs   | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+), 6 deletions(-)
 create mode 100644 src/tools/miri/tests/pass/intrinsics/select-unpredictable-drop.rs

diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs
index 20f34036b25c..ac3456eb904e 100644
--- a/library/core/src/intrinsics/mod.rs
+++ b/library/core/src/intrinsics/mod.rs
@@ -55,7 +55,7 @@
 #![allow(missing_docs)]
 
 use crate::ffi::va_list::{VaArgSafe, VaList};
-use crate::marker::{ConstParamTy, Destruct, DiscriminantKind, PointeeSized, Tuple};
+use crate::marker::{ConstParamTy, DiscriminantKind, PointeeSized, Tuple};
 use crate::{mem, ptr};
 
 mod bounds;
@@ -482,11 +482,14 @@ pub const fn unlikely(b: bool) -> bool {
 #[rustc_nounwind]
 #[miri::intrinsic_fallback_is_spec]
 #[inline]
-pub const fn select_unpredictable(b: bool, true_val: T, false_val: T) -> T
-where
-    T: [const] Destruct,
-{
-    if b { true_val } else { false_val }
+pub const fn select_unpredictable(b: bool, true_val: T, false_val: T) -> T {
+    if b {
+        forget(false_val);
+        true_val
+    } else {
+        forget(true_val);
+        false_val
+    }
 }
 
 /// A guard for unsafe functions that cannot ever be executed if `T` is uninhabited:
diff --git a/src/tools/miri/tests/pass/intrinsics/select-unpredictable-drop.rs b/src/tools/miri/tests/pass/intrinsics/select-unpredictable-drop.rs
new file mode 100644
index 000000000000..ecf9f4b92058
--- /dev/null
+++ b/src/tools/miri/tests/pass/intrinsics/select-unpredictable-drop.rs
@@ -0,0 +1,19 @@
+//! Check that `select_unpredictable` properly forgets the value it does not select.
+#![feature(core_intrinsics)]
+use std::cell::Cell;
+use std::intrinsics::select_unpredictable;
+
+fn main() {
+    let (true_val, false_val) = (Cell::new(false), Cell::new(false));
+    _ = select_unpredictable(true, TraceDrop(&true_val), TraceDrop(&false_val));
+    assert!(true_val.get());
+    assert!(!false_val.get());
+}
+
+struct TraceDrop<'a>(&'a Cell);
+
+impl<'a> Drop for TraceDrop<'a> {
+    fn drop(&mut self) {
+        self.0.set(true);
+    }
+}

From 3e789ed57663cac867309eb4d9e1f14094b02900 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Mon, 12 Jan 2026 16:38:10 +0100
Subject: [PATCH 0284/1843] Stop cloning old `llvmint` repositories to generate
 intrinsics

---
 src/intrinsic/archs.rs       | 1424 +---------------------------------
 src/intrinsic/llvm.rs        |    2 +
 src/intrinsic/old_archs.rs   | 1384 +++++++++++++++++++++++++++++++++
 tools/generate_intrinsics.py |   50 +-
 4 files changed, 1420 insertions(+), 1440 deletions(-)
 create mode 100644 src/intrinsic/old_archs.rs

diff --git a/src/intrinsic/archs.rs b/src/intrinsic/archs.rs
index 43e7c352c34a..89a6cf7f3d6e 100644
--- a/src/intrinsic/archs.rs
+++ b/src/intrinsic/archs.rs
@@ -6,47 +6,13 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
         unimplemented!("***** unsupported LLVM intrinsic {}", full_name)
     };
     let Some((arch, name)) = name.split_once('.') else {
-        unimplemented!("***** unsupported LLVM intrinsic {}", name)
+        unimplemented!("***** unsupported LLVM intrinsic llvm.{}", name)
     };
+    let old_arch_res = old_archs(arch, name);
+    if let ArchCheckResult::Ok(res) = old_arch_res {
+        return res;
+    }
     match arch {
-        "AMDGPU" => {
-            #[expect(non_snake_case)]
-            fn AMDGPU(name: &str, full_name: &str) -> &'static str {
-                match name {
-                    // AMDGPU
-                    "div.fixup.f32" => "__builtin_amdgpu_div_fixup",
-                    "div.fixup.f64" => "__builtin_amdgpu_div_fixup",
-                    "div.fixup.v2f64" => "__builtin_amdgpu_div_fixup",
-                    "div.fixup.v4f32" => "__builtin_amdgpu_div_fixup",
-                    "div.fmas.f32" => "__builtin_amdgpu_div_fmas",
-                    "div.fmas.f64" => "__builtin_amdgpu_div_fmas",
-                    "div.fmas.v2f64" => "__builtin_amdgpu_div_fmas",
-                    "div.fmas.v4f32" => "__builtin_amdgpu_div_fmas",
-                    "ldexp.f32" => "__builtin_amdgpu_ldexp",
-                    "ldexp.f64" => "__builtin_amdgpu_ldexp",
-                    "ldexp.v2f64" => "__builtin_amdgpu_ldexp",
-                    "ldexp.v4f32" => "__builtin_amdgpu_ldexp",
-                    "rcp.f32" => "__builtin_amdgpu_rcp",
-                    "rcp.f64" => "__builtin_amdgpu_rcp",
-                    "rcp.v2f64" => "__builtin_amdgpu_rcp",
-                    "rcp.v4f32" => "__builtin_amdgpu_rcp",
-                    "rsq.clamped.f32" => "__builtin_amdgpu_rsq_clamped",
-                    "rsq.clamped.f64" => "__builtin_amdgpu_rsq_clamped",
-                    "rsq.clamped.v2f64" => "__builtin_amdgpu_rsq_clamped",
-                    "rsq.clamped.v4f32" => "__builtin_amdgpu_rsq_clamped",
-                    "rsq.f32" => "__builtin_amdgpu_rsq",
-                    "rsq.f64" => "__builtin_amdgpu_rsq",
-                    "rsq.v2f64" => "__builtin_amdgpu_rsq",
-                    "rsq.v4f32" => "__builtin_amdgpu_rsq",
-                    "trig.preop.f32" => "__builtin_amdgpu_trig_preop",
-                    "trig.preop.f64" => "__builtin_amdgpu_trig_preop",
-                    "trig.preop.v2f64" => "__builtin_amdgpu_trig_preop",
-                    "trig.preop.v4f32" => "__builtin_amdgpu_trig_preop",
-                    _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),
-                }
-            }
-            AMDGPU(name, full_name)
-        }
         "aarch64" => {
             fn aarch64(name: &str, full_name: &str) -> &'static str {
                 match name {
@@ -537,8 +503,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "ldcl" => "__builtin_arm_ldcl",
                     "mcr" => "__builtin_arm_mcr",
                     "mcr2" => "__builtin_arm_mcr2",
-                    "mcrr" => "__builtin_arm_mcrr",
-                    "mcrr2" => "__builtin_arm_mcrr2",
                     "mrc" => "__builtin_arm_mrc",
                     "mrc2" => "__builtin_arm_mrc2",
                     "qadd" => "__builtin_arm_qadd",
@@ -595,7 +559,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "stcl" => "__builtin_arm_stcl",
                     "sxtab16" => "__builtin_arm_sxtab16",
                     "sxtb16" => "__builtin_arm_sxtb16",
-                    "thread.pointer" => "__builtin_thread_pointer",
                     "uadd16" => "__builtin_arm_uadd16",
                     "uadd8" => "__builtin_arm_uadd8",
                     "uasx" => "__builtin_arm_uasx",
@@ -646,16 +609,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
             }
             bpf(name, full_name)
         }
-        "cuda" => {
-            fn cuda(name: &str, full_name: &str) -> &'static str {
-                match name {
-                    // cuda
-                    "syncthreads" => "__syncthreads",
-                    _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),
-                }
-            }
-            cuda(name, full_name)
-        }
         "hexagon" => {
             fn hexagon(name: &str, full_name: &str) -> &'static str {
                 match name {
@@ -959,19 +912,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "F2.dfcmpge" => "__builtin_HEXAGON_F2_dfcmpge",
                     "F2.dfcmpgt" => "__builtin_HEXAGON_F2_dfcmpgt",
                     "F2.dfcmpuo" => "__builtin_HEXAGON_F2_dfcmpuo",
-                    "F2.dffixupd" => "__builtin_HEXAGON_F2_dffixupd",
-                    "F2.dffixupn" => "__builtin_HEXAGON_F2_dffixupn",
-                    "F2.dffixupr" => "__builtin_HEXAGON_F2_dffixupr",
-                    "F2.dffma" => "__builtin_HEXAGON_F2_dffma",
-                    "F2.dffma.lib" => "__builtin_HEXAGON_F2_dffma_lib",
-                    "F2.dffma.sc" => "__builtin_HEXAGON_F2_dffma_sc",
-                    "F2.dffms" => "__builtin_HEXAGON_F2_dffms",
-                    "F2.dffms.lib" => "__builtin_HEXAGON_F2_dffms_lib",
                     "F2.dfimm.n" => "__builtin_HEXAGON_F2_dfimm_n",
                     "F2.dfimm.p" => "__builtin_HEXAGON_F2_dfimm_p",
                     "F2.dfmax" => "__builtin_HEXAGON_F2_dfmax",
                     "F2.dfmin" => "__builtin_HEXAGON_F2_dfmin",
-                    "F2.dfmpy" => "__builtin_HEXAGON_F2_dfmpy",
                     "F2.dfmpyfix" => "__builtin_HEXAGON_F2_dfmpyfix",
                     "F2.dfmpyhh" => "__builtin_HEXAGON_F2_dfmpyhh",
                     "F2.dfmpylh" => "__builtin_HEXAGON_F2_dfmpylh",
@@ -1398,7 +1342,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "S2.asr.r.vw" => "__builtin_HEXAGON_S2_asr_r_vw",
                     "S2.brev" => "__builtin_HEXAGON_S2_brev",
                     "S2.brevp" => "__builtin_HEXAGON_S2_brevp",
-                    "S2.cabacencbin" => "__builtin_HEXAGON_S2_cabacencbin",
                     "S2.cl0" => "__builtin_HEXAGON_S2_cl0",
                     "S2.cl0p" => "__builtin_HEXAGON_S2_cl0p",
                     "S2.cl1" => "__builtin_HEXAGON_S2_cl1",
@@ -1569,7 +1512,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "S6.vsplatrbp" => "__builtin_HEXAGON_S6_vsplatrbp",
                     "S6.vtrunehb.ppp" => "__builtin_HEXAGON_S6_vtrunehb_ppp",
                     "S6.vtrunohb.ppp" => "__builtin_HEXAGON_S6_vtrunohb_ppp",
-                    "SI.to.SXTHI.asrh" => "__builtin_SI_to_SXTHI_asrh",
                     "V6.extractw" => "__builtin_HEXAGON_V6_extractw",
                     "V6.extractw.128B" => "__builtin_HEXAGON_V6_extractw_128B",
                     "V6.get.qfext" => "__builtin_HEXAGON_V6_get_qfext",
@@ -2160,14 +2102,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "V6.vlsrwv.128B" => "__builtin_HEXAGON_V6_vlsrwv_128B",
                     "V6.vlut4" => "__builtin_HEXAGON_V6_vlut4",
                     "V6.vlut4.128B" => "__builtin_HEXAGON_V6_vlut4_128B",
-                    "V6.vlutb" => "__builtin_HEXAGON_V6_vlutb",
-                    "V6.vlutb.128B" => "__builtin_HEXAGON_V6_vlutb_128B",
-                    "V6.vlutb.acc" => "__builtin_HEXAGON_V6_vlutb_acc",
-                    "V6.vlutb.acc.128B" => "__builtin_HEXAGON_V6_vlutb_acc_128B",
-                    "V6.vlutb.dv" => "__builtin_HEXAGON_V6_vlutb_dv",
-                    "V6.vlutb.dv.128B" => "__builtin_HEXAGON_V6_vlutb_dv_128B",
-                    "V6.vlutb.dv.acc" => "__builtin_HEXAGON_V6_vlutb_dv_acc",
-                    "V6.vlutb.dv.acc.128B" => "__builtin_HEXAGON_V6_vlutb_dv_acc_128B",
                     "V6.vlutvvb" => "__builtin_HEXAGON_V6_vlutvvb",
                     "V6.vlutvvb.128B" => "__builtin_HEXAGON_V6_vlutvvb_128B",
                     "V6.vlutvvb.nm" => "__builtin_HEXAGON_V6_vlutvvb_nm",
@@ -2669,17 +2603,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "Y6.dmresume" => "__builtin_HEXAGON_Y6_dmresume",
                     "Y6.dmstart" => "__builtin_HEXAGON_Y6_dmstart",
                     "Y6.dmwait" => "__builtin_HEXAGON_Y6_dmwait",
-                    "brev.ldb" => "__builtin_brev_ldb",
-                    "brev.ldd" => "__builtin_brev_ldd",
-                    "brev.ldh" => "__builtin_brev_ldh",
-                    "brev.ldub" => "__builtin_brev_ldub",
-                    "brev.lduh" => "__builtin_brev_lduh",
-                    "brev.ldw" => "__builtin_brev_ldw",
-                    "brev.stb" => "__builtin_brev_stb",
-                    "brev.std" => "__builtin_brev_std",
-                    "brev.sth" => "__builtin_brev_sth",
-                    "brev.sthhi" => "__builtin_brev_sthhi",
-                    "brev.stw" => "__builtin_brev_stw",
                     "circ.ldb" => "__builtin_circ_ldb",
                     "circ.ldd" => "__builtin_circ_ldd",
                     "circ.ldh" => "__builtin_circ_ldh",
@@ -2691,7 +2614,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "circ.sth" => "__builtin_circ_sth",
                     "circ.sthhi" => "__builtin_circ_sthhi",
                     "circ.stw" => "__builtin_circ_stw",
-                    "mm256i.vaddw" => "__builtin__mm256i_vaddw",
                     "prefetch" => "__builtin_HEXAGON_prefetch",
                     "vmemcpy" => "__builtin_hexagon_vmemcpy",
                     "vmemset" => "__builtin_hexagon_vmemset",
@@ -4910,8 +4832,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
             fn nvvm(name: &str, full_name: &str) -> &'static str {
                 match name {
                     // nvvm
-                    "abs.i" => "__nvvm_abs_i",
-                    "abs.ll" => "__nvvm_abs_ll",
                     "activemask" => "__nvvm_activemask",
                     "add.rm.d" => "__nvvm_add_rm_d",
                     "add.rm.f" => "__nvvm_add_rm_f",
@@ -4925,10 +4845,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "add.rz.d" => "__nvvm_add_rz_d",
                     "add.rz.f" => "__nvvm_add_rz_f",
                     "add.rz.ftz.f" => "__nvvm_add_rz_ftz_f",
-                    "bar.sync" => "__nvvm_bar_sync",
                     "bar.warp.sync" => "__nvvm_bar_warp_sync",
-                    "barrier0" => "__nvvm_bar0",
-                    // [DUPLICATE]: "barrier0" => "__syncthreads",
                     "barrier0.and" => "__nvvm_bar0_and",
                     "barrier0.or" => "__nvvm_bar0_or",
                     "barrier0.popc" => "__nvvm_bar0_popc",
@@ -4938,17 +4855,9 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "bf16x2.to.ue8m0x2.rz.satfinite" => "__nvvm_bf16x2_to_ue8m0x2_rz_satfinite",
                     "bf2h.rn" => "__nvvm_bf2h_rn",
                     "bf2h.rn.ftz" => "__nvvm_bf2h_rn_ftz",
-                    "bitcast.d2ll" => "__nvvm_bitcast_d2ll",
-                    "bitcast.f2i" => "__nvvm_bitcast_f2i",
-                    "bitcast.i2f" => "__nvvm_bitcast_i2f",
-                    "bitcast.ll2d" => "__nvvm_bitcast_ll2d",
-                    "brev32" => "__nvvm_brev32",
-                    "brev64" => "__nvvm_brev64",
                     "ceil.d" => "__nvvm_ceil_d",
                     "ceil.f" => "__nvvm_ceil_f",
                     "ceil.ftz.f" => "__nvvm_ceil_ftz_f",
-                    "clz.i" => "__nvvm_clz_i",
-                    "clz.ll" => "__nvvm_clz_ll",
                     "cos.approx.f" => "__nvvm_cos_approx_f",
                     "cos.approx.ftz.f" => "__nvvm_cos_approx_ftz_f",
                     "cp.async.commit.group" => "__nvvm_cp_async_commit_group",
@@ -5012,9 +4921,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "e4m3x2.to.f16x2.rn.relu" => "__nvvm_e4m3x2_to_f16x2_rn_relu",
                     "e5m2x2.to.f16x2.rn" => "__nvvm_e5m2x2_to_f16x2_rn",
                     "e5m2x2.to.f16x2.rn.relu" => "__nvvm_e5m2x2_to_f16x2_rn_relu",
-                    "ex2.approx.d" => "__nvvm_ex2_approx_d",
-                    "ex2.approx.f" => "__nvvm_ex2_approx_f",
-                    "ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
                     "exit" => "__nvvm_exit",
                     "f16x2.to.e4m3x2.rn" => "__nvvm_f16x2_to_e4m3x2_rn",
                     "f16x2.to.e4m3x2.rn.relu" => "__nvvm_f16x2_to_e4m3x2_rn_relu",
@@ -5100,9 +5006,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                         "__nvvm_f32x4_to_e5m2x4_rs_relu_satfinite"
                     }
                     "f32x4.to.e5m2x4.rs.satfinite" => "__nvvm_f32x4_to_e5m2x4_rs_satfinite",
-                    "fabs.d" => "__nvvm_fabs_d",
-                    "fabs.f" => "__nvvm_fabs_f",
-                    "fabs.ftz.f" => "__nvvm_fabs_ftz_f",
                     "ff.to.e2m1x2.rn.relu.satfinite" => "__nvvm_ff_to_e2m1x2_rn_relu_satfinite",
                     "ff.to.e2m1x2.rn.satfinite" => "__nvvm_ff_to_e2m1x2_rn_satfinite",
                     "ff.to.e2m3x2.rn.relu.satfinite" => "__nvvm_ff_to_e2m3x2_rn_relu_satfinite",
@@ -5219,7 +5122,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "fmin.xorsign.abs.bf16x2" => "__nvvm_fmin_xorsign_abs_bf16x2",
                     "fmin.xorsign.abs.f" => "__nvvm_fmin_xorsign_abs_f",
                     "fns" => "__nvvm_fns",
-                    "h2f" => "__nvvm_h2f",
                     "i2d.rm" => "__nvvm_i2d_rm",
                     "i2d.rn" => "__nvvm_i2d_rn",
                     "i2d.rp" => "__nvvm_i2d_rp",
@@ -5250,10 +5152,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "lohi.i2d" => "__nvvm_lohi_i2d",
                     "match.any.sync.i32" => "__nvvm_match_any_sync_i32",
                     "match.any.sync.i64" => "__nvvm_match_any_sync_i64",
-                    "max.i" => "__nvvm_max_i",
-                    "max.ll" => "__nvvm_max_ll",
-                    "max.ui" => "__nvvm_max_ui",
-                    "max.ull" => "__nvvm_max_ull",
                     "mbarrier.arrive" => "__nvvm_mbarrier_arrive",
                     "mbarrier.arrive.drop" => "__nvvm_mbarrier_arrive_drop",
                     "mbarrier.arrive.drop.noComplete" => "__nvvm_mbarrier_arrive_drop_noComplete",
@@ -5276,10 +5174,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "membar.cta" => "__nvvm_membar_cta",
                     "membar.gl" => "__nvvm_membar_gl",
                     "membar.sys" => "__nvvm_membar_sys",
-                    "min.i" => "__nvvm_min_i",
-                    "min.ll" => "__nvvm_min_ll",
-                    "min.ui" => "__nvvm_min_ui",
-                    "min.ull" => "__nvvm_min_ull",
                     "mul.rm.d" => "__nvvm_mul_rm_d",
                     "mul.rm.f" => "__nvvm_mul_rm_f",
                     "mul.rm.ftz.f" => "__nvvm_mul_rm_ftz_f",
@@ -5304,8 +5198,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "neg.bf16" => "__nvvm_neg_bf16",
                     "neg.bf16x2" => "__nvvm_neg_bf16x2",
                     "pm.event.mask" => "__nvvm_pm_event_mask",
-                    "popc.i" => "__nvvm_popc_i",
-                    "popc.ll" => "__nvvm_popc_ll",
                     "prmt" => "__nvvm_prmt",
                     "rcp.approx.ftz.d" => "__nvvm_rcp_approx_ftz_d",
                     "rcp.approx.ftz.f" => "__nvvm_rcp_approx_ftz_f",
@@ -5322,9 +5214,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "rcp.rz.f" => "__nvvm_rcp_rz_f",
                     "rcp.rz.ftz.f" => "__nvvm_rcp_rz_ftz_f",
                     "read.ptx.sreg.clock" => "__nvvm_read_ptx_sreg_clock",
-                    // [DUPLICATE]: "read.ptx.sreg.clock" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.clock64" => "__nvvm_read_ptx_sreg_clock64",
-                    // [DUPLICATE]: "read.ptx.sreg.clock64" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.ctaid.w" => "__nvvm_read_ptx_sreg_ctaid_w",
                     "read.ptx.sreg.ctaid.x" => "__nvvm_read_ptx_sreg_ctaid_x",
                     "read.ptx.sreg.ctaid.y" => "__nvvm_read_ptx_sreg_ctaid_y",
@@ -5364,49 +5254,33 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "read.ptx.sreg.globaltimer" => "__nvvm_read_ptx_sreg_globaltimer",
                     "read.ptx.sreg.globaltimer.lo" => "__nvvm_read_ptx_sreg_globaltimer_lo",
                     "read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_gridid",
-                    // [DUPLICATE]: "read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.laneid" => "__nvvm_read_ptx_sreg_laneid",
-                    // [DUPLICATE]: "read.ptx.sreg.laneid" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.lanemask.eq" => "__nvvm_read_ptx_sreg_lanemask_eq",
-                    // [DUPLICATE]: "read.ptx.sreg.lanemask.eq" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.lanemask.ge" => "__nvvm_read_ptx_sreg_lanemask_ge",
-                    // [DUPLICATE]: "read.ptx.sreg.lanemask.ge" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.lanemask.gt" => "__nvvm_read_ptx_sreg_lanemask_gt",
-                    // [DUPLICATE]: "read.ptx.sreg.lanemask.gt" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.lanemask.le" => "__nvvm_read_ptx_sreg_lanemask_le",
-                    // [DUPLICATE]: "read.ptx.sreg.lanemask.le" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.lanemask.lt" => "__nvvm_read_ptx_sreg_lanemask_lt",
-                    // [DUPLICATE]: "read.ptx.sreg.lanemask.lt" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.nctaid.w" => "__nvvm_read_ptx_sreg_nctaid_w",
                     "read.ptx.sreg.nctaid.x" => "__nvvm_read_ptx_sreg_nctaid_x",
                     "read.ptx.sreg.nctaid.y" => "__nvvm_read_ptx_sreg_nctaid_y",
                     "read.ptx.sreg.nctaid.z" => "__nvvm_read_ptx_sreg_nctaid_z",
                     "read.ptx.sreg.nsmid" => "__nvvm_read_ptx_sreg_nsmid",
-                    // [DUPLICATE]: "read.ptx.sreg.nsmid" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.ntid.w" => "__nvvm_read_ptx_sreg_ntid_w",
                     "read.ptx.sreg.ntid.x" => "__nvvm_read_ptx_sreg_ntid_x",
                     "read.ptx.sreg.ntid.y" => "__nvvm_read_ptx_sreg_ntid_y",
                     "read.ptx.sreg.ntid.z" => "__nvvm_read_ptx_sreg_ntid_z",
                     "read.ptx.sreg.nwarpid" => "__nvvm_read_ptx_sreg_nwarpid",
-                    // [DUPLICATE]: "read.ptx.sreg.nwarpid" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.pm0" => "__nvvm_read_ptx_sreg_pm0",
-                    // [DUPLICATE]: "read.ptx.sreg.pm0" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.pm1" => "__nvvm_read_ptx_sreg_pm1",
-                    // [DUPLICATE]: "read.ptx.sreg.pm1" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.pm2" => "__nvvm_read_ptx_sreg_pm2",
-                    // [DUPLICATE]: "read.ptx.sreg.pm2" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.pm3" => "__nvvm_read_ptx_sreg_pm3",
-                    // [DUPLICATE]: "read.ptx.sreg.pm3" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.smid" => "__nvvm_read_ptx_sreg_smid",
-                    // [DUPLICATE]: "read.ptx.sreg.smid" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.tid.w" => "__nvvm_read_ptx_sreg_tid_w",
                     "read.ptx.sreg.tid.x" => "__nvvm_read_ptx_sreg_tid_x",
                     "read.ptx.sreg.tid.y" => "__nvvm_read_ptx_sreg_tid_y",
                     "read.ptx.sreg.tid.z" => "__nvvm_read_ptx_sreg_tid_z",
                     "read.ptx.sreg.warpid" => "__nvvm_read_ptx_sreg_warpid",
-                    // [DUPLICATE]: "read.ptx.sreg.warpid" => "__nvvm_read_ptx_sreg_",
                     "read.ptx.sreg.warpsize" => "__nvvm_read_ptx_sreg_warpsize",
-                    // [DUPLICATE]: "read.ptx.sreg.warpsize" => "__nvvm_read_ptx_sreg_",
                     "redux.sync.add" => "__nvvm_redux_sync_add",
                     "redux.sync.and" => "__nvvm_redux_sync_and",
                     "redux.sync.fmax" => "__nvvm_redux_sync_fmax",
@@ -5424,9 +5298,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "redux.sync.umin" => "__nvvm_redux_sync_umin",
                     "redux.sync.xor" => "__nvvm_redux_sync_xor",
                     "reflect" => "__nvvm_reflect",
-                    "rotate.b32" => "__nvvm_rotate_b32",
-                    "rotate.b64" => "__nvvm_rotate_b64",
-                    "rotate.right.b64" => "__nvvm_rotate_right_b64",
                     "round.d" => "__nvvm_round_d",
                     "round.f" => "__nvvm_round_f",
                     "round.ftz.f" => "__nvvm_round_ftz_f",
@@ -5692,7 +5563,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sust.p.3d.v4i16.trap" => "__nvvm_sust_p_3d_v4i16_trap",
                     "sust.p.3d.v4i32.trap" => "__nvvm_sust_p_3d_v4i32_trap",
                     "sust.p.3d.v4i8.trap" => "__nvvm_sust_p_3d_v4i8_trap",
-                    "swap.lo.hi.b64" => "__nvvm_swap_lo_hi_b64",
                     "trunc.d" => "__nvvm_trunc_d",
                     "trunc.f" => "__nvvm_trunc_f",
                     "trunc.ftz.f" => "__nvvm_trunc_ftz_f",
@@ -6132,89 +6002,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "packed2zoned" => "__builtin_ppc_packed2zoned",
                     "pdepd" => "__builtin_pdepd",
                     "pextd" => "__builtin_pextd",
-                    "qpx.qvfabs" => "__builtin_qpx_qvfabs",
-                    "qpx.qvfadd" => "__builtin_qpx_qvfadd",
-                    "qpx.qvfadds" => "__builtin_qpx_qvfadds",
-                    "qpx.qvfcfid" => "__builtin_qpx_qvfcfid",
-                    "qpx.qvfcfids" => "__builtin_qpx_qvfcfids",
-                    "qpx.qvfcfidu" => "__builtin_qpx_qvfcfidu",
-                    "qpx.qvfcfidus" => "__builtin_qpx_qvfcfidus",
-                    "qpx.qvfcmpeq" => "__builtin_qpx_qvfcmpeq",
-                    "qpx.qvfcmpgt" => "__builtin_qpx_qvfcmpgt",
-                    "qpx.qvfcmplt" => "__builtin_qpx_qvfcmplt",
-                    "qpx.qvfcpsgn" => "__builtin_qpx_qvfcpsgn",
-                    "qpx.qvfctid" => "__builtin_qpx_qvfctid",
-                    "qpx.qvfctidu" => "__builtin_qpx_qvfctidu",
-                    "qpx.qvfctiduz" => "__builtin_qpx_qvfctiduz",
-                    "qpx.qvfctidz" => "__builtin_qpx_qvfctidz",
-                    "qpx.qvfctiw" => "__builtin_qpx_qvfctiw",
-                    "qpx.qvfctiwu" => "__builtin_qpx_qvfctiwu",
-                    "qpx.qvfctiwuz" => "__builtin_qpx_qvfctiwuz",
-                    "qpx.qvfctiwz" => "__builtin_qpx_qvfctiwz",
-                    "qpx.qvflogical" => "__builtin_qpx_qvflogical",
-                    "qpx.qvfmadd" => "__builtin_qpx_qvfmadd",
-                    "qpx.qvfmadds" => "__builtin_qpx_qvfmadds",
-                    "qpx.qvfmsub" => "__builtin_qpx_qvfmsub",
-                    "qpx.qvfmsubs" => "__builtin_qpx_qvfmsubs",
-                    "qpx.qvfmul" => "__builtin_qpx_qvfmul",
-                    "qpx.qvfmuls" => "__builtin_qpx_qvfmuls",
-                    "qpx.qvfnabs" => "__builtin_qpx_qvfnabs",
-                    "qpx.qvfneg" => "__builtin_qpx_qvfneg",
-                    "qpx.qvfnmadd" => "__builtin_qpx_qvfnmadd",
-                    "qpx.qvfnmadds" => "__builtin_qpx_qvfnmadds",
-                    "qpx.qvfnmsub" => "__builtin_qpx_qvfnmsub",
-                    "qpx.qvfnmsubs" => "__builtin_qpx_qvfnmsubs",
-                    "qpx.qvfperm" => "__builtin_qpx_qvfperm",
-                    "qpx.qvfre" => "__builtin_qpx_qvfre",
-                    "qpx.qvfres" => "__builtin_qpx_qvfres",
-                    "qpx.qvfrim" => "__builtin_qpx_qvfrim",
-                    "qpx.qvfrin" => "__builtin_qpx_qvfrin",
-                    "qpx.qvfrip" => "__builtin_qpx_qvfrip",
-                    "qpx.qvfriz" => "__builtin_qpx_qvfriz",
-                    "qpx.qvfrsp" => "__builtin_qpx_qvfrsp",
-                    "qpx.qvfrsqrte" => "__builtin_qpx_qvfrsqrte",
-                    "qpx.qvfrsqrtes" => "__builtin_qpx_qvfrsqrtes",
-                    "qpx.qvfsel" => "__builtin_qpx_qvfsel",
-                    "qpx.qvfsub" => "__builtin_qpx_qvfsub",
-                    "qpx.qvfsubs" => "__builtin_qpx_qvfsubs",
-                    "qpx.qvftstnan" => "__builtin_qpx_qvftstnan",
-                    "qpx.qvfxmadd" => "__builtin_qpx_qvfxmadd",
-                    "qpx.qvfxmadds" => "__builtin_qpx_qvfxmadds",
-                    "qpx.qvfxmul" => "__builtin_qpx_qvfxmul",
-                    "qpx.qvfxmuls" => "__builtin_qpx_qvfxmuls",
-                    "qpx.qvfxxcpnmadd" => "__builtin_qpx_qvfxxcpnmadd",
-                    "qpx.qvfxxcpnmadds" => "__builtin_qpx_qvfxxcpnmadds",
-                    "qpx.qvfxxmadd" => "__builtin_qpx_qvfxxmadd",
-                    "qpx.qvfxxmadds" => "__builtin_qpx_qvfxxmadds",
-                    "qpx.qvfxxnpmadd" => "__builtin_qpx_qvfxxnpmadd",
-                    "qpx.qvfxxnpmadds" => "__builtin_qpx_qvfxxnpmadds",
-                    "qpx.qvgpci" => "__builtin_qpx_qvgpci",
-                    "qpx.qvlfcd" => "__builtin_qpx_qvlfcd",
-                    "qpx.qvlfcda" => "__builtin_qpx_qvlfcda",
-                    "qpx.qvlfcs" => "__builtin_qpx_qvlfcs",
-                    "qpx.qvlfcsa" => "__builtin_qpx_qvlfcsa",
-                    "qpx.qvlfd" => "__builtin_qpx_qvlfd",
-                    "qpx.qvlfda" => "__builtin_qpx_qvlfda",
-                    "qpx.qvlfiwa" => "__builtin_qpx_qvlfiwa",
-                    "qpx.qvlfiwaa" => "__builtin_qpx_qvlfiwaa",
-                    "qpx.qvlfiwz" => "__builtin_qpx_qvlfiwz",
-                    "qpx.qvlfiwza" => "__builtin_qpx_qvlfiwza",
-                    "qpx.qvlfs" => "__builtin_qpx_qvlfs",
-                    "qpx.qvlfsa" => "__builtin_qpx_qvlfsa",
-                    "qpx.qvlpcld" => "__builtin_qpx_qvlpcld",
-                    "qpx.qvlpcls" => "__builtin_qpx_qvlpcls",
-                    "qpx.qvlpcrd" => "__builtin_qpx_qvlpcrd",
-                    "qpx.qvlpcrs" => "__builtin_qpx_qvlpcrs",
-                    "qpx.qvstfcd" => "__builtin_qpx_qvstfcd",
-                    "qpx.qvstfcda" => "__builtin_qpx_qvstfcda",
-                    "qpx.qvstfcs" => "__builtin_qpx_qvstfcs",
-                    "qpx.qvstfcsa" => "__builtin_qpx_qvstfcsa",
-                    "qpx.qvstfd" => "__builtin_qpx_qvstfd",
-                    "qpx.qvstfda" => "__builtin_qpx_qvstfda",
-                    "qpx.qvstfiw" => "__builtin_qpx_qvstfiw",
-                    "qpx.qvstfiwa" => "__builtin_qpx_qvstfiwa",
-                    "qpx.qvstfs" => "__builtin_qpx_qvstfs",
-                    "qpx.qvstfsa" => "__builtin_qpx_qvstfsa",
                     "readflm" => "__builtin_readflm",
                     "rlwimi" => "__builtin_ppc_rlwimi",
                     "rlwnm" => "__builtin_ppc_rlwnm",
@@ -6329,33 +6116,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
             }
             ppc(name, full_name)
         }
-        "ptx" => {
-            fn ptx(name: &str, full_name: &str) -> &'static str {
-                match name {
-                    // ptx
-                    "bar.sync" => "__builtin_ptx_bar_sync",
-                    "read.clock" => "__builtin_ptx_read_clock",
-                    "read.clock64" => "__builtin_ptx_read_clock64",
-                    "read.gridid" => "__builtin_ptx_read_gridid",
-                    "read.laneid" => "__builtin_ptx_read_laneid",
-                    "read.lanemask.eq" => "__builtin_ptx_read_lanemask_eq",
-                    "read.lanemask.ge" => "__builtin_ptx_read_lanemask_ge",
-                    "read.lanemask.gt" => "__builtin_ptx_read_lanemask_gt",
-                    "read.lanemask.le" => "__builtin_ptx_read_lanemask_le",
-                    "read.lanemask.lt" => "__builtin_ptx_read_lanemask_lt",
-                    "read.nsmid" => "__builtin_ptx_read_nsmid",
-                    "read.nwarpid" => "__builtin_ptx_read_nwarpid",
-                    "read.pm0" => "__builtin_ptx_read_pm0",
-                    "read.pm1" => "__builtin_ptx_read_pm1",
-                    "read.pm2" => "__builtin_ptx_read_pm2",
-                    "read.pm3" => "__builtin_ptx_read_pm3",
-                    "read.smid" => "__builtin_ptx_read_smid",
-                    "read.warpid" => "__builtin_ptx_read_warpid",
-                    _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),
-                }
-            }
-            ptx(name, full_name)
-        }
         "r600" => {
             fn r600(name: &str, full_name: &str) -> &'static str {
                 match name {
@@ -7902,10 +7662,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "aadd64" => "__builtin_ia32_aadd64",
                     "aand32" => "__builtin_ia32_aand32",
                     "aand64" => "__builtin_ia32_aand64",
-                    "addcarry.u32" => "__builtin_ia32_addcarry_u32",
-                    "addcarry.u64" => "__builtin_ia32_addcarry_u64",
-                    "addcarryx.u32" => "__builtin_ia32_addcarryx_u32",
-                    "addcarryx.u64" => "__builtin_ia32_addcarryx_u64",
                     "aesni.aesdec" => "__builtin_ia32_aesdec128",
                     "aesni.aesdec.256" => "__builtin_ia32_aesdec256",
                     "aesni.aesdec.512" => "__builtin_ia32_aesdec512",
@@ -7924,18 +7680,11 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "aor64" => "__builtin_ia32_aor64",
                     "avx.addsub.pd.256" => "__builtin_ia32_addsubpd256",
                     "avx.addsub.ps.256" => "__builtin_ia32_addsubps256",
-                    "avx.blend.pd.256" => "__builtin_ia32_blendpd256",
-                    "avx.blend.ps.256" => "__builtin_ia32_blendps256",
                     "avx.blendv.pd.256" => "__builtin_ia32_blendvpd256",
                     "avx.blendv.ps.256" => "__builtin_ia32_blendvps256",
-                    "avx.cmp.pd.256" => "__builtin_ia32_cmppd256",
-                    "avx.cmp.ps.256" => "__builtin_ia32_cmpps256",
                     "avx.cvt.pd2.ps.256" => "__builtin_ia32_cvtpd2ps256",
                     "avx.cvt.pd2dq.256" => "__builtin_ia32_cvtpd2dq256",
-                    "avx.cvt.ps2.pd.256" => "__builtin_ia32_cvtps2pd256",
                     "avx.cvt.ps2dq.256" => "__builtin_ia32_cvtps2dq256",
-                    "avx.cvtdq2.pd.256" => "__builtin_ia32_cvtdq2pd256",
-                    "avx.cvtdq2.ps.256" => "__builtin_ia32_cvtdq2ps256",
                     "avx.cvtt.pd2dq.256" => "__builtin_ia32_cvttpd2dq256",
                     "avx.cvtt.ps2dq.256" => "__builtin_ia32_cvttps2dq256",
                     "avx.dp.ps.256" => "__builtin_ia32_dpps256",
@@ -7965,22 +7714,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx.round.pd.256" => "__builtin_ia32_roundpd256",
                     "avx.round.ps.256" => "__builtin_ia32_roundps256",
                     "avx.rsqrt.ps.256" => "__builtin_ia32_rsqrtps256",
-                    "avx.sqrt.pd.256" => "__builtin_ia32_sqrtpd256",
-                    "avx.sqrt.ps.256" => "__builtin_ia32_sqrtps256",
-                    "avx.storeu.dq.256" => "__builtin_ia32_storedqu256",
-                    "avx.storeu.pd.256" => "__builtin_ia32_storeupd256",
-                    "avx.storeu.ps.256" => "__builtin_ia32_storeups256",
-                    "avx.vbroadcastf128.pd.256" => "__builtin_ia32_vbroadcastf128_pd256",
-                    "avx.vbroadcastf128.ps.256" => "__builtin_ia32_vbroadcastf128_ps256",
-                    "avx.vextractf128.pd.256" => "__builtin_ia32_vextractf128_pd256",
-                    "avx.vextractf128.ps.256" => "__builtin_ia32_vextractf128_ps256",
-                    "avx.vextractf128.si.256" => "__builtin_ia32_vextractf128_si256",
-                    "avx.vinsertf128.pd.256" => "__builtin_ia32_vinsertf128_pd256",
-                    "avx.vinsertf128.ps.256" => "__builtin_ia32_vinsertf128_ps256",
-                    "avx.vinsertf128.si.256" => "__builtin_ia32_vinsertf128_si256",
-                    "avx.vperm2f128.pd.256" => "__builtin_ia32_vperm2f128_pd256",
-                    "avx.vperm2f128.ps.256" => "__builtin_ia32_vperm2f128_ps256",
-                    "avx.vperm2f128.si.256" => "__builtin_ia32_vperm2f128_si256",
                     "avx.vpermilvar.pd" => "__builtin_ia32_vpermilvarpd",
                     "avx.vpermilvar.pd.256" => "__builtin_ia32_vpermilvarpd256",
                     "avx.vpermilvar.ps" => "__builtin_ia32_vpermilvarps",
@@ -8265,33 +7998,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx2.maskstore.d.256" => "__builtin_ia32_maskstored256",
                     "avx2.maskstore.q" => "__builtin_ia32_maskstoreq",
                     "avx2.maskstore.q.256" => "__builtin_ia32_maskstoreq256",
-                    "avx2.movntdqa" => "__builtin_ia32_movntdqa256",
                     "avx2.mpsadbw" => "__builtin_ia32_mpsadbw256",
-                    "avx2.pabs.b" => "__builtin_ia32_pabsb256",
-                    "avx2.pabs.d" => "__builtin_ia32_pabsd256",
-                    "avx2.pabs.w" => "__builtin_ia32_pabsw256",
                     "avx2.packssdw" => "__builtin_ia32_packssdw256",
                     "avx2.packsswb" => "__builtin_ia32_packsswb256",
                     "avx2.packusdw" => "__builtin_ia32_packusdw256",
                     "avx2.packuswb" => "__builtin_ia32_packuswb256",
-                    "avx2.padds.b" => "__builtin_ia32_paddsb256",
-                    "avx2.padds.w" => "__builtin_ia32_paddsw256",
-                    "avx2.paddus.b" => "__builtin_ia32_paddusb256",
-                    "avx2.paddus.w" => "__builtin_ia32_paddusw256",
                     "avx2.pavg.b" => "__builtin_ia32_pavgb256",
                     "avx2.pavg.w" => "__builtin_ia32_pavgw256",
-                    "avx2.pblendd.128" => "__builtin_ia32_pblendd128",
-                    "avx2.pblendd.256" => "__builtin_ia32_pblendd256",
                     "avx2.pblendvb" => "__builtin_ia32_pblendvb256",
-                    "avx2.pblendw" => "__builtin_ia32_pblendw256",
-                    "avx2.pbroadcastb.128" => "__builtin_ia32_pbroadcastb128",
-                    "avx2.pbroadcastb.256" => "__builtin_ia32_pbroadcastb256",
-                    "avx2.pbroadcastd.128" => "__builtin_ia32_pbroadcastd128",
-                    "avx2.pbroadcastd.256" => "__builtin_ia32_pbroadcastd256",
-                    "avx2.pbroadcastq.128" => "__builtin_ia32_pbroadcastq128",
-                    "avx2.pbroadcastq.256" => "__builtin_ia32_pbroadcastq256",
-                    "avx2.pbroadcastw.128" => "__builtin_ia32_pbroadcastw128",
-                    "avx2.pbroadcastw.256" => "__builtin_ia32_pbroadcastw256",
                     "avx2.permd" => "__builtin_ia32_permvarsi256",
                     "avx2.permps" => "__builtin_ia32_permvarsf256",
                     "avx2.phadd.d" => "__builtin_ia32_phaddd256",
@@ -8302,44 +8016,16 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx2.phsub.w" => "__builtin_ia32_phsubw256",
                     "avx2.pmadd.ub.sw" => "__builtin_ia32_pmaddubsw256",
                     "avx2.pmadd.wd" => "__builtin_ia32_pmaddwd256",
-                    "avx2.pmaxs.b" => "__builtin_ia32_pmaxsb256",
-                    "avx2.pmaxs.d" => "__builtin_ia32_pmaxsd256",
-                    "avx2.pmaxs.w" => "__builtin_ia32_pmaxsw256",
-                    "avx2.pmaxu.b" => "__builtin_ia32_pmaxub256",
-                    "avx2.pmaxu.d" => "__builtin_ia32_pmaxud256",
-                    "avx2.pmaxu.w" => "__builtin_ia32_pmaxuw256",
-                    "avx2.pmins.b" => "__builtin_ia32_pminsb256",
-                    "avx2.pmins.d" => "__builtin_ia32_pminsd256",
-                    "avx2.pmins.w" => "__builtin_ia32_pminsw256",
-                    "avx2.pminu.b" => "__builtin_ia32_pminub256",
-                    "avx2.pminu.d" => "__builtin_ia32_pminud256",
-                    "avx2.pminu.w" => "__builtin_ia32_pminuw256",
                     "avx2.pmovmskb" => "__builtin_ia32_pmovmskb256",
-                    "avx2.pmovsxbd" => "__builtin_ia32_pmovsxbd256",
-                    "avx2.pmovsxbq" => "__builtin_ia32_pmovsxbq256",
-                    "avx2.pmovsxbw" => "__builtin_ia32_pmovsxbw256",
-                    "avx2.pmovsxdq" => "__builtin_ia32_pmovsxdq256",
-                    "avx2.pmovsxwd" => "__builtin_ia32_pmovsxwd256",
-                    "avx2.pmovsxwq" => "__builtin_ia32_pmovsxwq256",
-                    "avx2.pmovzxbd" => "__builtin_ia32_pmovzxbd256",
-                    "avx2.pmovzxbq" => "__builtin_ia32_pmovzxbq256",
-                    "avx2.pmovzxbw" => "__builtin_ia32_pmovzxbw256",
-                    "avx2.pmovzxdq" => "__builtin_ia32_pmovzxdq256",
-                    "avx2.pmovzxwd" => "__builtin_ia32_pmovzxwd256",
-                    "avx2.pmovzxwq" => "__builtin_ia32_pmovzxwq256",
-                    "avx2.pmul.dq" => "__builtin_ia32_pmuldq256",
                     "avx2.pmul.hr.sw" => "__builtin_ia32_pmulhrsw256",
                     "avx2.pmulh.w" => "__builtin_ia32_pmulhw256",
                     "avx2.pmulhu.w" => "__builtin_ia32_pmulhuw256",
-                    "avx2.pmulu.dq" => "__builtin_ia32_pmuludq256",
                     "avx2.psad.bw" => "__builtin_ia32_psadbw256",
                     "avx2.pshuf.b" => "__builtin_ia32_pshufb256",
                     "avx2.psign.b" => "__builtin_ia32_psignb256",
                     "avx2.psign.d" => "__builtin_ia32_psignd256",
                     "avx2.psign.w" => "__builtin_ia32_psignw256",
                     "avx2.psll.d" => "__builtin_ia32_pslld256",
-                    "avx2.psll.dq" => "__builtin_ia32_pslldqi256",
-                    "avx2.psll.dq.bs" => "__builtin_ia32_pslldqi256_byteshift",
                     "avx2.psll.q" => "__builtin_ia32_psllq256",
                     "avx2.psll.w" => "__builtin_ia32_psllw256",
                     "avx2.pslli.d" => "__builtin_ia32_pslldi256",
@@ -8356,8 +8042,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx2.psrav.d" => "__builtin_ia32_psrav4si",
                     "avx2.psrav.d.256" => "__builtin_ia32_psrav8si",
                     "avx2.psrl.d" => "__builtin_ia32_psrld256",
-                    "avx2.psrl.dq" => "__builtin_ia32_psrldqi256",
-                    "avx2.psrl.dq.bs" => "__builtin_ia32_psrldqi256_byteshift",
                     "avx2.psrl.q" => "__builtin_ia32_psrlq256",
                     "avx2.psrl.w" => "__builtin_ia32_psrlw256",
                     "avx2.psrli.d" => "__builtin_ia32_psrldi256",
@@ -8367,15 +8051,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx2.psrlv.d.256" => "__builtin_ia32_psrlv8si",
                     "avx2.psrlv.q" => "__builtin_ia32_psrlv2di",
                     "avx2.psrlv.q.256" => "__builtin_ia32_psrlv4di",
-                    "avx2.psubs.b" => "__builtin_ia32_psubsb256",
-                    "avx2.psubs.w" => "__builtin_ia32_psubsw256",
-                    "avx2.psubus.b" => "__builtin_ia32_psubusb256",
-                    "avx2.psubus.w" => "__builtin_ia32_psubusw256",
-                    "avx2.vbroadcast.sd.pd.256" => "__builtin_ia32_vbroadcastsd_pd256",
-                    "avx2.vbroadcast.ss.ps" => "__builtin_ia32_vbroadcastss_ps",
-                    "avx2.vbroadcast.ss.ps.256" => "__builtin_ia32_vbroadcastss_ps256",
-                    "avx2.vextracti128" => "__builtin_ia32_extract128i256",
-                    "avx2.vinserti128" => "__builtin_ia32_insert128i256",
                     "avx2.vpdpbssd.128" => "__builtin_ia32_vpdpbssd128",
                     "avx2.vpdpbssd.256" => "__builtin_ia32_vpdpbssd256",
                     "avx2.vpdpbssds.128" => "__builtin_ia32_vpdpbssds128",
@@ -8400,7 +8075,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx2.vpdpwuud.256" => "__builtin_ia32_vpdpwuud256",
                     "avx2.vpdpwuuds.128" => "__builtin_ia32_vpdpwuuds128",
                     "avx2.vpdpwuuds.256" => "__builtin_ia32_vpdpwuuds256",
-                    "avx2.vperm2i128" => "__builtin_ia32_permti256",
                     "avx512.add.pd.512" => "__builtin_ia32_addpd512",
                     "avx512.add.ps.512" => "__builtin_ia32_addps512",
                     "avx512.broadcastmb.128" => "__builtin_ia32_broadcastmb128",
@@ -8415,194 +8089,32 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.conflict.q.128" => "__builtin_ia32_vpconflictdi_128",
                     "avx512.conflict.q.256" => "__builtin_ia32_vpconflictdi_256",
                     "avx512.conflict.q.512" => "__builtin_ia32_vpconflictdi_512",
-                    "avx512.cvtb2mask.128" => "__builtin_ia32_cvtb2mask128",
-                    "avx512.cvtb2mask.256" => "__builtin_ia32_cvtb2mask256",
-                    "avx512.cvtb2mask.512" => "__builtin_ia32_cvtb2mask512",
-                    "avx512.cvtd2mask.128" => "__builtin_ia32_cvtd2mask128",
-                    "avx512.cvtd2mask.256" => "__builtin_ia32_cvtd2mask256",
-                    "avx512.cvtd2mask.512" => "__builtin_ia32_cvtd2mask512",
-                    "avx512.cvtmask2b.128" => "__builtin_ia32_cvtmask2b128",
-                    "avx512.cvtmask2b.256" => "__builtin_ia32_cvtmask2b256",
-                    "avx512.cvtmask2b.512" => "__builtin_ia32_cvtmask2b512",
-                    "avx512.cvtmask2d.128" => "__builtin_ia32_cvtmask2d128",
-                    "avx512.cvtmask2d.256" => "__builtin_ia32_cvtmask2d256",
-                    "avx512.cvtmask2d.512" => "__builtin_ia32_cvtmask2d512",
-                    "avx512.cvtmask2q.128" => "__builtin_ia32_cvtmask2q128",
-                    "avx512.cvtmask2q.256" => "__builtin_ia32_cvtmask2q256",
-                    "avx512.cvtmask2q.512" => "__builtin_ia32_cvtmask2q512",
-                    "avx512.cvtmask2w.128" => "__builtin_ia32_cvtmask2w128",
-                    "avx512.cvtmask2w.256" => "__builtin_ia32_cvtmask2w256",
-                    "avx512.cvtmask2w.512" => "__builtin_ia32_cvtmask2w512",
-                    "avx512.cvtq2mask.128" => "__builtin_ia32_cvtq2mask128",
-                    "avx512.cvtq2mask.256" => "__builtin_ia32_cvtq2mask256",
-                    "avx512.cvtq2mask.512" => "__builtin_ia32_cvtq2mask512",
-                    "avx512.cvtsd2usi" => "__builtin_ia32_cvtsd2usi",
-                    "avx512.cvtsd2usi64" => "__builtin_ia32_cvtsd2usi64",
-                    "avx512.cvtsi2sd32" => "__builtin_ia32_cvtsi2sd32",
                     "avx512.cvtsi2sd64" => "__builtin_ia32_cvtsi2sd64",
                     "avx512.cvtsi2ss32" => "__builtin_ia32_cvtsi2ss32",
                     "avx512.cvtsi2ss64" => "__builtin_ia32_cvtsi2ss64",
-                    "avx512.cvtss2usi" => "__builtin_ia32_cvtss2usi",
-                    "avx512.cvtss2usi64" => "__builtin_ia32_cvtss2usi64",
                     "avx512.cvttsd2si" => "__builtin_ia32_vcvttsd2si32",
                     "avx512.cvttsd2si64" => "__builtin_ia32_vcvttsd2si64",
                     "avx512.cvttsd2usi" => "__builtin_ia32_vcvttsd2usi32",
-                    // [DUPLICATE]: "avx512.cvttsd2usi" => "__builtin_ia32_cvttsd2usi",
                     "avx512.cvttsd2usi64" => "__builtin_ia32_vcvttsd2usi64",
-                    // [DUPLICATE]: "avx512.cvttsd2usi64" => "__builtin_ia32_cvttsd2usi64",
                     "avx512.cvttss2si" => "__builtin_ia32_vcvttss2si32",
                     "avx512.cvttss2si64" => "__builtin_ia32_vcvttss2si64",
                     "avx512.cvttss2usi" => "__builtin_ia32_vcvttss2usi32",
-                    // [DUPLICATE]: "avx512.cvttss2usi" => "__builtin_ia32_cvttss2usi",
                     "avx512.cvttss2usi64" => "__builtin_ia32_vcvttss2usi64",
-                    // [DUPLICATE]: "avx512.cvttss2usi64" => "__builtin_ia32_cvttss2usi64",
-                    "avx512.cvtusi2sd" => "__builtin_ia32_cvtusi2sd",
-                    // [DUPLICATE]: "avx512.cvtusi2sd" => "__builtin_ia32_cvtusi2sd32",
                     "avx512.cvtusi2ss" => "__builtin_ia32_cvtusi2ss32",
-                    // [DUPLICATE]: "avx512.cvtusi2ss" => "__builtin_ia32_cvtusi2ss",
                     "avx512.cvtusi642sd" => "__builtin_ia32_cvtusi2sd64",
-                    // [DUPLICATE]: "avx512.cvtusi642sd" => "__builtin_ia32_cvtusi642sd",
                     "avx512.cvtusi642ss" => "__builtin_ia32_cvtusi2ss64",
-                    // [DUPLICATE]: "avx512.cvtusi642ss" => "__builtin_ia32_cvtusi642ss",
-                    "avx512.cvtw2mask.128" => "__builtin_ia32_cvtw2mask128",
-                    "avx512.cvtw2mask.256" => "__builtin_ia32_cvtw2mask256",
-                    "avx512.cvtw2mask.512" => "__builtin_ia32_cvtw2mask512",
                     "avx512.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128",
                     "avx512.dbpsadbw.256" => "__builtin_ia32_dbpsadbw256",
                     "avx512.dbpsadbw.512" => "__builtin_ia32_dbpsadbw512",
                     "avx512.div.pd.512" => "__builtin_ia32_divpd512",
                     "avx512.div.ps.512" => "__builtin_ia32_divps512",
-                    "avx512.exp2.pd" => "__builtin_ia32_exp2pd_mask",
-                    "avx512.exp2.ps" => "__builtin_ia32_exp2ps_mask",
-                    "avx512.gather.dpd.512" => "__builtin_ia32_gathersiv8df",
-                    "avx512.gather.dpi.512" => "__builtin_ia32_gathersiv16si",
-                    "avx512.gather.dpq.512" => "__builtin_ia32_gathersiv8di",
-                    "avx512.gather.dps.512" => "__builtin_ia32_gathersiv16sf",
-                    "avx512.gather.qpd.512" => "__builtin_ia32_gatherdiv8df",
-                    "avx512.gather.qpi.512" => "__builtin_ia32_gatherdiv16si",
-                    "avx512.gather.qpq.512" => "__builtin_ia32_gatherdiv8di",
-                    "avx512.gather.qps.512" => "__builtin_ia32_gatherdiv16sf",
-                    "avx512.gather3div2.df" => "__builtin_ia32_gather3div2df",
-                    "avx512.gather3div2.di" => "__builtin_ia32_gather3div2di",
-                    "avx512.gather3div4.df" => "__builtin_ia32_gather3div4df",
-                    "avx512.gather3div4.di" => "__builtin_ia32_gather3div4di",
-                    "avx512.gather3div4.sf" => "__builtin_ia32_gather3div4sf",
-                    "avx512.gather3div4.si" => "__builtin_ia32_gather3div4si",
-                    "avx512.gather3div8.sf" => "__builtin_ia32_gather3div8sf",
-                    "avx512.gather3div8.si" => "__builtin_ia32_gather3div8si",
-                    "avx512.gather3siv2.df" => "__builtin_ia32_gather3siv2df",
-                    "avx512.gather3siv2.di" => "__builtin_ia32_gather3siv2di",
-                    "avx512.gather3siv4.df" => "__builtin_ia32_gather3siv4df",
-                    "avx512.gather3siv4.di" => "__builtin_ia32_gather3siv4di",
-                    "avx512.gather3siv4.sf" => "__builtin_ia32_gather3siv4sf",
-                    "avx512.gather3siv4.si" => "__builtin_ia32_gather3siv4si",
-                    "avx512.gather3siv8.sf" => "__builtin_ia32_gather3siv8sf",
-                    "avx512.gather3siv8.si" => "__builtin_ia32_gather3siv8si",
-                    "avx512.gatherpf.dpd.512" => "__builtin_ia32_gatherpfdpd",
-                    "avx512.gatherpf.dps.512" => "__builtin_ia32_gatherpfdps",
-                    "avx512.gatherpf.qpd.512" => "__builtin_ia32_gatherpfqpd",
-                    "avx512.gatherpf.qps.512" => "__builtin_ia32_gatherpfqps",
-                    "avx512.kand.w" => "__builtin_ia32_kandhi",
-                    "avx512.kandn.w" => "__builtin_ia32_kandnhi",
-                    "avx512.knot.w" => "__builtin_ia32_knothi",
-                    "avx512.kor.w" => "__builtin_ia32_korhi",
-                    "avx512.kortestc.w" => "__builtin_ia32_kortestchi",
-                    "avx512.kortestz.w" => "__builtin_ia32_kortestzhi",
-                    "avx512.kunpck.bw" => "__builtin_ia32_kunpckhi",
-                    "avx512.kunpck.dq" => "__builtin_ia32_kunpckdi",
-                    "avx512.kunpck.wd" => "__builtin_ia32_kunpcksi",
-                    "avx512.kxnor.w" => "__builtin_ia32_kxnorhi",
-                    "avx512.kxor.w" => "__builtin_ia32_kxorhi",
-                    "avx512.mask.add.pd.128" => "__builtin_ia32_addpd128_mask",
-                    "avx512.mask.add.pd.256" => "__builtin_ia32_addpd256_mask",
-                    "avx512.mask.add.pd.512" => "__builtin_ia32_addpd512_mask",
-                    "avx512.mask.add.ps.128" => "__builtin_ia32_addps128_mask",
-                    "avx512.mask.add.ps.256" => "__builtin_ia32_addps256_mask",
-                    "avx512.mask.add.ps.512" => "__builtin_ia32_addps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.add.sd.round" => "__builtin_ia32_addsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.add.ss.round" => "__builtin_ia32_addss_round_mask",
-                    "avx512.mask.and.pd.128" => "__builtin_ia32_andpd128_mask",
-                    "avx512.mask.and.pd.256" => "__builtin_ia32_andpd256_mask",
-                    "avx512.mask.and.pd.512" => "__builtin_ia32_andpd512_mask",
-                    "avx512.mask.and.ps.128" => "__builtin_ia32_andps128_mask",
-                    "avx512.mask.and.ps.256" => "__builtin_ia32_andps256_mask",
-                    "avx512.mask.and.ps.512" => "__builtin_ia32_andps512_mask",
-                    "avx512.mask.andn.pd.128" => "__builtin_ia32_andnpd128_mask",
-                    "avx512.mask.andn.pd.256" => "__builtin_ia32_andnpd256_mask",
-                    "avx512.mask.andn.pd.512" => "__builtin_ia32_andnpd512_mask",
-                    "avx512.mask.andn.ps.128" => "__builtin_ia32_andnps128_mask",
-                    "avx512.mask.andn.ps.256" => "__builtin_ia32_andnps256_mask",
-                    "avx512.mask.andn.ps.512" => "__builtin_ia32_andnps512_mask",
-                    "avx512.mask.blend.d.512" => "__builtin_ia32_blendmd_512_mask",
-                    "avx512.mask.blend.pd.512" => "__builtin_ia32_blendmpd_512_mask",
-                    "avx512.mask.blend.ps.512" => "__builtin_ia32_blendmps_512_mask",
-                    "avx512.mask.blend.q.512" => "__builtin_ia32_blendmq_512_mask",
-                    "avx512.mask.broadcastf32x2.256" => "__builtin_ia32_broadcastf32x2_256_mask",
-                    "avx512.mask.broadcastf32x2.512" => "__builtin_ia32_broadcastf32x2_512_mask",
-                    "avx512.mask.broadcastf32x4.256" => "__builtin_ia32_broadcastf32x4_256_mask",
-                    "avx512.mask.broadcastf32x4.512" => "__builtin_ia32_broadcastf32x4_512",
-                    "avx512.mask.broadcastf32x8.512" => "__builtin_ia32_broadcastf32x8_512_mask",
-                    "avx512.mask.broadcastf64x2.256" => "__builtin_ia32_broadcastf64x2_256_mask",
-                    "avx512.mask.broadcastf64x2.512" => "__builtin_ia32_broadcastf64x2_512_mask",
-                    "avx512.mask.broadcastf64x4.512" => "__builtin_ia32_broadcastf64x4_512",
-                    "avx512.mask.broadcasti32x2.128" => "__builtin_ia32_broadcasti32x2_128_mask",
-                    "avx512.mask.broadcasti32x2.256" => "__builtin_ia32_broadcasti32x2_256_mask",
-                    "avx512.mask.broadcasti32x2.512" => "__builtin_ia32_broadcasti32x2_512_mask",
-                    "avx512.mask.broadcasti32x4.256" => "__builtin_ia32_broadcasti32x4_256_mask",
-                    "avx512.mask.broadcasti32x4.512" => "__builtin_ia32_broadcasti32x4_512",
-                    "avx512.mask.broadcasti32x8.512" => "__builtin_ia32_broadcasti32x8_512_mask",
-                    "avx512.mask.broadcasti64x2.256" => "__builtin_ia32_broadcasti64x2_256_mask",
-                    "avx512.mask.broadcasti64x2.512" => "__builtin_ia32_broadcasti64x2_512_mask",
-                    "avx512.mask.broadcasti64x4.512" => "__builtin_ia32_broadcasti64x4_512",
-                    "avx512.mask.cmp.pd.128" => "__builtin_ia32_cmppd128_mask",
-                    "avx512.mask.cmp.pd.256" => "__builtin_ia32_cmppd256_mask",
-                    "avx512.mask.cmp.pd.512" => "__builtin_ia32_cmppd512_mask",
-                    "avx512.mask.cmp.ps.128" => "__builtin_ia32_cmpps128_mask",
-                    "avx512.mask.cmp.ps.256" => "__builtin_ia32_cmpps256_mask",
-                    "avx512.mask.cmp.ps.512" => "__builtin_ia32_cmpps512_mask",
                     "avx512.mask.cmp.sd" => "__builtin_ia32_cmpsd_mask",
                     "avx512.mask.cmp.ss" => "__builtin_ia32_cmpss_mask",
-                    "avx512.mask.compress.d.128" => "__builtin_ia32_compresssi128_mask",
-                    "avx512.mask.compress.d.256" => "__builtin_ia32_compresssi256_mask",
-                    "avx512.mask.compress.d.512" => "__builtin_ia32_compresssi512_mask",
-                    "avx512.mask.compress.pd.128" => "__builtin_ia32_compressdf128_mask",
-                    "avx512.mask.compress.pd.256" => "__builtin_ia32_compressdf256_mask",
-                    "avx512.mask.compress.pd.512" => "__builtin_ia32_compressdf512_mask",
-                    "avx512.mask.compress.ps.128" => "__builtin_ia32_compresssf128_mask",
-                    "avx512.mask.compress.ps.256" => "__builtin_ia32_compresssf256_mask",
-                    "avx512.mask.compress.ps.512" => "__builtin_ia32_compresssf512_mask",
-                    "avx512.mask.compress.q.128" => "__builtin_ia32_compressdi128_mask",
-                    "avx512.mask.compress.q.256" => "__builtin_ia32_compressdi256_mask",
-                    "avx512.mask.compress.q.512" => "__builtin_ia32_compressdi512_mask",
-                    "avx512.mask.compress.store.d.128" => "__builtin_ia32_compressstoresi128_mask",
-                    "avx512.mask.compress.store.d.256" => "__builtin_ia32_compressstoresi256_mask",
-                    "avx512.mask.compress.store.d.512" => "__builtin_ia32_compressstoresi512_mask",
-                    "avx512.mask.compress.store.pd.128" => "__builtin_ia32_compressstoredf128_mask",
-                    "avx512.mask.compress.store.pd.256" => "__builtin_ia32_compressstoredf256_mask",
-                    "avx512.mask.compress.store.pd.512" => "__builtin_ia32_compressstoredf512_mask",
-                    "avx512.mask.compress.store.ps.128" => "__builtin_ia32_compressstoresf128_mask",
-                    "avx512.mask.compress.store.ps.256" => "__builtin_ia32_compressstoresf256_mask",
-                    "avx512.mask.compress.store.ps.512" => "__builtin_ia32_compressstoresf512_mask",
-                    "avx512.mask.compress.store.q.128" => "__builtin_ia32_compressstoredi128_mask",
-                    "avx512.mask.compress.store.q.256" => "__builtin_ia32_compressstoredi256_mask",
-                    "avx512.mask.compress.store.q.512" => "__builtin_ia32_compressstoredi512_mask",
-                    "avx512.mask.conflict.d.128" => "__builtin_ia32_vpconflictsi_128_mask",
-                    "avx512.mask.conflict.d.256" => "__builtin_ia32_vpconflictsi_256_mask",
-                    "avx512.mask.conflict.d.512" => "__builtin_ia32_vpconflictsi_512_mask",
-                    "avx512.mask.conflict.q.128" => "__builtin_ia32_vpconflictdi_128_mask",
-                    "avx512.mask.conflict.q.256" => "__builtin_ia32_vpconflictdi_256_mask",
-                    "avx512.mask.conflict.q.512" => "__builtin_ia32_vpconflictdi_512_mask",
-                    "avx512.mask.cvtdq2pd.128" => "__builtin_ia32_cvtdq2pd128_mask",
-                    "avx512.mask.cvtdq2pd.256" => "__builtin_ia32_cvtdq2pd256_mask",
-                    "avx512.mask.cvtdq2pd.512" => "__builtin_ia32_cvtdq2pd512_mask",
-                    "avx512.mask.cvtdq2ps.128" => "__builtin_ia32_cvtdq2ps128_mask",
-                    "avx512.mask.cvtdq2ps.256" => "__builtin_ia32_cvtdq2ps256_mask",
-                    "avx512.mask.cvtdq2ps.512" => "__builtin_ia32_cvtdq2ps512_mask",
                     "avx512.mask.cvtpd2dq.128" => "__builtin_ia32_cvtpd2dq128_mask",
-                    "avx512.mask.cvtpd2dq.256" => "__builtin_ia32_cvtpd2dq256_mask",
                     "avx512.mask.cvtpd2dq.512" => "__builtin_ia32_cvtpd2dq512_mask",
                     "avx512.mask.cvtpd2ps" => "__builtin_ia32_cvtpd2ps_mask",
-                    "avx512.mask.cvtpd2ps.256" => "__builtin_ia32_cvtpd2ps256_mask",
                     "avx512.mask.cvtpd2ps.512" => "__builtin_ia32_cvtpd2ps512_mask",
                     "avx512.mask.cvtpd2qq.128" => "__builtin_ia32_cvtpd2qq128_mask",
                     "avx512.mask.cvtpd2qq.256" => "__builtin_ia32_cvtpd2qq256_mask",
@@ -8616,8 +8128,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.cvtps2dq.128" => "__builtin_ia32_cvtps2dq128_mask",
                     "avx512.mask.cvtps2dq.256" => "__builtin_ia32_cvtps2dq256_mask",
                     "avx512.mask.cvtps2dq.512" => "__builtin_ia32_cvtps2dq512_mask",
-                    "avx512.mask.cvtps2pd.128" => "__builtin_ia32_cvtps2pd128_mask",
-                    "avx512.mask.cvtps2pd.256" => "__builtin_ia32_cvtps2pd256_mask",
                     "avx512.mask.cvtps2pd.512" => "__builtin_ia32_cvtps2pd512_mask",
                     "avx512.mask.cvtps2qq.128" => "__builtin_ia32_cvtps2qq128_mask",
                     "avx512.mask.cvtps2qq.256" => "__builtin_ia32_cvtps2qq256_mask",
@@ -8628,16 +8138,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.cvtps2uqq.128" => "__builtin_ia32_cvtps2uqq128_mask",
                     "avx512.mask.cvtps2uqq.256" => "__builtin_ia32_cvtps2uqq256_mask",
                     "avx512.mask.cvtps2uqq.512" => "__builtin_ia32_cvtps2uqq512_mask",
-                    "avx512.mask.cvtqq2pd.128" => "__builtin_ia32_cvtqq2pd128_mask",
-                    "avx512.mask.cvtqq2pd.256" => "__builtin_ia32_cvtqq2pd256_mask",
-                    "avx512.mask.cvtqq2pd.512" => "__builtin_ia32_cvtqq2pd512_mask",
                     "avx512.mask.cvtqq2ps.128" => "__builtin_ia32_cvtqq2ps128_mask",
-                    "avx512.mask.cvtqq2ps.256" => "__builtin_ia32_cvtqq2ps256_mask",
-                    "avx512.mask.cvtqq2ps.512" => "__builtin_ia32_cvtqq2ps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.cvtsd2ss.round" => "__builtin_ia32_cvtsd2ss_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.cvtss2sd.round" => "__builtin_ia32_cvtss2sd_round_mask",
                     "avx512.mask.cvttpd2dq.128" => "__builtin_ia32_cvttpd2dq128_mask",
-                    "avx512.mask.cvttpd2dq.256" => "__builtin_ia32_cvttpd2dq256_mask",
                     "avx512.mask.cvttpd2dq.512" => "__builtin_ia32_cvttpd2dq512_mask",
                     "avx512.mask.cvttpd2qq.128" => "__builtin_ia32_cvttpd2qq128_mask",
                     "avx512.mask.cvttpd2qq.256" => "__builtin_ia32_cvttpd2qq256_mask",
@@ -8648,8 +8152,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.cvttpd2uqq.128" => "__builtin_ia32_cvttpd2uqq128_mask",
                     "avx512.mask.cvttpd2uqq.256" => "__builtin_ia32_cvttpd2uqq256_mask",
                     "avx512.mask.cvttpd2uqq.512" => "__builtin_ia32_cvttpd2uqq512_mask",
-                    "avx512.mask.cvttps2dq.128" => "__builtin_ia32_cvttps2dq128_mask",
-                    "avx512.mask.cvttps2dq.256" => "__builtin_ia32_cvttps2dq256_mask",
                     "avx512.mask.cvttps2dq.512" => "__builtin_ia32_cvttps2dq512_mask",
                     "avx512.mask.cvttps2qq.128" => "__builtin_ia32_cvttps2qq128_mask",
                     "avx512.mask.cvttps2qq.256" => "__builtin_ia32_cvttps2qq256_mask",
@@ -8660,53 +8162,9 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.cvttps2uqq.128" => "__builtin_ia32_cvttps2uqq128_mask",
                     "avx512.mask.cvttps2uqq.256" => "__builtin_ia32_cvttps2uqq256_mask",
                     "avx512.mask.cvttps2uqq.512" => "__builtin_ia32_cvttps2uqq512_mask",
-                    "avx512.mask.cvtudq2pd.128" => "__builtin_ia32_cvtudq2pd128_mask",
-                    "avx512.mask.cvtudq2pd.256" => "__builtin_ia32_cvtudq2pd256_mask",
-                    "avx512.mask.cvtudq2pd.512" => "__builtin_ia32_cvtudq2pd512_mask",
-                    "avx512.mask.cvtudq2ps.128" => "__builtin_ia32_cvtudq2ps128_mask",
-                    "avx512.mask.cvtudq2ps.256" => "__builtin_ia32_cvtudq2ps256_mask",
-                    "avx512.mask.cvtudq2ps.512" => "__builtin_ia32_cvtudq2ps512_mask",
-                    "avx512.mask.cvtuqq2pd.128" => "__builtin_ia32_cvtuqq2pd128_mask",
-                    "avx512.mask.cvtuqq2pd.256" => "__builtin_ia32_cvtuqq2pd256_mask",
-                    "avx512.mask.cvtuqq2pd.512" => "__builtin_ia32_cvtuqq2pd512_mask",
                     "avx512.mask.cvtuqq2ps.128" => "__builtin_ia32_cvtuqq2ps128_mask",
-                    "avx512.mask.cvtuqq2ps.256" => "__builtin_ia32_cvtuqq2ps256_mask",
-                    "avx512.mask.cvtuqq2ps.512" => "__builtin_ia32_cvtuqq2ps512_mask",
-                    "avx512.mask.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128_mask",
-                    "avx512.mask.dbpsadbw.256" => "__builtin_ia32_dbpsadbw256_mask",
-                    "avx512.mask.dbpsadbw.512" => "__builtin_ia32_dbpsadbw512_mask",
-                    "avx512.mask.div.pd.128" => "__builtin_ia32_divpd_mask",
-                    "avx512.mask.div.pd.256" => "__builtin_ia32_divpd256_mask",
-                    "avx512.mask.div.pd.512" => "__builtin_ia32_divpd512_mask",
-                    "avx512.mask.div.ps.128" => "__builtin_ia32_divps_mask",
-                    "avx512.mask.div.ps.256" => "__builtin_ia32_divps256_mask",
-                    "avx512.mask.div.ps.512" => "__builtin_ia32_divps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.div.sd.round" => "__builtin_ia32_divsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.div.ss.round" => "__builtin_ia32_divss_round_mask",
-                    "avx512.mask.expand.d.128" => "__builtin_ia32_expandsi128_mask",
-                    "avx512.mask.expand.d.256" => "__builtin_ia32_expandsi256_mask",
-                    "avx512.mask.expand.d.512" => "__builtin_ia32_expandsi512_mask",
-                    "avx512.mask.expand.load.d.128" => "__builtin_ia32_expandloadsi128_mask",
-                    "avx512.mask.expand.load.d.256" => "__builtin_ia32_expandloadsi256_mask",
-                    "avx512.mask.expand.load.d.512" => "__builtin_ia32_expandloadsi512_mask",
-                    "avx512.mask.expand.load.pd.128" => "__builtin_ia32_expandloaddf128_mask",
-                    "avx512.mask.expand.load.pd.256" => "__builtin_ia32_expandloaddf256_mask",
-                    "avx512.mask.expand.load.pd.512" => "__builtin_ia32_expandloaddf512_mask",
-                    "avx512.mask.expand.load.ps.128" => "__builtin_ia32_expandloadsf128_mask",
-                    "avx512.mask.expand.load.ps.256" => "__builtin_ia32_expandloadsf256_mask",
-                    "avx512.mask.expand.load.ps.512" => "__builtin_ia32_expandloadsf512_mask",
-                    "avx512.mask.expand.load.q.128" => "__builtin_ia32_expandloaddi128_mask",
-                    "avx512.mask.expand.load.q.256" => "__builtin_ia32_expandloaddi256_mask",
-                    "avx512.mask.expand.load.q.512" => "__builtin_ia32_expandloaddi512_mask",
-                    "avx512.mask.expand.pd.128" => "__builtin_ia32_expanddf128_mask",
-                    "avx512.mask.expand.pd.256" => "__builtin_ia32_expanddf256_mask",
-                    "avx512.mask.expand.pd.512" => "__builtin_ia32_expanddf512_mask",
-                    "avx512.mask.expand.ps.128" => "__builtin_ia32_expandsf128_mask",
-                    "avx512.mask.expand.ps.256" => "__builtin_ia32_expandsf256_mask",
-                    "avx512.mask.expand.ps.512" => "__builtin_ia32_expandsf512_mask",
-                    "avx512.mask.expand.q.128" => "__builtin_ia32_expanddi128_mask",
-                    "avx512.mask.expand.q.256" => "__builtin_ia32_expanddi256_mask",
-                    "avx512.mask.expand.q.512" => "__builtin_ia32_expanddi512_mask",
                     "avx512.mask.fixupimm.pd.128" => "__builtin_ia32_fixupimmpd128_mask",
                     "avx512.mask.fixupimm.pd.256" => "__builtin_ia32_fixupimmpd256_mask",
                     "avx512.mask.fixupimm.pd.512" => "__builtin_ia32_fixupimmpd512_mask",
@@ -8715,12 +8173,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.fixupimm.ps.512" => "__builtin_ia32_fixupimmps512_mask",
                     "avx512.mask.fixupimm.sd" => "__builtin_ia32_fixupimmsd_mask",
                     "avx512.mask.fixupimm.ss" => "__builtin_ia32_fixupimmss_mask",
-                    "avx512.mask.fpclass.pd.128" => "__builtin_ia32_fpclasspd128_mask",
-                    "avx512.mask.fpclass.pd.256" => "__builtin_ia32_fpclasspd256_mask",
-                    "avx512.mask.fpclass.pd.512" => "__builtin_ia32_fpclasspd512_mask",
-                    "avx512.mask.fpclass.ps.128" => "__builtin_ia32_fpclassps128_mask",
-                    "avx512.mask.fpclass.ps.256" => "__builtin_ia32_fpclassps256_mask",
-                    "avx512.mask.fpclass.ps.512" => "__builtin_ia32_fpclassps512_mask",
                     "avx512.mask.fpclass.sd" => "__builtin_ia32_fpclasssd_mask",
                     "avx512.mask.fpclass.ss" => "__builtin_ia32_fpclassss_mask",
                     "avx512.mask.getexp.pd.128" => "__builtin_ia32_getexppd128_mask",
@@ -8739,226 +8191,19 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.getmant.ps.512" => "__builtin_ia32_getmantps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.getmant.sd" => "__builtin_ia32_getmantsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.getmant.ss" => "__builtin_ia32_getmantss_round_mask",
-                    "avx512.mask.insertf32x4.256" => "__builtin_ia32_insertf32x4_256_mask",
-                    "avx512.mask.insertf32x4.512" => "__builtin_ia32_insertf32x4_mask",
-                    "avx512.mask.insertf32x8.512" => "__builtin_ia32_insertf32x8_mask",
-                    "avx512.mask.insertf64x2.256" => "__builtin_ia32_insertf64x2_256_mask",
-                    "avx512.mask.insertf64x2.512" => "__builtin_ia32_insertf64x2_512_mask",
-                    "avx512.mask.insertf64x4.512" => "__builtin_ia32_insertf64x4_mask",
-                    "avx512.mask.inserti32x4.256" => "__builtin_ia32_inserti32x4_256_mask",
-                    "avx512.mask.inserti32x4.512" => "__builtin_ia32_inserti32x4_mask",
-                    "avx512.mask.inserti32x8.512" => "__builtin_ia32_inserti32x8_mask",
-                    "avx512.mask.inserti64x2.256" => "__builtin_ia32_inserti64x2_256_mask",
-                    "avx512.mask.inserti64x2.512" => "__builtin_ia32_inserti64x2_512_mask",
-                    "avx512.mask.inserti64x4.512" => "__builtin_ia32_inserti64x4_mask",
-                    "avx512.mask.loadu.d.512" => "__builtin_ia32_loaddqusi512_mask",
-                    "avx512.mask.loadu.pd.512" => "__builtin_ia32_loadupd512_mask",
-                    "avx512.mask.loadu.ps.512" => "__builtin_ia32_loadups512_mask",
-                    "avx512.mask.loadu.q.512" => "__builtin_ia32_loaddqudi512_mask",
-                    "avx512.mask.lzcnt.d.512" => "__builtin_ia32_vplzcntd_512_mask",
-                    "avx512.mask.lzcnt.q.512" => "__builtin_ia32_vplzcntq_512_mask",
-                    "avx512.mask.max.pd.128" => "__builtin_ia32_maxpd_mask",
-                    "avx512.mask.max.pd.256" => "__builtin_ia32_maxpd256_mask",
-                    "avx512.mask.max.pd.512" => "__builtin_ia32_maxpd512_mask",
-                    "avx512.mask.max.ps.128" => "__builtin_ia32_maxps_mask",
-                    "avx512.mask.max.ps.256" => "__builtin_ia32_maxps256_mask",
-                    "avx512.mask.max.ps.512" => "__builtin_ia32_maxps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.max.sd.round" => "__builtin_ia32_maxsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.max.ss.round" => "__builtin_ia32_maxss_round_mask",
-                    "avx512.mask.min.pd.128" => "__builtin_ia32_minpd_mask",
-                    "avx512.mask.min.pd.256" => "__builtin_ia32_minpd256_mask",
-                    "avx512.mask.min.pd.512" => "__builtin_ia32_minpd512_mask",
-                    "avx512.mask.min.ps.128" => "__builtin_ia32_minps_mask",
-                    "avx512.mask.min.ps.256" => "__builtin_ia32_minps256_mask",
-                    "avx512.mask.min.ps.512" => "__builtin_ia32_minps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.min.sd.round" => "__builtin_ia32_minsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.min.ss.round" => "__builtin_ia32_minss_round_mask",
-                    "avx512.mask.move.sd" => "__builtin_ia32_movsd_mask",
-                    "avx512.mask.move.ss" => "__builtin_ia32_movss_mask",
-                    "avx512.mask.mul.pd.128" => "__builtin_ia32_mulpd_mask",
-                    "avx512.mask.mul.pd.256" => "__builtin_ia32_mulpd256_mask",
-                    "avx512.mask.mul.pd.512" => "__builtin_ia32_mulpd512_mask",
-                    "avx512.mask.mul.ps.128" => "__builtin_ia32_mulps_mask",
-                    "avx512.mask.mul.ps.256" => "__builtin_ia32_mulps256_mask",
-                    "avx512.mask.mul.ps.512" => "__builtin_ia32_mulps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.mul.sd.round" => "__builtin_ia32_mulsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.mul.ss.round" => "__builtin_ia32_mulss_round_mask",
-                    "avx512.mask.or.pd.128" => "__builtin_ia32_orpd128_mask",
-                    "avx512.mask.or.pd.256" => "__builtin_ia32_orpd256_mask",
-                    "avx512.mask.or.pd.512" => "__builtin_ia32_orpd512_mask",
-                    "avx512.mask.or.ps.128" => "__builtin_ia32_orps128_mask",
-                    "avx512.mask.or.ps.256" => "__builtin_ia32_orps256_mask",
-                    "avx512.mask.or.ps.512" => "__builtin_ia32_orps512_mask",
-                    "avx512.mask.pabs.b.128" => "__builtin_ia32_pabsb128_mask",
-                    "avx512.mask.pabs.b.256" => "__builtin_ia32_pabsb256_mask",
-                    "avx512.mask.pabs.b.512" => "__builtin_ia32_pabsb512_mask",
-                    "avx512.mask.pabs.d.128" => "__builtin_ia32_pabsd128_mask",
-                    "avx512.mask.pabs.d.256" => "__builtin_ia32_pabsd256_mask",
-                    "avx512.mask.pabs.d.512" => "__builtin_ia32_pabsd512_mask",
-                    "avx512.mask.pabs.q.128" => "__builtin_ia32_pabsq128_mask",
-                    "avx512.mask.pabs.q.256" => "__builtin_ia32_pabsq256_mask",
-                    "avx512.mask.pabs.q.512" => "__builtin_ia32_pabsq512_mask",
-                    "avx512.mask.pabs.w.128" => "__builtin_ia32_pabsw128_mask",
-                    "avx512.mask.pabs.w.256" => "__builtin_ia32_pabsw256_mask",
-                    "avx512.mask.pabs.w.512" => "__builtin_ia32_pabsw512_mask",
-                    "avx512.mask.packssdw.128" => "__builtin_ia32_packssdw128_mask",
-                    "avx512.mask.packssdw.256" => "__builtin_ia32_packssdw256_mask",
-                    "avx512.mask.packssdw.512" => "__builtin_ia32_packssdw512_mask",
-                    "avx512.mask.packsswb.128" => "__builtin_ia32_packsswb128_mask",
-                    "avx512.mask.packsswb.256" => "__builtin_ia32_packsswb256_mask",
-                    "avx512.mask.packsswb.512" => "__builtin_ia32_packsswb512_mask",
-                    "avx512.mask.packusdw.128" => "__builtin_ia32_packusdw128_mask",
-                    "avx512.mask.packusdw.256" => "__builtin_ia32_packusdw256_mask",
-                    "avx512.mask.packusdw.512" => "__builtin_ia32_packusdw512_mask",
-                    "avx512.mask.packuswb.128" => "__builtin_ia32_packuswb128_mask",
-                    "avx512.mask.packuswb.256" => "__builtin_ia32_packuswb256_mask",
-                    "avx512.mask.packuswb.512" => "__builtin_ia32_packuswb512_mask",
-                    "avx512.mask.padd.b.128" => "__builtin_ia32_paddb128_mask",
-                    "avx512.mask.padd.b.256" => "__builtin_ia32_paddb256_mask",
-                    "avx512.mask.padd.b.512" => "__builtin_ia32_paddb512_mask",
-                    "avx512.mask.padd.d.128" => "__builtin_ia32_paddd128_mask",
-                    "avx512.mask.padd.d.256" => "__builtin_ia32_paddd256_mask",
-                    "avx512.mask.padd.d.512" => "__builtin_ia32_paddd512_mask",
-                    "avx512.mask.padd.q.128" => "__builtin_ia32_paddq128_mask",
-                    "avx512.mask.padd.q.256" => "__builtin_ia32_paddq256_mask",
-                    "avx512.mask.padd.q.512" => "__builtin_ia32_paddq512_mask",
-                    "avx512.mask.padd.w.128" => "__builtin_ia32_paddw128_mask",
-                    "avx512.mask.padd.w.256" => "__builtin_ia32_paddw256_mask",
-                    "avx512.mask.padd.w.512" => "__builtin_ia32_paddw512_mask",
-                    "avx512.mask.padds.b.128" => "__builtin_ia32_paddsb128_mask",
-                    "avx512.mask.padds.b.256" => "__builtin_ia32_paddsb256_mask",
-                    "avx512.mask.padds.b.512" => "__builtin_ia32_paddsb512_mask",
-                    "avx512.mask.padds.w.128" => "__builtin_ia32_paddsw128_mask",
-                    "avx512.mask.padds.w.256" => "__builtin_ia32_paddsw256_mask",
-                    "avx512.mask.padds.w.512" => "__builtin_ia32_paddsw512_mask",
-                    "avx512.mask.paddus.b.128" => "__builtin_ia32_paddusb128_mask",
-                    "avx512.mask.paddus.b.256" => "__builtin_ia32_paddusb256_mask",
-                    "avx512.mask.paddus.b.512" => "__builtin_ia32_paddusb512_mask",
-                    "avx512.mask.paddus.w.128" => "__builtin_ia32_paddusw128_mask",
-                    "avx512.mask.paddus.w.256" => "__builtin_ia32_paddusw256_mask",
-                    "avx512.mask.paddus.w.512" => "__builtin_ia32_paddusw512_mask",
-                    "avx512.mask.pand.d.512" => "__builtin_ia32_pandd512_mask",
-                    "avx512.mask.pand.q.512" => "__builtin_ia32_pandq512_mask",
-                    "avx512.mask.pavg.b.128" => "__builtin_ia32_pavgb128_mask",
-                    "avx512.mask.pavg.b.256" => "__builtin_ia32_pavgb256_mask",
-                    "avx512.mask.pavg.b.512" => "__builtin_ia32_pavgb512_mask",
-                    "avx512.mask.pavg.w.128" => "__builtin_ia32_pavgw128_mask",
-                    "avx512.mask.pavg.w.256" => "__builtin_ia32_pavgw256_mask",
-                    "avx512.mask.pavg.w.512" => "__builtin_ia32_pavgw512_mask",
-                    "avx512.mask.pbroadcast.b.gpr.128" => "__builtin_ia32_pbroadcastb128_gpr_mask",
-                    "avx512.mask.pbroadcast.b.gpr.256" => "__builtin_ia32_pbroadcastb256_gpr_mask",
-                    "avx512.mask.pbroadcast.b.gpr.512" => "__builtin_ia32_pbroadcastb512_gpr_mask",
-                    "avx512.mask.pbroadcast.d.gpr.128" => "__builtin_ia32_pbroadcastd128_gpr_mask",
-                    "avx512.mask.pbroadcast.d.gpr.256" => "__builtin_ia32_pbroadcastd256_gpr_mask",
-                    "avx512.mask.pbroadcast.d.gpr.512" => "__builtin_ia32_pbroadcastd512_gpr_mask",
-                    "avx512.mask.pbroadcast.q.gpr.128" => "__builtin_ia32_pbroadcastq128_gpr_mask",
-                    "avx512.mask.pbroadcast.q.gpr.256" => "__builtin_ia32_pbroadcastq256_gpr_mask",
-                    "avx512.mask.pbroadcast.q.gpr.512" => "__builtin_ia32_pbroadcastq512_gpr_mask",
-                    "avx512.mask.pbroadcast.q.mem.512" => "__builtin_ia32_pbroadcastq512_mem_mask",
-                    "avx512.mask.pbroadcast.w.gpr.128" => "__builtin_ia32_pbroadcastw128_gpr_mask",
-                    "avx512.mask.pbroadcast.w.gpr.256" => "__builtin_ia32_pbroadcastw256_gpr_mask",
-                    "avx512.mask.pbroadcast.w.gpr.512" => "__builtin_ia32_pbroadcastw512_gpr_mask",
-                    "avx512.mask.pcmpeq.b.128" => "__builtin_ia32_pcmpeqb128_mask",
-                    "avx512.mask.pcmpeq.b.256" => "__builtin_ia32_pcmpeqb256_mask",
-                    "avx512.mask.pcmpeq.b.512" => "__builtin_ia32_pcmpeqb512_mask",
-                    "avx512.mask.pcmpeq.d.128" => "__builtin_ia32_pcmpeqd128_mask",
-                    "avx512.mask.pcmpeq.d.256" => "__builtin_ia32_pcmpeqd256_mask",
-                    "avx512.mask.pcmpeq.d.512" => "__builtin_ia32_pcmpeqd512_mask",
-                    "avx512.mask.pcmpeq.q.128" => "__builtin_ia32_pcmpeqq128_mask",
-                    "avx512.mask.pcmpeq.q.256" => "__builtin_ia32_pcmpeqq256_mask",
-                    "avx512.mask.pcmpeq.q.512" => "__builtin_ia32_pcmpeqq512_mask",
-                    "avx512.mask.pcmpeq.w.128" => "__builtin_ia32_pcmpeqw128_mask",
-                    "avx512.mask.pcmpeq.w.256" => "__builtin_ia32_pcmpeqw256_mask",
-                    "avx512.mask.pcmpeq.w.512" => "__builtin_ia32_pcmpeqw512_mask",
-                    "avx512.mask.pcmpgt.b.128" => "__builtin_ia32_pcmpgtb128_mask",
-                    "avx512.mask.pcmpgt.b.256" => "__builtin_ia32_pcmpgtb256_mask",
-                    "avx512.mask.pcmpgt.b.512" => "__builtin_ia32_pcmpgtb512_mask",
-                    "avx512.mask.pcmpgt.d.128" => "__builtin_ia32_pcmpgtd128_mask",
-                    "avx512.mask.pcmpgt.d.256" => "__builtin_ia32_pcmpgtd256_mask",
-                    "avx512.mask.pcmpgt.d.512" => "__builtin_ia32_pcmpgtd512_mask",
-                    "avx512.mask.pcmpgt.q.128" => "__builtin_ia32_pcmpgtq128_mask",
-                    "avx512.mask.pcmpgt.q.256" => "__builtin_ia32_pcmpgtq256_mask",
-                    "avx512.mask.pcmpgt.q.512" => "__builtin_ia32_pcmpgtq512_mask",
-                    "avx512.mask.pcmpgt.w.128" => "__builtin_ia32_pcmpgtw128_mask",
-                    "avx512.mask.pcmpgt.w.256" => "__builtin_ia32_pcmpgtw256_mask",
-                    "avx512.mask.pcmpgt.w.512" => "__builtin_ia32_pcmpgtw512_mask",
-                    "avx512.mask.permvar.df.256" => "__builtin_ia32_permvardf256_mask",
-                    "avx512.mask.permvar.df.512" => "__builtin_ia32_permvardf512_mask",
-                    "avx512.mask.permvar.di.256" => "__builtin_ia32_permvardi256_mask",
-                    "avx512.mask.permvar.di.512" => "__builtin_ia32_permvardi512_mask",
-                    "avx512.mask.permvar.hi.128" => "__builtin_ia32_permvarhi128_mask",
-                    "avx512.mask.permvar.hi.256" => "__builtin_ia32_permvarhi256_mask",
-                    "avx512.mask.permvar.hi.512" => "__builtin_ia32_permvarhi512_mask",
-                    "avx512.mask.permvar.qi.128" => "__builtin_ia32_permvarqi128_mask",
-                    "avx512.mask.permvar.qi.256" => "__builtin_ia32_permvarqi256_mask",
-                    "avx512.mask.permvar.qi.512" => "__builtin_ia32_permvarqi512_mask",
-                    "avx512.mask.permvar.sf.256" => "__builtin_ia32_permvarsf256_mask",
-                    "avx512.mask.permvar.sf.512" => "__builtin_ia32_permvarsf512_mask",
-                    "avx512.mask.permvar.si.256" => "__builtin_ia32_permvarsi256_mask",
-                    "avx512.mask.permvar.si.512" => "__builtin_ia32_permvarsi512_mask",
-                    "avx512.mask.pmaddubs.w.128" => "__builtin_ia32_pmaddubsw128_mask",
-                    "avx512.mask.pmaddubs.w.256" => "__builtin_ia32_pmaddubsw256_mask",
-                    "avx512.mask.pmaddubs.w.512" => "__builtin_ia32_pmaddubsw512_mask",
-                    "avx512.mask.pmaddw.d.128" => "__builtin_ia32_pmaddwd128_mask",
-                    "avx512.mask.pmaddw.d.256" => "__builtin_ia32_pmaddwd256_mask",
-                    "avx512.mask.pmaddw.d.512" => "__builtin_ia32_pmaddwd512_mask",
-                    "avx512.mask.pmaxs.b.128" => "__builtin_ia32_pmaxsb128_mask",
-                    "avx512.mask.pmaxs.b.256" => "__builtin_ia32_pmaxsb256_mask",
-                    "avx512.mask.pmaxs.b.512" => "__builtin_ia32_pmaxsb512_mask",
-                    "avx512.mask.pmaxs.d.128" => "__builtin_ia32_pmaxsd128_mask",
-                    "avx512.mask.pmaxs.d.256" => "__builtin_ia32_pmaxsd256_mask",
-                    "avx512.mask.pmaxs.d.512" => "__builtin_ia32_pmaxsd512_mask",
-                    "avx512.mask.pmaxs.q.128" => "__builtin_ia32_pmaxsq128_mask",
-                    "avx512.mask.pmaxs.q.256" => "__builtin_ia32_pmaxsq256_mask",
-                    "avx512.mask.pmaxs.q.512" => "__builtin_ia32_pmaxsq512_mask",
-                    "avx512.mask.pmaxs.w.128" => "__builtin_ia32_pmaxsw128_mask",
-                    "avx512.mask.pmaxs.w.256" => "__builtin_ia32_pmaxsw256_mask",
-                    "avx512.mask.pmaxs.w.512" => "__builtin_ia32_pmaxsw512_mask",
-                    "avx512.mask.pmaxu.b.128" => "__builtin_ia32_pmaxub128_mask",
-                    "avx512.mask.pmaxu.b.256" => "__builtin_ia32_pmaxub256_mask",
-                    "avx512.mask.pmaxu.b.512" => "__builtin_ia32_pmaxub512_mask",
-                    "avx512.mask.pmaxu.d.128" => "__builtin_ia32_pmaxud128_mask",
-                    "avx512.mask.pmaxu.d.256" => "__builtin_ia32_pmaxud256_mask",
-                    "avx512.mask.pmaxu.d.512" => "__builtin_ia32_pmaxud512_mask",
-                    "avx512.mask.pmaxu.q.128" => "__builtin_ia32_pmaxuq128_mask",
-                    "avx512.mask.pmaxu.q.256" => "__builtin_ia32_pmaxuq256_mask",
-                    "avx512.mask.pmaxu.q.512" => "__builtin_ia32_pmaxuq512_mask",
-                    "avx512.mask.pmaxu.w.128" => "__builtin_ia32_pmaxuw128_mask",
-                    "avx512.mask.pmaxu.w.256" => "__builtin_ia32_pmaxuw256_mask",
-                    "avx512.mask.pmaxu.w.512" => "__builtin_ia32_pmaxuw512_mask",
-                    "avx512.mask.pmins.b.128" => "__builtin_ia32_pminsb128_mask",
-                    "avx512.mask.pmins.b.256" => "__builtin_ia32_pminsb256_mask",
-                    "avx512.mask.pmins.b.512" => "__builtin_ia32_pminsb512_mask",
-                    "avx512.mask.pmins.d.128" => "__builtin_ia32_pminsd128_mask",
-                    "avx512.mask.pmins.d.256" => "__builtin_ia32_pminsd256_mask",
-                    "avx512.mask.pmins.d.512" => "__builtin_ia32_pminsd512_mask",
-                    "avx512.mask.pmins.q.128" => "__builtin_ia32_pminsq128_mask",
-                    "avx512.mask.pmins.q.256" => "__builtin_ia32_pminsq256_mask",
-                    "avx512.mask.pmins.q.512" => "__builtin_ia32_pminsq512_mask",
-                    "avx512.mask.pmins.w.128" => "__builtin_ia32_pminsw128_mask",
-                    "avx512.mask.pmins.w.256" => "__builtin_ia32_pminsw256_mask",
-                    "avx512.mask.pmins.w.512" => "__builtin_ia32_pminsw512_mask",
-                    "avx512.mask.pminu.b.128" => "__builtin_ia32_pminub128_mask",
-                    "avx512.mask.pminu.b.256" => "__builtin_ia32_pminub256_mask",
-                    "avx512.mask.pminu.b.512" => "__builtin_ia32_pminub512_mask",
-                    "avx512.mask.pminu.d.128" => "__builtin_ia32_pminud128_mask",
-                    "avx512.mask.pminu.d.256" => "__builtin_ia32_pminud256_mask",
-                    "avx512.mask.pminu.d.512" => "__builtin_ia32_pminud512_mask",
-                    "avx512.mask.pminu.q.128" => "__builtin_ia32_pminuq128_mask",
-                    "avx512.mask.pminu.q.256" => "__builtin_ia32_pminuq256_mask",
-                    "avx512.mask.pminu.q.512" => "__builtin_ia32_pminuq512_mask",
-                    "avx512.mask.pminu.w.128" => "__builtin_ia32_pminuw128_mask",
-                    "avx512.mask.pminu.w.256" => "__builtin_ia32_pminuw256_mask",
-                    "avx512.mask.pminu.w.512" => "__builtin_ia32_pminuw512_mask",
                     "avx512.mask.pmov.db.128" => "__builtin_ia32_pmovdb128_mask",
                     "avx512.mask.pmov.db.256" => "__builtin_ia32_pmovdb256_mask",
-                    "avx512.mask.pmov.db.512" => "__builtin_ia32_pmovdb512_mask",
                     "avx512.mask.pmov.db.mem.128" => "__builtin_ia32_pmovdb128mem_mask",
                     "avx512.mask.pmov.db.mem.256" => "__builtin_ia32_pmovdb256mem_mask",
                     "avx512.mask.pmov.db.mem.512" => "__builtin_ia32_pmovdb512mem_mask",
                     "avx512.mask.pmov.dw.128" => "__builtin_ia32_pmovdw128_mask",
                     "avx512.mask.pmov.dw.256" => "__builtin_ia32_pmovdw256_mask",
-                    "avx512.mask.pmov.dw.512" => "__builtin_ia32_pmovdw512_mask",
                     "avx512.mask.pmov.dw.mem.128" => "__builtin_ia32_pmovdw128mem_mask",
                     "avx512.mask.pmov.dw.mem.256" => "__builtin_ia32_pmovdw256mem_mask",
                     "avx512.mask.pmov.dw.mem.512" => "__builtin_ia32_pmovdw512mem_mask",
@@ -8969,20 +8214,15 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.pmov.qb.mem.256" => "__builtin_ia32_pmovqb256mem_mask",
                     "avx512.mask.pmov.qb.mem.512" => "__builtin_ia32_pmovqb512mem_mask",
                     "avx512.mask.pmov.qd.128" => "__builtin_ia32_pmovqd128_mask",
-                    "avx512.mask.pmov.qd.256" => "__builtin_ia32_pmovqd256_mask",
-                    "avx512.mask.pmov.qd.512" => "__builtin_ia32_pmovqd512_mask",
                     "avx512.mask.pmov.qd.mem.128" => "__builtin_ia32_pmovqd128mem_mask",
                     "avx512.mask.pmov.qd.mem.256" => "__builtin_ia32_pmovqd256mem_mask",
                     "avx512.mask.pmov.qd.mem.512" => "__builtin_ia32_pmovqd512mem_mask",
                     "avx512.mask.pmov.qw.128" => "__builtin_ia32_pmovqw128_mask",
                     "avx512.mask.pmov.qw.256" => "__builtin_ia32_pmovqw256_mask",
-                    "avx512.mask.pmov.qw.512" => "__builtin_ia32_pmovqw512_mask",
                     "avx512.mask.pmov.qw.mem.128" => "__builtin_ia32_pmovqw128mem_mask",
                     "avx512.mask.pmov.qw.mem.256" => "__builtin_ia32_pmovqw256mem_mask",
                     "avx512.mask.pmov.qw.mem.512" => "__builtin_ia32_pmovqw512mem_mask",
                     "avx512.mask.pmov.wb.128" => "__builtin_ia32_pmovwb128_mask",
-                    "avx512.mask.pmov.wb.256" => "__builtin_ia32_pmovwb256_mask",
-                    "avx512.mask.pmov.wb.512" => "__builtin_ia32_pmovwb512_mask",
                     "avx512.mask.pmov.wb.mem.128" => "__builtin_ia32_pmovwb128mem_mask",
                     "avx512.mask.pmov.wb.mem.256" => "__builtin_ia32_pmovwb256mem_mask",
                     "avx512.mask.pmov.wb.mem.512" => "__builtin_ia32_pmovwb512mem_mask",
@@ -9022,24 +8262,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.pmovs.wb.mem.128" => "__builtin_ia32_pmovswb128mem_mask",
                     "avx512.mask.pmovs.wb.mem.256" => "__builtin_ia32_pmovswb256mem_mask",
                     "avx512.mask.pmovs.wb.mem.512" => "__builtin_ia32_pmovswb512mem_mask",
-                    "avx512.mask.pmovsxb.d.128" => "__builtin_ia32_pmovsxbd128_mask",
-                    "avx512.mask.pmovsxb.d.256" => "__builtin_ia32_pmovsxbd256_mask",
-                    "avx512.mask.pmovsxb.d.512" => "__builtin_ia32_pmovsxbd512_mask",
-                    "avx512.mask.pmovsxb.q.128" => "__builtin_ia32_pmovsxbq128_mask",
-                    "avx512.mask.pmovsxb.q.256" => "__builtin_ia32_pmovsxbq256_mask",
-                    "avx512.mask.pmovsxb.q.512" => "__builtin_ia32_pmovsxbq512_mask",
-                    "avx512.mask.pmovsxb.w.128" => "__builtin_ia32_pmovsxbw128_mask",
-                    "avx512.mask.pmovsxb.w.256" => "__builtin_ia32_pmovsxbw256_mask",
-                    "avx512.mask.pmovsxb.w.512" => "__builtin_ia32_pmovsxbw512_mask",
-                    "avx512.mask.pmovsxd.q.128" => "__builtin_ia32_pmovsxdq128_mask",
-                    "avx512.mask.pmovsxd.q.256" => "__builtin_ia32_pmovsxdq256_mask",
-                    "avx512.mask.pmovsxd.q.512" => "__builtin_ia32_pmovsxdq512_mask",
-                    "avx512.mask.pmovsxw.d.128" => "__builtin_ia32_pmovsxwd128_mask",
-                    "avx512.mask.pmovsxw.d.256" => "__builtin_ia32_pmovsxwd256_mask",
-                    "avx512.mask.pmovsxw.d.512" => "__builtin_ia32_pmovsxwd512_mask",
-                    "avx512.mask.pmovsxw.q.128" => "__builtin_ia32_pmovsxwq128_mask",
-                    "avx512.mask.pmovsxw.q.256" => "__builtin_ia32_pmovsxwq256_mask",
-                    "avx512.mask.pmovsxw.q.512" => "__builtin_ia32_pmovsxwq512_mask",
                     "avx512.mask.pmovus.db.128" => "__builtin_ia32_pmovusdb128_mask",
                     "avx512.mask.pmovus.db.256" => "__builtin_ia32_pmovusdb256_mask",
                     "avx512.mask.pmovus.db.512" => "__builtin_ia32_pmovusdb512_mask",
@@ -9076,191 +8298,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.pmovus.wb.mem.128" => "__builtin_ia32_pmovuswb128mem_mask",
                     "avx512.mask.pmovus.wb.mem.256" => "__builtin_ia32_pmovuswb256mem_mask",
                     "avx512.mask.pmovus.wb.mem.512" => "__builtin_ia32_pmovuswb512mem_mask",
-                    "avx512.mask.pmovzxb.d.128" => "__builtin_ia32_pmovzxbd128_mask",
-                    "avx512.mask.pmovzxb.d.256" => "__builtin_ia32_pmovzxbd256_mask",
-                    "avx512.mask.pmovzxb.d.512" => "__builtin_ia32_pmovzxbd512_mask",
-                    "avx512.mask.pmovzxb.q.128" => "__builtin_ia32_pmovzxbq128_mask",
-                    "avx512.mask.pmovzxb.q.256" => "__builtin_ia32_pmovzxbq256_mask",
-                    "avx512.mask.pmovzxb.q.512" => "__builtin_ia32_pmovzxbq512_mask",
-                    "avx512.mask.pmovzxb.w.128" => "__builtin_ia32_pmovzxbw128_mask",
-                    "avx512.mask.pmovzxb.w.256" => "__builtin_ia32_pmovzxbw256_mask",
-                    "avx512.mask.pmovzxb.w.512" => "__builtin_ia32_pmovzxbw512_mask",
-                    "avx512.mask.pmovzxd.q.128" => "__builtin_ia32_pmovzxdq128_mask",
-                    "avx512.mask.pmovzxd.q.256" => "__builtin_ia32_pmovzxdq256_mask",
-                    "avx512.mask.pmovzxd.q.512" => "__builtin_ia32_pmovzxdq512_mask",
-                    "avx512.mask.pmovzxw.d.128" => "__builtin_ia32_pmovzxwd128_mask",
-                    "avx512.mask.pmovzxw.d.256" => "__builtin_ia32_pmovzxwd256_mask",
-                    "avx512.mask.pmovzxw.d.512" => "__builtin_ia32_pmovzxwd512_mask",
-                    "avx512.mask.pmovzxw.q.128" => "__builtin_ia32_pmovzxwq128_mask",
-                    "avx512.mask.pmovzxw.q.256" => "__builtin_ia32_pmovzxwq256_mask",
-                    "avx512.mask.pmovzxw.q.512" => "__builtin_ia32_pmovzxwq512_mask",
-                    "avx512.mask.pmul.dq.128" => "__builtin_ia32_pmuldq128_mask",
-                    "avx512.mask.pmul.dq.256" => "__builtin_ia32_pmuldq256_mask",
-                    "avx512.mask.pmul.dq.512" => "__builtin_ia32_pmuldq512_mask",
-                    "avx512.mask.pmul.hr.sw.128" => "__builtin_ia32_pmulhrsw128_mask",
-                    "avx512.mask.pmul.hr.sw.256" => "__builtin_ia32_pmulhrsw256_mask",
-                    "avx512.mask.pmul.hr.sw.512" => "__builtin_ia32_pmulhrsw512_mask",
-                    "avx512.mask.pmulh.w.128" => "__builtin_ia32_pmulhw128_mask",
-                    "avx512.mask.pmulh.w.256" => "__builtin_ia32_pmulhw256_mask",
-                    "avx512.mask.pmulh.w.512" => "__builtin_ia32_pmulhw512_mask",
-                    "avx512.mask.pmulhu.w.128" => "__builtin_ia32_pmulhuw128_mask",
-                    "avx512.mask.pmulhu.w.256" => "__builtin_ia32_pmulhuw256_mask",
-                    "avx512.mask.pmulhu.w.512" => "__builtin_ia32_pmulhuw512_mask",
-                    "avx512.mask.pmull.d.128" => "__builtin_ia32_pmulld128_mask",
-                    "avx512.mask.pmull.d.256" => "__builtin_ia32_pmulld256_mask",
-                    "avx512.mask.pmull.d.512" => "__builtin_ia32_pmulld512_mask",
-                    "avx512.mask.pmull.q.128" => "__builtin_ia32_pmullq128_mask",
-                    "avx512.mask.pmull.q.256" => "__builtin_ia32_pmullq256_mask",
-                    "avx512.mask.pmull.q.512" => "__builtin_ia32_pmullq512_mask",
-                    "avx512.mask.pmull.w.128" => "__builtin_ia32_pmullw128_mask",
-                    "avx512.mask.pmull.w.256" => "__builtin_ia32_pmullw256_mask",
-                    "avx512.mask.pmull.w.512" => "__builtin_ia32_pmullw512_mask",
-                    "avx512.mask.pmultishift.qb.128" => "__builtin_ia32_vpmultishiftqb128_mask",
-                    "avx512.mask.pmultishift.qb.256" => "__builtin_ia32_vpmultishiftqb256_mask",
-                    "avx512.mask.pmultishift.qb.512" => "__builtin_ia32_vpmultishiftqb512_mask",
-                    "avx512.mask.pmulu.dq.128" => "__builtin_ia32_pmuludq128_mask",
-                    "avx512.mask.pmulu.dq.256" => "__builtin_ia32_pmuludq256_mask",
-                    "avx512.mask.pmulu.dq.512" => "__builtin_ia32_pmuludq512_mask",
-                    "avx512.mask.prol.d.128" => "__builtin_ia32_prold128_mask",
-                    "avx512.mask.prol.d.256" => "__builtin_ia32_prold256_mask",
-                    "avx512.mask.prol.d.512" => "__builtin_ia32_prold512_mask",
-                    "avx512.mask.prol.q.128" => "__builtin_ia32_prolq128_mask",
-                    "avx512.mask.prol.q.256" => "__builtin_ia32_prolq256_mask",
-                    "avx512.mask.prol.q.512" => "__builtin_ia32_prolq512_mask",
-                    "avx512.mask.prolv.d.128" => "__builtin_ia32_prolvd128_mask",
-                    "avx512.mask.prolv.d.256" => "__builtin_ia32_prolvd256_mask",
-                    "avx512.mask.prolv.d.512" => "__builtin_ia32_prolvd512_mask",
-                    "avx512.mask.prolv.q.128" => "__builtin_ia32_prolvq128_mask",
-                    "avx512.mask.prolv.q.256" => "__builtin_ia32_prolvq256_mask",
-                    "avx512.mask.prolv.q.512" => "__builtin_ia32_prolvq512_mask",
-                    "avx512.mask.pror.d.128" => "__builtin_ia32_prord128_mask",
-                    "avx512.mask.pror.d.256" => "__builtin_ia32_prord256_mask",
-                    "avx512.mask.pror.d.512" => "__builtin_ia32_prord512_mask",
-                    "avx512.mask.pror.q.128" => "__builtin_ia32_prorq128_mask",
-                    "avx512.mask.pror.q.256" => "__builtin_ia32_prorq256_mask",
-                    "avx512.mask.pror.q.512" => "__builtin_ia32_prorq512_mask",
-                    "avx512.mask.prorv.d.128" => "__builtin_ia32_prorvd128_mask",
-                    "avx512.mask.prorv.d.256" => "__builtin_ia32_prorvd256_mask",
-                    "avx512.mask.prorv.d.512" => "__builtin_ia32_prorvd512_mask",
-                    "avx512.mask.prorv.q.128" => "__builtin_ia32_prorvq128_mask",
-                    "avx512.mask.prorv.q.256" => "__builtin_ia32_prorvq256_mask",
-                    "avx512.mask.prorv.q.512" => "__builtin_ia32_prorvq512_mask",
-                    "avx512.mask.pshuf.b.128" => "__builtin_ia32_pshufb128_mask",
-                    "avx512.mask.pshuf.b.256" => "__builtin_ia32_pshufb256_mask",
-                    "avx512.mask.pshuf.b.512" => "__builtin_ia32_pshufb512_mask",
-                    "avx512.mask.psll.d" => "__builtin_ia32_pslld512_mask",
-                    "avx512.mask.psll.d.128" => "__builtin_ia32_pslld128_mask",
-                    "avx512.mask.psll.d.256" => "__builtin_ia32_pslld256_mask",
-                    "avx512.mask.psll.di.128" => "__builtin_ia32_pslldi128_mask",
-                    "avx512.mask.psll.di.256" => "__builtin_ia32_pslldi256_mask",
-                    "avx512.mask.psll.di.512" => "__builtin_ia32_pslldi512_mask",
-                    "avx512.mask.psll.q" => "__builtin_ia32_psllq512_mask",
-                    "avx512.mask.psll.q.128" => "__builtin_ia32_psllq128_mask",
-                    "avx512.mask.psll.q.256" => "__builtin_ia32_psllq256_mask",
-                    "avx512.mask.psll.qi.128" => "__builtin_ia32_psllqi128_mask",
-                    "avx512.mask.psll.qi.256" => "__builtin_ia32_psllqi256_mask",
-                    "avx512.mask.psll.qi.512" => "__builtin_ia32_psllqi512_mask",
-                    "avx512.mask.psll.w.128" => "__builtin_ia32_psllw128_mask",
-                    "avx512.mask.psll.w.256" => "__builtin_ia32_psllw256_mask",
-                    "avx512.mask.psll.w.512" => "__builtin_ia32_psllw512_mask",
-                    "avx512.mask.psll.wi.128" => "__builtin_ia32_psllwi128_mask",
-                    "avx512.mask.psll.wi.256" => "__builtin_ia32_psllwi256_mask",
-                    "avx512.mask.psll.wi.512" => "__builtin_ia32_psllwi512_mask",
-                    "avx512.mask.psllv.d" => "__builtin_ia32_psllv16si_mask",
-                    "avx512.mask.psllv.q" => "__builtin_ia32_psllv8di_mask",
-                    "avx512.mask.psllv16.hi" => "__builtin_ia32_psllv16hi_mask",
-                    "avx512.mask.psllv2.di" => "__builtin_ia32_psllv2di_mask",
-                    "avx512.mask.psllv32hi" => "__builtin_ia32_psllv32hi_mask",
-                    "avx512.mask.psllv4.di" => "__builtin_ia32_psllv4di_mask",
-                    "avx512.mask.psllv4.si" => "__builtin_ia32_psllv4si_mask",
-                    "avx512.mask.psllv8.hi" => "__builtin_ia32_psllv8hi_mask",
-                    "avx512.mask.psllv8.si" => "__builtin_ia32_psllv8si_mask",
-                    "avx512.mask.psra.d" => "__builtin_ia32_psrad512_mask",
-                    "avx512.mask.psra.d.128" => "__builtin_ia32_psrad128_mask",
-                    "avx512.mask.psra.d.256" => "__builtin_ia32_psrad256_mask",
-                    "avx512.mask.psra.di.128" => "__builtin_ia32_psradi128_mask",
-                    "avx512.mask.psra.di.256" => "__builtin_ia32_psradi256_mask",
-                    "avx512.mask.psra.di.512" => "__builtin_ia32_psradi512_mask",
-                    "avx512.mask.psra.q" => "__builtin_ia32_psraq512_mask",
-                    "avx512.mask.psra.q.128" => "__builtin_ia32_psraq128_mask",
-                    "avx512.mask.psra.q.256" => "__builtin_ia32_psraq256_mask",
-                    "avx512.mask.psra.qi.128" => "__builtin_ia32_psraqi128_mask",
-                    "avx512.mask.psra.qi.256" => "__builtin_ia32_psraqi256_mask",
-                    "avx512.mask.psra.qi.512" => "__builtin_ia32_psraqi512_mask",
-                    "avx512.mask.psra.w.128" => "__builtin_ia32_psraw128_mask",
-                    "avx512.mask.psra.w.256" => "__builtin_ia32_psraw256_mask",
-                    "avx512.mask.psra.w.512" => "__builtin_ia32_psraw512_mask",
-                    "avx512.mask.psra.wi.128" => "__builtin_ia32_psrawi128_mask",
-                    "avx512.mask.psra.wi.256" => "__builtin_ia32_psrawi256_mask",
-                    "avx512.mask.psra.wi.512" => "__builtin_ia32_psrawi512_mask",
-                    "avx512.mask.psrav.d" => "__builtin_ia32_psrav16si_mask",
-                    "avx512.mask.psrav.q" => "__builtin_ia32_psrav8di_mask",
-                    "avx512.mask.psrav.q.128" => "__builtin_ia32_psravq128_mask",
-                    "avx512.mask.psrav.q.256" => "__builtin_ia32_psravq256_mask",
-                    "avx512.mask.psrav16.hi" => "__builtin_ia32_psrav16hi_mask",
-                    "avx512.mask.psrav32.hi" => "__builtin_ia32_psrav32hi_mask",
-                    "avx512.mask.psrav4.si" => "__builtin_ia32_psrav4si_mask",
-                    "avx512.mask.psrav8.hi" => "__builtin_ia32_psrav8hi_mask",
-                    "avx512.mask.psrav8.si" => "__builtin_ia32_psrav8si_mask",
-                    "avx512.mask.psrl.d" => "__builtin_ia32_psrld512_mask",
-                    "avx512.mask.psrl.d.128" => "__builtin_ia32_psrld128_mask",
-                    "avx512.mask.psrl.d.256" => "__builtin_ia32_psrld256_mask",
-                    "avx512.mask.psrl.di.128" => "__builtin_ia32_psrldi128_mask",
-                    "avx512.mask.psrl.di.256" => "__builtin_ia32_psrldi256_mask",
-                    "avx512.mask.psrl.di.512" => "__builtin_ia32_psrldi512_mask",
-                    "avx512.mask.psrl.q" => "__builtin_ia32_psrlq512_mask",
-                    "avx512.mask.psrl.q.128" => "__builtin_ia32_psrlq128_mask",
-                    "avx512.mask.psrl.q.256" => "__builtin_ia32_psrlq256_mask",
-                    "avx512.mask.psrl.qi.128" => "__builtin_ia32_psrlqi128_mask",
-                    "avx512.mask.psrl.qi.256" => "__builtin_ia32_psrlqi256_mask",
-                    "avx512.mask.psrl.qi.512" => "__builtin_ia32_psrlqi512_mask",
-                    "avx512.mask.psrl.w.128" => "__builtin_ia32_psrlw128_mask",
-                    "avx512.mask.psrl.w.256" => "__builtin_ia32_psrlw256_mask",
-                    "avx512.mask.psrl.w.512" => "__builtin_ia32_psrlw512_mask",
-                    "avx512.mask.psrl.wi.128" => "__builtin_ia32_psrlwi128_mask",
-                    "avx512.mask.psrl.wi.256" => "__builtin_ia32_psrlwi256_mask",
-                    "avx512.mask.psrl.wi.512" => "__builtin_ia32_psrlwi512_mask",
-                    "avx512.mask.psrlv.d" => "__builtin_ia32_psrlv16si_mask",
-                    "avx512.mask.psrlv.q" => "__builtin_ia32_psrlv8di_mask",
-                    "avx512.mask.psrlv16.hi" => "__builtin_ia32_psrlv16hi_mask",
-                    "avx512.mask.psrlv2.di" => "__builtin_ia32_psrlv2di_mask",
-                    "avx512.mask.psrlv32hi" => "__builtin_ia32_psrlv32hi_mask",
-                    "avx512.mask.psrlv4.di" => "__builtin_ia32_psrlv4di_mask",
-                    "avx512.mask.psrlv4.si" => "__builtin_ia32_psrlv4si_mask",
-                    "avx512.mask.psrlv8.hi" => "__builtin_ia32_psrlv8hi_mask",
-                    "avx512.mask.psrlv8.si" => "__builtin_ia32_psrlv8si_mask",
-                    "avx512.mask.psub.b.128" => "__builtin_ia32_psubb128_mask",
-                    "avx512.mask.psub.b.256" => "__builtin_ia32_psubb256_mask",
-                    "avx512.mask.psub.b.512" => "__builtin_ia32_psubb512_mask",
-                    "avx512.mask.psub.d.128" => "__builtin_ia32_psubd128_mask",
-                    "avx512.mask.psub.d.256" => "__builtin_ia32_psubd256_mask",
-                    "avx512.mask.psub.d.512" => "__builtin_ia32_psubd512_mask",
-                    "avx512.mask.psub.q.128" => "__builtin_ia32_psubq128_mask",
-                    "avx512.mask.psub.q.256" => "__builtin_ia32_psubq256_mask",
-                    "avx512.mask.psub.q.512" => "__builtin_ia32_psubq512_mask",
-                    "avx512.mask.psub.w.128" => "__builtin_ia32_psubw128_mask",
-                    "avx512.mask.psub.w.256" => "__builtin_ia32_psubw256_mask",
-                    "avx512.mask.psub.w.512" => "__builtin_ia32_psubw512_mask",
-                    "avx512.mask.psubs.b.128" => "__builtin_ia32_psubsb128_mask",
-                    "avx512.mask.psubs.b.256" => "__builtin_ia32_psubsb256_mask",
-                    "avx512.mask.psubs.b.512" => "__builtin_ia32_psubsb512_mask",
-                    "avx512.mask.psubs.w.128" => "__builtin_ia32_psubsw128_mask",
-                    "avx512.mask.psubs.w.256" => "__builtin_ia32_psubsw256_mask",
-                    "avx512.mask.psubs.w.512" => "__builtin_ia32_psubsw512_mask",
-                    "avx512.mask.psubus.b.128" => "__builtin_ia32_psubusb128_mask",
-                    "avx512.mask.psubus.b.256" => "__builtin_ia32_psubusb256_mask",
-                    "avx512.mask.psubus.b.512" => "__builtin_ia32_psubusb512_mask",
-                    "avx512.mask.psubus.w.128" => "__builtin_ia32_psubusw128_mask",
-                    "avx512.mask.psubus.w.256" => "__builtin_ia32_psubusw256_mask",
-                    "avx512.mask.psubus.w.512" => "__builtin_ia32_psubusw512_mask",
-                    "avx512.mask.pternlog.d.128" => "__builtin_ia32_pternlogd128_mask",
-                    "avx512.mask.pternlog.d.256" => "__builtin_ia32_pternlogd256_mask",
-                    "avx512.mask.pternlog.d.512" => "__builtin_ia32_pternlogd512_mask",
-                    "avx512.mask.pternlog.q.128" => "__builtin_ia32_pternlogq128_mask",
-                    "avx512.mask.pternlog.q.256" => "__builtin_ia32_pternlogq256_mask",
-                    "avx512.mask.pternlog.q.512" => "__builtin_ia32_pternlogq512_mask",
-                    "avx512.mask.ptestm.d.512" => "__builtin_ia32_ptestmd512",
-                    "avx512.mask.ptestm.q.512" => "__builtin_ia32_ptestmq512",
                     "avx512.mask.range.pd.128" => "__builtin_ia32_rangepd128_mask",
                     "avx512.mask.range.pd.256" => "__builtin_ia32_rangepd256_mask",
                     "avx512.mask.range.pd.512" => "__builtin_ia32_rangepd512_mask",
@@ -9293,181 +8330,11 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.mask.scalef.ps.512" => "__builtin_ia32_scalefps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.scalef.sd" => "__builtin_ia32_scalefsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.scalef.ss" => "__builtin_ia32_scalefss_round_mask",
-                    "avx512.mask.shuf.f32x4" => "__builtin_ia32_shuf_f32x4_mask",
-                    "avx512.mask.shuf.f32x4.256" => "__builtin_ia32_shuf_f32x4_256_mask",
-                    "avx512.mask.shuf.f64x2" => "__builtin_ia32_shuf_f64x2_mask",
-                    "avx512.mask.shuf.f64x2.256" => "__builtin_ia32_shuf_f64x2_256_mask",
-                    "avx512.mask.shuf.i32x4" => "__builtin_ia32_shuf_i32x4_mask",
-                    "avx512.mask.shuf.i32x4.256" => "__builtin_ia32_shuf_i32x4_256_mask",
-                    "avx512.mask.shuf.i64x2" => "__builtin_ia32_shuf_i64x2_mask",
-                    "avx512.mask.shuf.i64x2.256" => "__builtin_ia32_shuf_i64x2_256_mask",
-                    "avx512.mask.shuf.pd.128" => "__builtin_ia32_shufpd128_mask",
-                    "avx512.mask.shuf.pd.256" => "__builtin_ia32_shufpd256_mask",
-                    "avx512.mask.shuf.pd.512" => "__builtin_ia32_shufpd512_mask",
-                    "avx512.mask.shuf.ps.128" => "__builtin_ia32_shufps128_mask",
-                    "avx512.mask.shuf.ps.256" => "__builtin_ia32_shufps256_mask",
-                    "avx512.mask.shuf.ps.512" => "__builtin_ia32_shufps512_mask",
-                    "avx512.mask.sqrt.pd.128" => "__builtin_ia32_sqrtpd128_mask",
-                    "avx512.mask.sqrt.pd.256" => "__builtin_ia32_sqrtpd256_mask",
-                    "avx512.mask.sqrt.pd.512" => "__builtin_ia32_sqrtpd512_mask",
-                    "avx512.mask.sqrt.ps.128" => "__builtin_ia32_sqrtps128_mask",
-                    "avx512.mask.sqrt.ps.256" => "__builtin_ia32_sqrtps256_mask",
-                    "avx512.mask.sqrt.ps.512" => "__builtin_ia32_sqrtps512_mask",
-                    // [INVALID CONVERSION]: "avx512.mask.sqrt.sd" => "__builtin_ia32_sqrtsd_round_mask",
-                    // [INVALID CONVERSION]: "avx512.mask.sqrt.ss" => "__builtin_ia32_sqrtss_round_mask",
-                    "avx512.mask.store.ss" => "__builtin_ia32_storess_mask",
-                    "avx512.mask.storeu.d.512" => "__builtin_ia32_storedqusi512_mask",
-                    "avx512.mask.storeu.pd.512" => "__builtin_ia32_storeupd512_mask",
-                    "avx512.mask.storeu.ps.512" => "__builtin_ia32_storeups512_mask",
-                    "avx512.mask.storeu.q.512" => "__builtin_ia32_storedqudi512_mask",
-                    "avx512.mask.sub.pd.128" => "__builtin_ia32_subpd128_mask",
-                    "avx512.mask.sub.pd.256" => "__builtin_ia32_subpd256_mask",
-                    "avx512.mask.sub.pd.512" => "__builtin_ia32_subpd512_mask",
-                    "avx512.mask.sub.ps.128" => "__builtin_ia32_subps128_mask",
-                    "avx512.mask.sub.ps.256" => "__builtin_ia32_subps256_mask",
-                    "avx512.mask.sub.ps.512" => "__builtin_ia32_subps512_mask",
                     // [INVALID CONVERSION]: "avx512.mask.sub.sd.round" => "__builtin_ia32_subsd_round_mask",
                     // [INVALID CONVERSION]: "avx512.mask.sub.ss.round" => "__builtin_ia32_subss_round_mask",
-                    "avx512.mask.valign.d.128" => "__builtin_ia32_alignd128_mask",
-                    "avx512.mask.valign.d.256" => "__builtin_ia32_alignd256_mask",
-                    "avx512.mask.valign.d.512" => "__builtin_ia32_alignd512_mask",
-                    "avx512.mask.valign.q.128" => "__builtin_ia32_alignq128_mask",
-                    "avx512.mask.valign.q.256" => "__builtin_ia32_alignq256_mask",
-                    "avx512.mask.valign.q.512" => "__builtin_ia32_alignq512_mask",
-                    "avx512.mask.vcvtph2ps.128" => "__builtin_ia32_vcvtph2ps_mask",
-                    "avx512.mask.vcvtph2ps.256" => "__builtin_ia32_vcvtph2ps256_mask",
-                    "avx512.mask.vcvtph2ps.512" => "__builtin_ia32_vcvtph2ps512_mask",
                     "avx512.mask.vcvtps2ph.128" => "__builtin_ia32_vcvtps2ph_mask",
                     "avx512.mask.vcvtps2ph.256" => "__builtin_ia32_vcvtps2ph256_mask",
                     "avx512.mask.vcvtps2ph.512" => "__builtin_ia32_vcvtps2ph512_mask",
-                    "avx512.mask.vextractf32x4.256" => "__builtin_ia32_extractf32x4_256_mask",
-                    "avx512.mask.vextractf32x4.512" => "__builtin_ia32_extractf32x4_mask",
-                    "avx512.mask.vextractf32x8.512" => "__builtin_ia32_extractf32x8_mask",
-                    "avx512.mask.vextractf64x2.256" => "__builtin_ia32_extractf64x2_256_mask",
-                    "avx512.mask.vextractf64x2.512" => "__builtin_ia32_extractf64x2_512_mask",
-                    "avx512.mask.vextractf64x4.512" => "__builtin_ia32_extractf64x4_mask",
-                    "avx512.mask.vextracti32x4.256" => "__builtin_ia32_extracti32x4_256_mask",
-                    "avx512.mask.vextracti32x4.512" => "__builtin_ia32_extracti32x4_mask",
-                    "avx512.mask.vextracti32x8.512" => "__builtin_ia32_extracti32x8_mask",
-                    "avx512.mask.vextracti64x2.256" => "__builtin_ia32_extracti64x2_256_mask",
-                    "avx512.mask.vextracti64x2.512" => "__builtin_ia32_extracti64x2_512_mask",
-                    "avx512.mask.vextracti64x4.512" => "__builtin_ia32_extracti64x4_mask",
-                    "avx512.mask.vfmadd.pd.128" => "__builtin_ia32_vfmaddpd128_mask",
-                    "avx512.mask.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256_mask",
-                    "avx512.mask.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
-                    "avx512.mask.vfmadd.ps.128" => "__builtin_ia32_vfmaddps128_mask",
-                    "avx512.mask.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256_mask",
-                    "avx512.mask.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
-                    "avx512.mask.vfmadd.sd" => "__builtin_ia32_vfmaddsd3_mask",
-                    "avx512.mask.vfmadd.ss" => "__builtin_ia32_vfmaddss3_mask",
-                    "avx512.mask.vfmaddsub.pd.128" => "__builtin_ia32_vfmaddsubpd128_mask",
-                    "avx512.mask.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256_mask",
-                    "avx512.mask.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask",
-                    "avx512.mask.vfmaddsub.ps.128" => "__builtin_ia32_vfmaddsubps128_mask",
-                    "avx512.mask.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256_mask",
-                    "avx512.mask.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
-                    "avx512.mask.vfnmadd.pd.128" => "__builtin_ia32_vfnmaddpd128_mask",
-                    "avx512.mask.vfnmadd.pd.256" => "__builtin_ia32_vfnmaddpd256_mask",
-                    "avx512.mask.vfnmadd.pd.512" => "__builtin_ia32_vfnmaddpd512_mask",
-                    "avx512.mask.vfnmadd.ps.128" => "__builtin_ia32_vfnmaddps128_mask",
-                    "avx512.mask.vfnmadd.ps.256" => "__builtin_ia32_vfnmaddps256_mask",
-                    "avx512.mask.vfnmadd.ps.512" => "__builtin_ia32_vfnmaddps512_mask",
-                    "avx512.mask.vfnmsub.pd.128" => "__builtin_ia32_vfnmsubpd128_mask",
-                    "avx512.mask.vfnmsub.pd.256" => "__builtin_ia32_vfnmsubpd256_mask",
-                    "avx512.mask.vfnmsub.pd.512" => "__builtin_ia32_vfnmsubpd512_mask",
-                    "avx512.mask.vfnmsub.ps.128" => "__builtin_ia32_vfnmsubps128_mask",
-                    "avx512.mask.vfnmsub.ps.256" => "__builtin_ia32_vfnmsubps256_mask",
-                    "avx512.mask.vfnmsub.ps.512" => "__builtin_ia32_vfnmsubps512_mask",
-                    "avx512.mask.vpermi2var.d.128" => "__builtin_ia32_vpermi2vard128_mask",
-                    "avx512.mask.vpermi2var.d.256" => "__builtin_ia32_vpermi2vard256_mask",
-                    "avx512.mask.vpermi2var.d.512" => "__builtin_ia32_vpermi2vard512_mask",
-                    "avx512.mask.vpermi2var.hi.128" => "__builtin_ia32_vpermi2varhi128_mask",
-                    "avx512.mask.vpermi2var.hi.256" => "__builtin_ia32_vpermi2varhi256_mask",
-                    "avx512.mask.vpermi2var.hi.512" => "__builtin_ia32_vpermi2varhi512_mask",
-                    "avx512.mask.vpermi2var.pd.128" => "__builtin_ia32_vpermi2varpd128_mask",
-                    "avx512.mask.vpermi2var.pd.256" => "__builtin_ia32_vpermi2varpd256_mask",
-                    "avx512.mask.vpermi2var.pd.512" => "__builtin_ia32_vpermi2varpd512_mask",
-                    "avx512.mask.vpermi2var.ps.128" => "__builtin_ia32_vpermi2varps128_mask",
-                    "avx512.mask.vpermi2var.ps.256" => "__builtin_ia32_vpermi2varps256_mask",
-                    "avx512.mask.vpermi2var.ps.512" => "__builtin_ia32_vpermi2varps512_mask",
-                    "avx512.mask.vpermi2var.q.128" => "__builtin_ia32_vpermi2varq128_mask",
-                    "avx512.mask.vpermi2var.q.256" => "__builtin_ia32_vpermi2varq256_mask",
-                    "avx512.mask.vpermi2var.q.512" => "__builtin_ia32_vpermi2varq512_mask",
-                    "avx512.mask.vpermi2var.qi.128" => "__builtin_ia32_vpermi2varqi128_mask",
-                    "avx512.mask.vpermi2var.qi.256" => "__builtin_ia32_vpermi2varqi256_mask",
-                    "avx512.mask.vpermi2var.qi.512" => "__builtin_ia32_vpermi2varqi512_mask",
-                    "avx512.mask.vpermilvar.pd.128" => "__builtin_ia32_vpermilvarpd_mask",
-                    "avx512.mask.vpermilvar.pd.256" => "__builtin_ia32_vpermilvarpd256_mask",
-                    "avx512.mask.vpermilvar.pd.512" => "__builtin_ia32_vpermilvarpd512_mask",
-                    "avx512.mask.vpermilvar.ps.128" => "__builtin_ia32_vpermilvarps_mask",
-                    "avx512.mask.vpermilvar.ps.256" => "__builtin_ia32_vpermilvarps256_mask",
-                    "avx512.mask.vpermilvar.ps.512" => "__builtin_ia32_vpermilvarps512_mask",
-                    "avx512.mask.vpermt.d.512" => "__builtin_ia32_vpermt2vard512_mask",
-                    "avx512.mask.vpermt.pd.512" => "__builtin_ia32_vpermt2varpd512_mask",
-                    "avx512.mask.vpermt.ps.512" => "__builtin_ia32_vpermt2varps512_mask",
-                    "avx512.mask.vpermt.q.512" => "__builtin_ia32_vpermt2varq512_mask",
-                    "avx512.mask.vpermt2var.d.128" => "__builtin_ia32_vpermt2vard128_mask",
-                    "avx512.mask.vpermt2var.d.256" => "__builtin_ia32_vpermt2vard256_mask",
-                    "avx512.mask.vpermt2var.d.512" => "__builtin_ia32_vpermt2vard512_mask",
-                    "avx512.mask.vpermt2var.hi.128" => "__builtin_ia32_vpermt2varhi128_mask",
-                    "avx512.mask.vpermt2var.hi.256" => "__builtin_ia32_vpermt2varhi256_mask",
-                    "avx512.mask.vpermt2var.hi.512" => "__builtin_ia32_vpermt2varhi512_mask",
-                    "avx512.mask.vpermt2var.pd.128" => "__builtin_ia32_vpermt2varpd128_mask",
-                    "avx512.mask.vpermt2var.pd.256" => "__builtin_ia32_vpermt2varpd256_mask",
-                    "avx512.mask.vpermt2var.pd.512" => "__builtin_ia32_vpermt2varpd512_mask",
-                    "avx512.mask.vpermt2var.ps.128" => "__builtin_ia32_vpermt2varps128_mask",
-                    "avx512.mask.vpermt2var.ps.256" => "__builtin_ia32_vpermt2varps256_mask",
-                    "avx512.mask.vpermt2var.ps.512" => "__builtin_ia32_vpermt2varps512_mask",
-                    "avx512.mask.vpermt2var.q.128" => "__builtin_ia32_vpermt2varq128_mask",
-                    "avx512.mask.vpermt2var.q.256" => "__builtin_ia32_vpermt2varq256_mask",
-                    "avx512.mask.vpermt2var.q.512" => "__builtin_ia32_vpermt2varq512_mask",
-                    "avx512.mask.vpermt2var.qi.128" => "__builtin_ia32_vpermt2varqi128_mask",
-                    "avx512.mask.vpermt2var.qi.256" => "__builtin_ia32_vpermt2varqi256_mask",
-                    "avx512.mask.vpermt2var.qi.512" => "__builtin_ia32_vpermt2varqi512_mask",
-                    "avx512.mask.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_mask",
-                    "avx512.mask.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_mask",
-                    "avx512.mask.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_mask",
-                    "avx512.mask.vpmadd52l.uq.128" => "__builtin_ia32_vpmadd52luq128_mask",
-                    "avx512.mask.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_mask",
-                    "avx512.mask.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_mask",
-                    "avx512.mask.xor.pd.128" => "__builtin_ia32_xorpd128_mask",
-                    "avx512.mask.xor.pd.256" => "__builtin_ia32_xorpd256_mask",
-                    "avx512.mask.xor.pd.512" => "__builtin_ia32_xorpd512_mask",
-                    "avx512.mask.xor.ps.128" => "__builtin_ia32_xorps128_mask",
-                    "avx512.mask.xor.ps.256" => "__builtin_ia32_xorps256_mask",
-                    "avx512.mask.xor.ps.512" => "__builtin_ia32_xorps512_mask",
-                    "avx512.mask3.vfmadd.pd.128" => "__builtin_ia32_vfmaddpd128_mask3",
-                    "avx512.mask3.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256_mask3",
-                    "avx512.mask3.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask3",
-                    "avx512.mask3.vfmadd.ps.128" => "__builtin_ia32_vfmaddps128_mask3",
-                    "avx512.mask3.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256_mask3",
-                    "avx512.mask3.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask3",
-                    "avx512.mask3.vfmadd.sd" => "__builtin_ia32_vfmaddsd3_mask3",
-                    "avx512.mask3.vfmadd.ss" => "__builtin_ia32_vfmaddss3_mask3",
-                    "avx512.mask3.vfmaddsub.pd.128" => "__builtin_ia32_vfmaddsubpd128_mask3",
-                    "avx512.mask3.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256_mask3",
-                    "avx512.mask3.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask3",
-                    "avx512.mask3.vfmaddsub.ps.128" => "__builtin_ia32_vfmaddsubps128_mask3",
-                    "avx512.mask3.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256_mask3",
-                    "avx512.mask3.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask3",
-                    "avx512.mask3.vfmsub.pd.128" => "__builtin_ia32_vfmsubpd128_mask3",
-                    "avx512.mask3.vfmsub.pd.256" => "__builtin_ia32_vfmsubpd256_mask3",
-                    "avx512.mask3.vfmsub.pd.512" => "__builtin_ia32_vfmsubpd512_mask3",
-                    "avx512.mask3.vfmsub.ps.128" => "__builtin_ia32_vfmsubps128_mask3",
-                    "avx512.mask3.vfmsub.ps.256" => "__builtin_ia32_vfmsubps256_mask3",
-                    "avx512.mask3.vfmsub.ps.512" => "__builtin_ia32_vfmsubps512_mask3",
-                    "avx512.mask3.vfmsubadd.pd.128" => "__builtin_ia32_vfmsubaddpd128_mask3",
-                    "avx512.mask3.vfmsubadd.pd.256" => "__builtin_ia32_vfmsubaddpd256_mask3",
-                    "avx512.mask3.vfmsubadd.pd.512" => "__builtin_ia32_vfmsubaddpd512_mask3",
-                    "avx512.mask3.vfmsubadd.ps.128" => "__builtin_ia32_vfmsubaddps128_mask3",
-                    "avx512.mask3.vfmsubadd.ps.256" => "__builtin_ia32_vfmsubaddps256_mask3",
-                    "avx512.mask3.vfmsubadd.ps.512" => "__builtin_ia32_vfmsubaddps512_mask3",
-                    "avx512.mask3.vfnmsub.pd.128" => "__builtin_ia32_vfnmsubpd128_mask3",
-                    "avx512.mask3.vfnmsub.pd.256" => "__builtin_ia32_vfnmsubpd256_mask3",
-                    "avx512.mask3.vfnmsub.pd.512" => "__builtin_ia32_vfnmsubpd512_mask3",
-                    "avx512.mask3.vfnmsub.ps.128" => "__builtin_ia32_vfnmsubps128_mask3",
-                    "avx512.mask3.vfnmsub.ps.256" => "__builtin_ia32_vfnmsubps256_mask3",
-                    "avx512.mask3.vfnmsub.ps.512" => "__builtin_ia32_vfnmsubps512_mask3",
                     "avx512.maskz.fixupimm.pd.128" => "__builtin_ia32_fixupimmpd128_maskz",
                     "avx512.maskz.fixupimm.pd.256" => "__builtin_ia32_fixupimmpd256_maskz",
                     "avx512.maskz.fixupimm.pd.512" => "__builtin_ia32_fixupimmpd512_maskz",
@@ -9476,55 +8343,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.maskz.fixupimm.ps.512" => "__builtin_ia32_fixupimmps512_maskz",
                     "avx512.maskz.fixupimm.sd" => "__builtin_ia32_fixupimmsd_maskz",
                     "avx512.maskz.fixupimm.ss" => "__builtin_ia32_fixupimmss_maskz",
-                    "avx512.maskz.pternlog.d.128" => "__builtin_ia32_pternlogd128_maskz",
-                    "avx512.maskz.pternlog.d.256" => "__builtin_ia32_pternlogd256_maskz",
-                    "avx512.maskz.pternlog.d.512" => "__builtin_ia32_pternlogd512_maskz",
-                    "avx512.maskz.pternlog.q.128" => "__builtin_ia32_pternlogq128_maskz",
-                    "avx512.maskz.pternlog.q.256" => "__builtin_ia32_pternlogq256_maskz",
-                    "avx512.maskz.pternlog.q.512" => "__builtin_ia32_pternlogq512_maskz",
-                    "avx512.maskz.vfmadd.pd.128" => "__builtin_ia32_vfmaddpd128_maskz",
-                    "avx512.maskz.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256_maskz",
-                    "avx512.maskz.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_maskz",
-                    "avx512.maskz.vfmadd.ps.128" => "__builtin_ia32_vfmaddps128_maskz",
-                    "avx512.maskz.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256_maskz",
-                    "avx512.maskz.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_maskz",
-                    "avx512.maskz.vfmadd.sd" => "__builtin_ia32_vfmaddsd3_maskz",
-                    "avx512.maskz.vfmadd.ss" => "__builtin_ia32_vfmaddss3_maskz",
-                    "avx512.maskz.vfmaddsub.pd.128" => "__builtin_ia32_vfmaddsubpd128_maskz",
-                    "avx512.maskz.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256_maskz",
-                    "avx512.maskz.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_maskz",
-                    "avx512.maskz.vfmaddsub.ps.128" => "__builtin_ia32_vfmaddsubps128_maskz",
-                    "avx512.maskz.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256_maskz",
-                    "avx512.maskz.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_maskz",
-                    "avx512.maskz.vpermt2var.d.128" => "__builtin_ia32_vpermt2vard128_maskz",
-                    "avx512.maskz.vpermt2var.d.256" => "__builtin_ia32_vpermt2vard256_maskz",
-                    "avx512.maskz.vpermt2var.d.512" => "__builtin_ia32_vpermt2vard512_maskz",
-                    "avx512.maskz.vpermt2var.hi.128" => "__builtin_ia32_vpermt2varhi128_maskz",
-                    "avx512.maskz.vpermt2var.hi.256" => "__builtin_ia32_vpermt2varhi256_maskz",
-                    "avx512.maskz.vpermt2var.hi.512" => "__builtin_ia32_vpermt2varhi512_maskz",
-                    "avx512.maskz.vpermt2var.pd.128" => "__builtin_ia32_vpermt2varpd128_maskz",
-                    "avx512.maskz.vpermt2var.pd.256" => "__builtin_ia32_vpermt2varpd256_maskz",
-                    "avx512.maskz.vpermt2var.pd.512" => "__builtin_ia32_vpermt2varpd512_maskz",
-                    "avx512.maskz.vpermt2var.ps.128" => "__builtin_ia32_vpermt2varps128_maskz",
-                    "avx512.maskz.vpermt2var.ps.256" => "__builtin_ia32_vpermt2varps256_maskz",
-                    "avx512.maskz.vpermt2var.ps.512" => "__builtin_ia32_vpermt2varps512_maskz",
-                    "avx512.maskz.vpermt2var.q.128" => "__builtin_ia32_vpermt2varq128_maskz",
-                    "avx512.maskz.vpermt2var.q.256" => "__builtin_ia32_vpermt2varq256_maskz",
-                    "avx512.maskz.vpermt2var.q.512" => "__builtin_ia32_vpermt2varq512_maskz",
-                    "avx512.maskz.vpermt2var.qi.128" => "__builtin_ia32_vpermt2varqi128_maskz",
-                    "avx512.maskz.vpermt2var.qi.256" => "__builtin_ia32_vpermt2varqi256_maskz",
-                    "avx512.maskz.vpermt2var.qi.512" => "__builtin_ia32_vpermt2varqi512_maskz",
-                    "avx512.maskz.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_maskz",
-                    "avx512.maskz.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_maskz",
-                    "avx512.maskz.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_maskz",
-                    "avx512.maskz.vpmadd52l.uq.128" => "__builtin_ia32_vpmadd52luq128_maskz",
-                    "avx512.maskz.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_maskz",
-                    "avx512.maskz.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_maskz",
                     "avx512.max.pd.512" => "__builtin_ia32_maxpd512",
                     "avx512.max.ps.512" => "__builtin_ia32_maxps512",
                     "avx512.min.pd.512" => "__builtin_ia32_minpd512",
                     "avx512.min.ps.512" => "__builtin_ia32_minps512",
-                    "avx512.movntdqa" => "__builtin_ia32_movntdqa512",
                     "avx512.mul.pd.512" => "__builtin_ia32_mulpd512",
                     "avx512.mul.ps.512" => "__builtin_ia32_mulps512",
                     "avx512.packssdw.512" => "__builtin_ia32_packssdw512",
@@ -9533,8 +8355,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.packuswb.512" => "__builtin_ia32_packuswb512",
                     "avx512.pavg.b.512" => "__builtin_ia32_pavgb512",
                     "avx512.pavg.w.512" => "__builtin_ia32_pavgw512",
-                    "avx512.pbroadcastd.512" => "__builtin_ia32_pbroadcastd512",
-                    "avx512.pbroadcastq.512" => "__builtin_ia32_pbroadcastq512",
                     "avx512.permvar.df.256" => "__builtin_ia32_permvardf256",
                     "avx512.permvar.df.512" => "__builtin_ia32_permvardf512",
                     "avx512.permvar.di.256" => "__builtin_ia32_permvardi256",
@@ -9549,11 +8369,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.permvar.si.512" => "__builtin_ia32_permvarsi512",
                     "avx512.pmaddubs.w.512" => "__builtin_ia32_pmaddubsw512",
                     "avx512.pmaddw.d.512" => "__builtin_ia32_pmaddwd512",
-                    "avx512.pmovzxbd" => "__builtin_ia32_pmovzxbd512",
-                    "avx512.pmovzxbq" => "__builtin_ia32_pmovzxbq512",
-                    "avx512.pmovzxdq" => "__builtin_ia32_pmovzxdq512",
-                    "avx512.pmovzxwd" => "__builtin_ia32_pmovzxwd512",
-                    "avx512.pmovzxwq" => "__builtin_ia32_pmovzxwq512",
                     "avx512.pmul.hr.sw.512" => "__builtin_ia32_pmulhrsw512",
                     "avx512.pmulh.w.512" => "__builtin_ia32_pmulhw512",
                     "avx512.pmulhu.w.512" => "__builtin_ia32_pmulhuw512",
@@ -9563,8 +8378,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.psad.bw.512" => "__builtin_ia32_psadbw512",
                     "avx512.pshuf.b.512" => "__builtin_ia32_pshufb512",
                     "avx512.psll.d.512" => "__builtin_ia32_pslld512",
-                    "avx512.psll.dq" => "__builtin_ia32_pslldqi512",
-                    "avx512.psll.dq.bs" => "__builtin_ia32_pslldqi512_byteshift",
                     "avx512.psll.q.512" => "__builtin_ia32_psllq512",
                     "avx512.psll.w.512" => "__builtin_ia32_psllw512",
                     "avx512.pslli.d.512" => "__builtin_ia32_pslldi512",
@@ -9593,8 +8406,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.psrav.w.256" => "__builtin_ia32_psrav16hi",
                     "avx512.psrav.w.512" => "__builtin_ia32_psrav32hi",
                     "avx512.psrl.d.512" => "__builtin_ia32_psrld512",
-                    "avx512.psrl.dq" => "__builtin_ia32_psrldqi512",
-                    "avx512.psrl.dq.bs" => "__builtin_ia32_psrldqi512_byteshift",
                     "avx512.psrl.q.512" => "__builtin_ia32_psrlq512",
                     "avx512.psrl.w.512" => "__builtin_ia32_psrlw512",
                     "avx512.psrli.d.512" => "__builtin_ia32_psrldi512",
@@ -9611,30 +8422,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.pternlog.q.128" => "__builtin_ia32_pternlogq128",
                     "avx512.pternlog.q.256" => "__builtin_ia32_pternlogq256",
                     "avx512.pternlog.q.512" => "__builtin_ia32_pternlogq512",
-                    "avx512.ptestm.b.128" => "__builtin_ia32_ptestmb128",
-                    "avx512.ptestm.b.256" => "__builtin_ia32_ptestmb256",
-                    "avx512.ptestm.b.512" => "__builtin_ia32_ptestmb512",
-                    "avx512.ptestm.d.128" => "__builtin_ia32_ptestmd128",
-                    "avx512.ptestm.d.256" => "__builtin_ia32_ptestmd256",
-                    "avx512.ptestm.d.512" => "__builtin_ia32_ptestmd512",
-                    "avx512.ptestm.q.128" => "__builtin_ia32_ptestmq128",
-                    "avx512.ptestm.q.256" => "__builtin_ia32_ptestmq256",
-                    "avx512.ptestm.q.512" => "__builtin_ia32_ptestmq512",
-                    "avx512.ptestm.w.128" => "__builtin_ia32_ptestmw128",
-                    "avx512.ptestm.w.256" => "__builtin_ia32_ptestmw256",
-                    "avx512.ptestm.w.512" => "__builtin_ia32_ptestmw512",
-                    "avx512.ptestnm.b.128" => "__builtin_ia32_ptestnmb128",
-                    "avx512.ptestnm.b.256" => "__builtin_ia32_ptestnmb256",
-                    "avx512.ptestnm.b.512" => "__builtin_ia32_ptestnmb512",
-                    "avx512.ptestnm.d.128" => "__builtin_ia32_ptestnmd128",
-                    "avx512.ptestnm.d.256" => "__builtin_ia32_ptestnmd256",
-                    "avx512.ptestnm.d.512" => "__builtin_ia32_ptestnmd512",
-                    "avx512.ptestnm.q.128" => "__builtin_ia32_ptestnmq128",
-                    "avx512.ptestnm.q.256" => "__builtin_ia32_ptestnmq256",
-                    "avx512.ptestnm.q.512" => "__builtin_ia32_ptestnmq512",
-                    "avx512.ptestnm.w.128" => "__builtin_ia32_ptestnmw128",
-                    "avx512.ptestnm.w.256" => "__builtin_ia32_ptestnmw256",
-                    "avx512.ptestnm.w.512" => "__builtin_ia32_ptestnmw512",
                     "avx512.rcp14.pd.128" => "__builtin_ia32_rcp14pd128_mask",
                     "avx512.rcp14.pd.256" => "__builtin_ia32_rcp14pd256_mask",
                     "avx512.rcp14.pd.512" => "__builtin_ia32_rcp14pd512_mask",
@@ -9643,14 +8430,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.rcp14.ps.512" => "__builtin_ia32_rcp14ps512_mask",
                     "avx512.rcp14.sd" => "__builtin_ia32_rcp14sd_mask",
                     "avx512.rcp14.ss" => "__builtin_ia32_rcp14ss_mask",
-                    "avx512.rcp28.pd" => "__builtin_ia32_rcp28pd_mask",
-                    "avx512.rcp28.ps" => "__builtin_ia32_rcp28ps_mask",
-                    "avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
-                    // [DUPLICATE]: "avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
-                    "avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
-                    // [DUPLICATE]: "avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
-                    "avx512.rndscale.sd" => "__builtin_ia32_rndscalesd",
-                    "avx512.rndscale.ss" => "__builtin_ia32_rndscaless",
                     "avx512.rsqrt14.pd.128" => "__builtin_ia32_rsqrt14pd128_mask",
                     "avx512.rsqrt14.pd.256" => "__builtin_ia32_rsqrt14pd256_mask",
                     "avx512.rsqrt14.pd.512" => "__builtin_ia32_rsqrt14pd512_mask",
@@ -9659,50 +8438,8 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx512.rsqrt14.ps.512" => "__builtin_ia32_rsqrt14ps512_mask",
                     "avx512.rsqrt14.sd" => "__builtin_ia32_rsqrt14sd_mask",
                     "avx512.rsqrt14.ss" => "__builtin_ia32_rsqrt14ss_mask",
-                    "avx512.rsqrt28.pd" => "__builtin_ia32_rsqrt28pd_mask",
-                    "avx512.rsqrt28.ps" => "__builtin_ia32_rsqrt28ps_mask",
-                    "avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
-                    // [DUPLICATE]: "avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
-                    "avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
-                    // [DUPLICATE]: "avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
-                    "avx512.scatter.dpd.512" => "__builtin_ia32_scattersiv8df",
-                    "avx512.scatter.dpi.512" => "__builtin_ia32_scattersiv16si",
-                    "avx512.scatter.dpq.512" => "__builtin_ia32_scattersiv8di",
-                    "avx512.scatter.dps.512" => "__builtin_ia32_scattersiv16sf",
-                    "avx512.scatter.qpd.512" => "__builtin_ia32_scatterdiv8df",
-                    "avx512.scatter.qpi.512" => "__builtin_ia32_scatterdiv16si",
-                    "avx512.scatter.qpq.512" => "__builtin_ia32_scatterdiv8di",
-                    "avx512.scatter.qps.512" => "__builtin_ia32_scatterdiv16sf",
-                    "avx512.scatterdiv2.df" => "__builtin_ia32_scatterdiv2df",
-                    "avx512.scatterdiv2.di" => "__builtin_ia32_scatterdiv2di",
-                    "avx512.scatterdiv4.df" => "__builtin_ia32_scatterdiv4df",
-                    "avx512.scatterdiv4.di" => "__builtin_ia32_scatterdiv4di",
-                    "avx512.scatterdiv4.sf" => "__builtin_ia32_scatterdiv4sf",
-                    "avx512.scatterdiv4.si" => "__builtin_ia32_scatterdiv4si",
-                    "avx512.scatterdiv8.sf" => "__builtin_ia32_scatterdiv8sf",
-                    "avx512.scatterdiv8.si" => "__builtin_ia32_scatterdiv8si",
-                    "avx512.scatterpf.dpd.512" => "__builtin_ia32_scatterpfdpd",
-                    "avx512.scatterpf.dps.512" => "__builtin_ia32_scatterpfdps",
-                    "avx512.scatterpf.qpd.512" => "__builtin_ia32_scatterpfqpd",
-                    "avx512.scatterpf.qps.512" => "__builtin_ia32_scatterpfqps",
-                    "avx512.scattersiv2.df" => "__builtin_ia32_scattersiv2df",
-                    "avx512.scattersiv2.di" => "__builtin_ia32_scattersiv2di",
-                    "avx512.scattersiv4.df" => "__builtin_ia32_scattersiv4df",
-                    "avx512.scattersiv4.di" => "__builtin_ia32_scattersiv4di",
-                    "avx512.scattersiv4.sf" => "__builtin_ia32_scattersiv4sf",
-                    "avx512.scattersiv4.si" => "__builtin_ia32_scattersiv4si",
-                    "avx512.scattersiv8.sf" => "__builtin_ia32_scattersiv8sf",
-                    "avx512.scattersiv8.si" => "__builtin_ia32_scattersiv8si",
-                    "avx512.sqrt.pd.512" => "__builtin_ia32_sqrtpd512_mask",
-                    "avx512.sqrt.ps.512" => "__builtin_ia32_sqrtps512_mask",
-                    "avx512.sqrt.sd" => "__builtin_ia32_sqrtrndsd",
-                    "avx512.sqrt.ss" => "__builtin_ia32_sqrtrndss",
                     "avx512.sub.pd.512" => "__builtin_ia32_subpd512",
                     "avx512.sub.ps.512" => "__builtin_ia32_subps512",
-                    "avx512.vbroadcast.sd.512" => "__builtin_ia32_vbroadcastsd512",
-                    "avx512.vbroadcast.sd.pd.512" => "__builtin_ia32_vbroadcastsd_pd512",
-                    "avx512.vbroadcast.ss.512" => "__builtin_ia32_vbroadcastss512",
-                    "avx512.vbroadcast.ss.ps.512" => "__builtin_ia32_vbroadcastss_ps512",
                     "avx512.vcomi.sd" => "__builtin_ia32_vcomisd",
                     "avx512.vcomi.ss" => "__builtin_ia32_vcomiss",
                     "avx512.vcvtsd2si32" => "__builtin_ia32_vcvtsd2si32",
@@ -9928,50 +8665,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "flags.read.u64" => "__builtin_ia32_readeflags_u64",
                     "flags.write.u32" => "__builtin_ia32_writeeflags_u32",
                     "flags.write.u64" => "__builtin_ia32_writeeflags_u64",
-                    "fma.mask.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
-                    "fma.mask.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
-                    "fma.mask.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask",
-                    "fma.mask.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
-                    "fma.mask.vfmsub.pd.512" => "__builtin_ia32_vfmsubpd512_mask",
-                    "fma.mask.vfmsub.ps.512" => "__builtin_ia32_vfmsubps512_mask",
-                    "fma.mask.vfmsubadd.pd.512" => "__builtin_ia32_vfmsubaddpd512_mask",
-                    "fma.mask.vfmsubadd.ps.512" => "__builtin_ia32_vfmsubaddps512_mask",
-                    "fma.mask.vfnmadd.pd.512" => "__builtin_ia32_vfnmaddpd512_mask",
-                    "fma.mask.vfnmadd.ps.512" => "__builtin_ia32_vfnmaddps512_mask",
-                    "fma.mask.vfnmsub.pd.512" => "__builtin_ia32_vfnmsubpd512_mask",
-                    "fma.mask.vfnmsub.ps.512" => "__builtin_ia32_vfnmsubps512_mask",
-                    "fma.vfmadd.pd" => "__builtin_ia32_vfmaddpd",
-                    "fma.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256",
-                    "fma.vfmadd.ps" => "__builtin_ia32_vfmaddps",
-                    "fma.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256",
-                    "fma.vfmadd.sd" => "__builtin_ia32_vfmaddsd",
-                    "fma.vfmadd.ss" => "__builtin_ia32_vfmaddss",
                     "fma.vfmaddsub.pd" => "__builtin_ia32_vfmaddsubpd",
                     "fma.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256",
                     "fma.vfmaddsub.ps" => "__builtin_ia32_vfmaddsubps",
                     "fma.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256",
-                    "fma.vfmsub.pd" => "__builtin_ia32_vfmsubpd",
-                    "fma.vfmsub.pd.256" => "__builtin_ia32_vfmsubpd256",
-                    "fma.vfmsub.ps" => "__builtin_ia32_vfmsubps",
-                    "fma.vfmsub.ps.256" => "__builtin_ia32_vfmsubps256",
-                    "fma.vfmsub.sd" => "__builtin_ia32_vfmsubsd",
-                    "fma.vfmsub.ss" => "__builtin_ia32_vfmsubss",
-                    "fma.vfmsubadd.pd" => "__builtin_ia32_vfmsubaddpd",
-                    "fma.vfmsubadd.pd.256" => "__builtin_ia32_vfmsubaddpd256",
-                    "fma.vfmsubadd.ps" => "__builtin_ia32_vfmsubaddps",
-                    "fma.vfmsubadd.ps.256" => "__builtin_ia32_vfmsubaddps256",
-                    "fma.vfnmadd.pd" => "__builtin_ia32_vfnmaddpd",
-                    "fma.vfnmadd.pd.256" => "__builtin_ia32_vfnmaddpd256",
-                    "fma.vfnmadd.ps" => "__builtin_ia32_vfnmaddps",
-                    "fma.vfnmadd.ps.256" => "__builtin_ia32_vfnmaddps256",
-                    "fma.vfnmadd.sd" => "__builtin_ia32_vfnmaddsd",
-                    "fma.vfnmadd.ss" => "__builtin_ia32_vfnmaddss",
-                    "fma.vfnmsub.pd" => "__builtin_ia32_vfnmsubpd",
-                    "fma.vfnmsub.pd.256" => "__builtin_ia32_vfnmsubpd256",
-                    "fma.vfnmsub.ps" => "__builtin_ia32_vfnmsubps",
-                    "fma.vfnmsub.ps.256" => "__builtin_ia32_vfnmsubps256",
-                    "fma.vfnmsub.sd" => "__builtin_ia32_vfnmsubsd",
-                    "fma.vfnmsub.ss" => "__builtin_ia32_vfnmsubss",
                     "fxrstor" => "__builtin_ia32_fxrstor",
                     "fxrstor64" => "__builtin_ia32_fxrstor64",
                     "fxsave" => "__builtin_ia32_fxsave",
@@ -9988,7 +8685,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "lwpval32" => "__builtin_ia32_lwpval32",
                     "lwpval64" => "__builtin_ia32_lwpval64",
                     "mmx.emms" => "__builtin_ia32_emms",
-                    "mmx.femms" => "__builtin_ia32_femms",
                     "monitorx" => "__builtin_ia32_monitorx",
                     "movdir64b" => "__builtin_ia32_movdir64b",
                     "movrsdi" => "__builtin_ia32_movrsdi",
@@ -10013,7 +8709,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "rdsspd" => "__builtin_ia32_rdsspd",
                     "rdsspq" => "__builtin_ia32_rdsspq",
                     "rdtsc" => "__builtin_ia32_rdtsc",
-                    "rdtscp" => "__builtin_ia32_rdtscp",
                     "rstorssp" => "__builtin_ia32_rstorssp",
                     "saveprevssp" => "__builtin_ia32_saveprevssp",
                     "senduipi" => "__builtin_ia32_senduipi",
@@ -10027,8 +8722,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sha256msg2" => "__builtin_ia32_sha256msg2",
                     "sha256rnds2" => "__builtin_ia32_sha256rnds2",
                     "slwpcb" => "__builtin_ia32_slwpcb",
-                    "sse.add.ss" => "__builtin_ia32_addss",
-                    "sse.cmp.ps" => "__builtin_ia32_cmpps",
                     "sse.cmp.ss" => "__builtin_ia32_cmpss",
                     "sse.comieq.ss" => "__builtin_ia32_comieq",
                     "sse.comige.ss" => "__builtin_ia32_comige",
@@ -10036,37 +8729,27 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse.comile.ss" => "__builtin_ia32_comile",
                     "sse.comilt.ss" => "__builtin_ia32_comilt",
                     "sse.comineq.ss" => "__builtin_ia32_comineq",
-                    "sse.cvtsi2ss" => "__builtin_ia32_cvtsi2ss",
-                    "sse.cvtsi642ss" => "__builtin_ia32_cvtsi642ss",
                     "sse.cvtss2si" => "__builtin_ia32_cvtss2si",
                     "sse.cvtss2si64" => "__builtin_ia32_cvtss2si64",
                     "sse.cvttss2si" => "__builtin_ia32_cvttss2si",
                     "sse.cvttss2si64" => "__builtin_ia32_cvttss2si64",
-                    "sse.div.ss" => "__builtin_ia32_divss",
                     "sse.max.ps" => "__builtin_ia32_maxps",
                     "sse.max.ss" => "__builtin_ia32_maxss",
                     "sse.min.ps" => "__builtin_ia32_minps",
                     "sse.min.ss" => "__builtin_ia32_minss",
                     "sse.movmsk.ps" => "__builtin_ia32_movmskps",
-                    "sse.mul.ss" => "__builtin_ia32_mulss",
                     "sse.rcp.ps" => "__builtin_ia32_rcpps",
                     "sse.rcp.ss" => "__builtin_ia32_rcpss",
                     "sse.rsqrt.ps" => "__builtin_ia32_rsqrtps",
                     "sse.rsqrt.ss" => "__builtin_ia32_rsqrtss",
                     "sse.sfence" => "__builtin_ia32_sfence",
-                    "sse.sqrt.ps" => "__builtin_ia32_sqrtps",
-                    "sse.sqrt.ss" => "__builtin_ia32_sqrtss",
-                    "sse.storeu.ps" => "__builtin_ia32_storeups",
-                    "sse.sub.ss" => "__builtin_ia32_subss",
                     "sse.ucomieq.ss" => "__builtin_ia32_ucomieq",
                     "sse.ucomige.ss" => "__builtin_ia32_ucomige",
                     "sse.ucomigt.ss" => "__builtin_ia32_ucomigt",
                     "sse.ucomile.ss" => "__builtin_ia32_ucomile",
                     "sse.ucomilt.ss" => "__builtin_ia32_ucomilt",
                     "sse.ucomineq.ss" => "__builtin_ia32_ucomineq",
-                    "sse2.add.sd" => "__builtin_ia32_addsd",
                     "sse2.clflush" => "__builtin_ia32_clflush",
-                    "sse2.cmp.pd" => "__builtin_ia32_cmppd",
                     "sse2.cmp.sd" => "__builtin_ia32_cmpsd",
                     "sse2.comieq.sd" => "__builtin_ia32_comisdeq",
                     "sse2.comige.sd" => "__builtin_ia32_comisdge",
@@ -10074,23 +8757,16 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse2.comile.sd" => "__builtin_ia32_comisdle",
                     "sse2.comilt.sd" => "__builtin_ia32_comisdlt",
                     "sse2.comineq.sd" => "__builtin_ia32_comisdneq",
-                    "sse2.cvtdq2pd" => "__builtin_ia32_cvtdq2pd",
-                    "sse2.cvtdq2ps" => "__builtin_ia32_cvtdq2ps",
                     "sse2.cvtpd2dq" => "__builtin_ia32_cvtpd2dq",
                     "sse2.cvtpd2ps" => "__builtin_ia32_cvtpd2ps",
                     "sse2.cvtps2dq" => "__builtin_ia32_cvtps2dq",
-                    "sse2.cvtps2pd" => "__builtin_ia32_cvtps2pd",
                     "sse2.cvtsd2si" => "__builtin_ia32_cvtsd2si",
                     "sse2.cvtsd2si64" => "__builtin_ia32_cvtsd2si64",
                     "sse2.cvtsd2ss" => "__builtin_ia32_cvtsd2ss",
-                    "sse2.cvtsi2sd" => "__builtin_ia32_cvtsi2sd",
-                    "sse2.cvtsi642sd" => "__builtin_ia32_cvtsi642sd",
-                    "sse2.cvtss2sd" => "__builtin_ia32_cvtss2sd",
                     "sse2.cvttpd2dq" => "__builtin_ia32_cvttpd2dq",
                     "sse2.cvttps2dq" => "__builtin_ia32_cvttps2dq",
                     "sse2.cvttsd2si" => "__builtin_ia32_cvttsd2si",
                     "sse2.cvttsd2si64" => "__builtin_ia32_cvttsd2si64",
-                    "sse2.div.sd" => "__builtin_ia32_divsd",
                     "sse2.lfence" => "__builtin_ia32_lfence",
                     "sse2.maskmov.dqu" => "__builtin_ia32_maskmovdqu",
                     "sse2.max.pd" => "__builtin_ia32_maxpd",
@@ -10099,33 +8775,18 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse2.min.pd" => "__builtin_ia32_minpd",
                     "sse2.min.sd" => "__builtin_ia32_minsd",
                     "sse2.movmsk.pd" => "__builtin_ia32_movmskpd",
-                    "sse2.mul.sd" => "__builtin_ia32_mulsd",
                     "sse2.packssdw.128" => "__builtin_ia32_packssdw128",
                     "sse2.packsswb.128" => "__builtin_ia32_packsswb128",
                     "sse2.packuswb.128" => "__builtin_ia32_packuswb128",
-                    "sse2.padds.b" => "__builtin_ia32_paddsb128",
-                    "sse2.padds.w" => "__builtin_ia32_paddsw128",
-                    "sse2.paddus.b" => "__builtin_ia32_paddusb128",
-                    "sse2.paddus.w" => "__builtin_ia32_paddusw128",
                     "sse2.pause" => "__builtin_ia32_pause",
                     "sse2.pavg.b" => "__builtin_ia32_pavgb128",
                     "sse2.pavg.w" => "__builtin_ia32_pavgw128",
                     "sse2.pmadd.wd" => "__builtin_ia32_pmaddwd128",
-                    "sse2.pmaxs.w" => "__builtin_ia32_pmaxsw128",
-                    "sse2.pmaxu.b" => "__builtin_ia32_pmaxub128",
-                    "sse2.pmins.w" => "__builtin_ia32_pminsw128",
-                    "sse2.pminu.b" => "__builtin_ia32_pminub128",
                     "sse2.pmovmskb.128" => "__builtin_ia32_pmovmskb128",
                     "sse2.pmulh.w" => "__builtin_ia32_pmulhw128",
                     "sse2.pmulhu.w" => "__builtin_ia32_pmulhuw128",
-                    "sse2.pmulu.dq" => "__builtin_ia32_pmuludq128",
                     "sse2.psad.bw" => "__builtin_ia32_psadbw128",
-                    "sse2.pshuf.d" => "__builtin_ia32_pshufd",
-                    "sse2.pshufh.w" => "__builtin_ia32_pshufhw",
-                    "sse2.pshufl.w" => "__builtin_ia32_pshuflw",
                     "sse2.psll.d" => "__builtin_ia32_pslld128",
-                    "sse2.psll.dq" => "__builtin_ia32_pslldqi128",
-                    "sse2.psll.dq.bs" => "__builtin_ia32_pslldqi128_byteshift",
                     "sse2.psll.q" => "__builtin_ia32_psllq128",
                     "sse2.psll.w" => "__builtin_ia32_psllw128",
                     "sse2.pslli.d" => "__builtin_ia32_pslldi128",
@@ -10136,23 +8797,11 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse2.psrai.d" => "__builtin_ia32_psradi128",
                     "sse2.psrai.w" => "__builtin_ia32_psrawi128",
                     "sse2.psrl.d" => "__builtin_ia32_psrld128",
-                    "sse2.psrl.dq" => "__builtin_ia32_psrldqi128",
-                    "sse2.psrl.dq.bs" => "__builtin_ia32_psrldqi128_byteshift",
                     "sse2.psrl.q" => "__builtin_ia32_psrlq128",
                     "sse2.psrl.w" => "__builtin_ia32_psrlw128",
                     "sse2.psrli.d" => "__builtin_ia32_psrldi128",
                     "sse2.psrli.q" => "__builtin_ia32_psrlqi128",
                     "sse2.psrli.w" => "__builtin_ia32_psrlwi128",
-                    "sse2.psubs.b" => "__builtin_ia32_psubsb128",
-                    "sse2.psubs.w" => "__builtin_ia32_psubsw128",
-                    "sse2.psubus.b" => "__builtin_ia32_psubusb128",
-                    "sse2.psubus.w" => "__builtin_ia32_psubusw128",
-                    "sse2.sqrt.pd" => "__builtin_ia32_sqrtpd",
-                    "sse2.sqrt.sd" => "__builtin_ia32_sqrtsd",
-                    "sse2.storel.dq" => "__builtin_ia32_storelv4si",
-                    "sse2.storeu.dq" => "__builtin_ia32_storedqu",
-                    "sse2.storeu.pd" => "__builtin_ia32_storeupd",
-                    "sse2.sub.sd" => "__builtin_ia32_subsd",
                     "sse2.ucomieq.sd" => "__builtin_ia32_ucomisdeq",
                     "sse2.ucomige.sd" => "__builtin_ia32_ucomisdge",
                     "sse2.ucomigt.sd" => "__builtin_ia32_ucomisdgt",
@@ -10168,41 +8817,15 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse3.ldu.dq" => "__builtin_ia32_lddqu",
                     "sse3.monitor" => "__builtin_ia32_monitor",
                     "sse3.mwait" => "__builtin_ia32_mwait",
-                    "sse41.blendpd" => "__builtin_ia32_blendpd",
-                    "sse41.blendps" => "__builtin_ia32_blendps",
                     "sse41.blendvpd" => "__builtin_ia32_blendvpd",
                     "sse41.blendvps" => "__builtin_ia32_blendvps",
                     "sse41.dppd" => "__builtin_ia32_dppd",
                     "sse41.dpps" => "__builtin_ia32_dpps",
-                    "sse41.extractps" => "__builtin_ia32_extractps128",
                     "sse41.insertps" => "__builtin_ia32_insertps128",
-                    "sse41.movntdqa" => "__builtin_ia32_movntdqa",
                     "sse41.mpsadbw" => "__builtin_ia32_mpsadbw128",
                     "sse41.packusdw" => "__builtin_ia32_packusdw128",
                     "sse41.pblendvb" => "__builtin_ia32_pblendvb128",
-                    "sse41.pblendw" => "__builtin_ia32_pblendw128",
                     "sse41.phminposuw" => "__builtin_ia32_phminposuw128",
-                    "sse41.pmaxsb" => "__builtin_ia32_pmaxsb128",
-                    "sse41.pmaxsd" => "__builtin_ia32_pmaxsd128",
-                    "sse41.pmaxud" => "__builtin_ia32_pmaxud128",
-                    "sse41.pmaxuw" => "__builtin_ia32_pmaxuw128",
-                    "sse41.pminsb" => "__builtin_ia32_pminsb128",
-                    "sse41.pminsd" => "__builtin_ia32_pminsd128",
-                    "sse41.pminud" => "__builtin_ia32_pminud128",
-                    "sse41.pminuw" => "__builtin_ia32_pminuw128",
-                    "sse41.pmovsxbd" => "__builtin_ia32_pmovsxbd128",
-                    "sse41.pmovsxbq" => "__builtin_ia32_pmovsxbq128",
-                    "sse41.pmovsxbw" => "__builtin_ia32_pmovsxbw128",
-                    "sse41.pmovsxdq" => "__builtin_ia32_pmovsxdq128",
-                    "sse41.pmovsxwd" => "__builtin_ia32_pmovsxwd128",
-                    "sse41.pmovsxwq" => "__builtin_ia32_pmovsxwq128",
-                    "sse41.pmovzxbd" => "__builtin_ia32_pmovzxbd128",
-                    "sse41.pmovzxbq" => "__builtin_ia32_pmovzxbq128",
-                    "sse41.pmovzxbw" => "__builtin_ia32_pmovzxbw128",
-                    "sse41.pmovzxdq" => "__builtin_ia32_pmovzxdq128",
-                    "sse41.pmovzxwd" => "__builtin_ia32_pmovzxwd128",
-                    "sse41.pmovzxwq" => "__builtin_ia32_pmovzxwq128",
-                    "sse41.pmuldq" => "__builtin_ia32_pmuldq128",
                     "sse41.ptestc" => "__builtin_ia32_ptestc128",
                     "sse41.ptestnzc" => "__builtin_ia32_ptestnzc128",
                     "sse41.ptestz" => "__builtin_ia32_ptestz128",
@@ -10232,11 +8855,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse4a.extrqi" => "__builtin_ia32_extrqi",
                     "sse4a.insertq" => "__builtin_ia32_insertq",
                     "sse4a.insertqi" => "__builtin_ia32_insertqi",
-                    "sse4a.movnt.sd" => "__builtin_ia32_movntsd",
-                    "sse4a.movnt.ss" => "__builtin_ia32_movntss",
-                    "ssse3.pabs.b.128" => "__builtin_ia32_pabsb128",
-                    "ssse3.pabs.d.128" => "__builtin_ia32_pabsd128",
-                    "ssse3.pabs.w.128" => "__builtin_ia32_pabsw128",
                     "ssse3.phadd.d.128" => "__builtin_ia32_phaddd128",
                     "ssse3.phadd.sw.128" => "__builtin_ia32_phaddsw128",
                     "ssse3.phadd.w.128" => "__builtin_ia32_phaddw128",
@@ -10251,8 +8869,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "ssse3.psign.w.128" => "__builtin_ia32_psignw128",
                     "sttilecfg" => "__builtin_ia32_tile_storeconfig",
                     "stui" => "__builtin_ia32_stui",
-                    "subborrow.u32" => "__builtin_ia32_subborrow_u32",
-                    "subborrow.u64" => "__builtin_ia32_subborrow_u64",
                     "t2rpntlvwz0rs" => "__builtin_ia32_t2rpntlvwz0rs",
                     "t2rpntlvwz0rst1" => "__builtin_ia32_t2rpntlvwz0rst1",
                     "t2rpntlvwz1rs" => "__builtin_ia32_t2rpntlvwz1rs",
@@ -10330,8 +8946,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "vcvtneoph2ps256" => "__builtin_ia32_vcvtneoph2ps256",
                     "vcvtneps2bf16128" => "__builtin_ia32_vcvtneps2bf16128",
                     "vcvtneps2bf16256" => "__builtin_ia32_vcvtneps2bf16256",
-                    "vcvtph2ps.128" => "__builtin_ia32_vcvtph2ps",
-                    "vcvtph2ps.256" => "__builtin_ia32_vcvtph2ps256",
                     "vcvtps2ph.128" => "__builtin_ia32_vcvtps2ph",
                     "vcvtps2ph.256" => "__builtin_ia32_vcvtps2ph256",
                     "vgf2p8affineinvqb.128" => "__builtin_ia32_vgf2p8affineinvqb_v16qi",
@@ -10375,16 +8989,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "xop.vfrcz.ps.256" => "__builtin_ia32_vfrczps256",
                     "xop.vfrcz.sd" => "__builtin_ia32_vfrczsd",
                     "xop.vfrcz.ss" => "__builtin_ia32_vfrczss",
-                    "xop.vpcmov" => "__builtin_ia32_vpcmov",
-                    "xop.vpcmov.256" => "__builtin_ia32_vpcmov_256",
-                    "xop.vpcomb" => "__builtin_ia32_vpcomb",
-                    "xop.vpcomd" => "__builtin_ia32_vpcomd",
-                    "xop.vpcomq" => "__builtin_ia32_vpcomq",
-                    "xop.vpcomub" => "__builtin_ia32_vpcomub",
-                    "xop.vpcomud" => "__builtin_ia32_vpcomud",
-                    "xop.vpcomuq" => "__builtin_ia32_vpcomuq",
-                    "xop.vpcomuw" => "__builtin_ia32_vpcomuw",
-                    "xop.vpcomw" => "__builtin_ia32_vpcomw",
                     "xop.vpermil2pd" => "__builtin_ia32_vpermil2pd",
                     "xop.vpermil2pd.256" => "__builtin_ia32_vpermil2pd256",
                     "xop.vpermil2ps" => "__builtin_ia32_vpermil2ps",
@@ -10417,14 +9021,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "xop.vpmadcsswd" => "__builtin_ia32_vpmadcsswd",
                     "xop.vpmadcswd" => "__builtin_ia32_vpmadcswd",
                     "xop.vpperm" => "__builtin_ia32_vpperm",
-                    "xop.vprotb" => "__builtin_ia32_vprotb",
-                    "xop.vprotbi" => "__builtin_ia32_vprotbi",
-                    "xop.vprotd" => "__builtin_ia32_vprotd",
-                    "xop.vprotdi" => "__builtin_ia32_vprotdi",
-                    "xop.vprotq" => "__builtin_ia32_vprotq",
-                    "xop.vprotqi" => "__builtin_ia32_vprotqi",
-                    "xop.vprotw" => "__builtin_ia32_vprotw",
-                    "xop.vprotwi" => "__builtin_ia32_vprotwi",
                     "xop.vpshab" => "__builtin_ia32_vpshab",
                     "xop.vpshad" => "__builtin_ia32_vpshad",
                     "xop.vpshaq" => "__builtin_ia32_vpshaq",
@@ -10454,6 +9050,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
             }
             xcore(name, full_name)
         }
-        _ => unimplemented!("***** unsupported LLVM architecture {arch}, intrinsic:{full_name}"),
+        _ => match old_arch_res {
+            ArchCheckResult::UnknownIntrinsic => {
+                unimplemented!("***** unsupported LLVM intrinsic {full_name}")
+            }
+            ArchCheckResult::UnknownArch => {
+                unimplemented!("***** unsupported LLVM architecture {arch}, intrinsic: {full_name}")
+            }
+            ArchCheckResult::Ok(_) => unreachable!(),
+        },
     }
 }
diff --git a/src/intrinsic/llvm.rs b/src/intrinsic/llvm.rs
index e3d189c95ced..72fc72d118b7 100644
--- a/src/intrinsic/llvm.rs
+++ b/src/intrinsic/llvm.rs
@@ -1604,5 +1604,7 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
     func
 }
 
+#[cfg(feature = "master")]
+include!("old_archs.rs");
 #[cfg(feature = "master")]
 include!("archs.rs");
diff --git a/src/intrinsic/old_archs.rs b/src/intrinsic/old_archs.rs
new file mode 100644
index 000000000000..3a59707b2ebe
--- /dev/null
+++ b/src/intrinsic/old_archs.rs
@@ -0,0 +1,1384 @@
+// All these translations used to be automatically generated in the past. However, since they never
+// changed, we decided to simplify the translation generation script to move them directly into this
+// file.
+//
+// So in short: avoid editing this file.
+
+pub(crate) enum ArchCheckResult {
+    Ok(&'static str),
+    UnknownIntrinsic,
+    UnknownArch,
+}
+
+pub(crate) fn old_archs(arch: &str, name: &str) -> ArchCheckResult {
+    ArchCheckResult::Ok(match arch {
+        "AMDGPU" => match name {
+            "div.fixup.f32" => "__builtin_amdgpu_div_fixup",
+            "div.fixup.f64" => "__builtin_amdgpu_div_fixup",
+            "div.fixup.v2f64" => "__builtin_amdgpu_div_fixup",
+            "div.fixup.v4f32" => "__builtin_amdgpu_div_fixup",
+            "div.fmas.f32" => "__builtin_amdgpu_div_fmas",
+            "div.fmas.f64" => "__builtin_amdgpu_div_fmas",
+            "div.fmas.v2f64" => "__builtin_amdgpu_div_fmas",
+            "div.fmas.v4f32" => "__builtin_amdgpu_div_fmas",
+            "ldexp.f32" => "__builtin_amdgpu_ldexp",
+            "ldexp.f64" => "__builtin_amdgpu_ldexp",
+            "ldexp.v2f64" => "__builtin_amdgpu_ldexp",
+            "ldexp.v4f32" => "__builtin_amdgpu_ldexp",
+            "rcp.f32" => "__builtin_amdgpu_rcp",
+            "rcp.f64" => "__builtin_amdgpu_rcp",
+            "rcp.v2f64" => "__builtin_amdgpu_rcp",
+            "rcp.v4f32" => "__builtin_amdgpu_rcp",
+            "rsq.clamped.f32" => "__builtin_amdgpu_rsq_clamped",
+            "rsq.clamped.f64" => "__builtin_amdgpu_rsq_clamped",
+            "rsq.clamped.v2f64" => "__builtin_amdgpu_rsq_clamped",
+            "rsq.clamped.v4f32" => "__builtin_amdgpu_rsq_clamped",
+            "rsq.f32" => "__builtin_amdgpu_rsq",
+            "rsq.f64" => "__builtin_amdgpu_rsq",
+            "rsq.v2f64" => "__builtin_amdgpu_rsq",
+            "rsq.v4f32" => "__builtin_amdgpu_rsq",
+            "trig.preop.f32" => "__builtin_amdgpu_trig_preop",
+            "trig.preop.f64" => "__builtin_amdgpu_trig_preop",
+            "trig.preop.v2f64" => "__builtin_amdgpu_trig_preop",
+            "trig.preop.v4f32" => "__builtin_amdgpu_trig_preop",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "arm" => match name {
+            "mcrr" => "__builtin_arm_mcrr",
+            "mcrr2" => "__builtin_arm_mcrr2",
+            "thread.pointer" => "__builtin_thread_pointer",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "cuda" => match name {
+            "syncthreads" => "__syncthreads",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "hexagon" => match name {
+            "F2.dffixupd" => "__builtin_HEXAGON_F2_dffixupd",
+            "F2.dffixupn" => "__builtin_HEXAGON_F2_dffixupn",
+            "F2.dffixupr" => "__builtin_HEXAGON_F2_dffixupr",
+            "F2.dffma" => "__builtin_HEXAGON_F2_dffma",
+            "F2.dffma.lib" => "__builtin_HEXAGON_F2_dffma_lib",
+            "F2.dffma.sc" => "__builtin_HEXAGON_F2_dffma_sc",
+            "F2.dffms" => "__builtin_HEXAGON_F2_dffms",
+            "F2.dffms.lib" => "__builtin_HEXAGON_F2_dffms_lib",
+            "F2.dfmpy" => "__builtin_HEXAGON_F2_dfmpy",
+            "S2.cabacencbin" => "__builtin_HEXAGON_S2_cabacencbin",
+            "SI.to.SXTHI.asrh" => "__builtin_SI_to_SXTHI_asrh",
+            "V6.vlutb" => "__builtin_HEXAGON_V6_vlutb",
+            "V6.vlutb.128B" => "__builtin_HEXAGON_V6_vlutb_128B",
+            "V6.vlutb.acc" => "__builtin_HEXAGON_V6_vlutb_acc",
+            "V6.vlutb.acc.128B" => "__builtin_HEXAGON_V6_vlutb_acc_128B",
+            "V6.vlutb.dv" => "__builtin_HEXAGON_V6_vlutb_dv",
+            "V6.vlutb.dv.128B" => "__builtin_HEXAGON_V6_vlutb_dv_128B",
+            "V6.vlutb.dv.acc" => "__builtin_HEXAGON_V6_vlutb_dv_acc",
+            "V6.vlutb.dv.acc.128B" => "__builtin_HEXAGON_V6_vlutb_dv_acc_128B",
+            "brev.ldb" => "__builtin_brev_ldb",
+            "brev.ldd" => "__builtin_brev_ldd",
+            "brev.ldh" => "__builtin_brev_ldh",
+            "brev.ldub" => "__builtin_brev_ldub",
+            "brev.lduh" => "__builtin_brev_lduh",
+            "brev.ldw" => "__builtin_brev_ldw",
+            "brev.stb" => "__builtin_brev_stb",
+            "brev.std" => "__builtin_brev_std",
+            "brev.sth" => "__builtin_brev_sth",
+            "brev.sthhi" => "__builtin_brev_sthhi",
+            "brev.stw" => "__builtin_brev_stw",
+            "mm256i.vaddw" => "__builtin__mm256i_vaddw",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "nvvm" => match name {
+            "abs.i" => "__nvvm_abs_i",
+            "abs.ll" => "__nvvm_abs_ll",
+            "bar.sync" => "__nvvm_bar_sync",
+            "barrier0" => "__nvvm_bar0",
+            "bitcast.d2ll" => "__nvvm_bitcast_d2ll",
+            "bitcast.f2i" => "__nvvm_bitcast_f2i",
+            "bitcast.i2f" => "__nvvm_bitcast_i2f",
+            "bitcast.ll2d" => "__nvvm_bitcast_ll2d",
+            "brev32" => "__nvvm_brev32",
+            "brev64" => "__nvvm_brev64",
+            "clz.i" => "__nvvm_clz_i",
+            "clz.ll" => "__nvvm_clz_ll",
+            "ex2.approx.d" => "__nvvm_ex2_approx_d",
+            "ex2.approx.f" => "__nvvm_ex2_approx_f",
+            "ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
+            "fabs.d" => "__nvvm_fabs_d",
+            "fabs.f" => "__nvvm_fabs_f",
+            "fabs.ftz.f" => "__nvvm_fabs_ftz_f",
+            "h2f" => "__nvvm_h2f",
+            "max.i" => "__nvvm_max_i",
+            "max.ll" => "__nvvm_max_ll",
+            "max.ui" => "__nvvm_max_ui",
+            "max.ull" => "__nvvm_max_ull",
+            "min.i" => "__nvvm_min_i",
+            "min.ll" => "__nvvm_min_ll",
+            "min.ui" => "__nvvm_min_ui",
+            "min.ull" => "__nvvm_min_ull",
+            "popc.i" => "__nvvm_popc_i",
+            "popc.ll" => "__nvvm_popc_ll",
+            "rotate.b32" => "__nvvm_rotate_b32",
+            "rotate.b64" => "__nvvm_rotate_b64",
+            "rotate.right.b64" => "__nvvm_rotate_right_b64",
+            "swap.lo.hi.b64" => "__nvvm_swap_lo_hi_b64",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "ppc" => match name {
+            "qpx.qvfabs" => "__builtin_qpx_qvfabs",
+            "qpx.qvfadd" => "__builtin_qpx_qvfadd",
+            "qpx.qvfadds" => "__builtin_qpx_qvfadds",
+            "qpx.qvfcfid" => "__builtin_qpx_qvfcfid",
+            "qpx.qvfcfids" => "__builtin_qpx_qvfcfids",
+            "qpx.qvfcfidu" => "__builtin_qpx_qvfcfidu",
+            "qpx.qvfcfidus" => "__builtin_qpx_qvfcfidus",
+            "qpx.qvfcmpeq" => "__builtin_qpx_qvfcmpeq",
+            "qpx.qvfcmpgt" => "__builtin_qpx_qvfcmpgt",
+            "qpx.qvfcmplt" => "__builtin_qpx_qvfcmplt",
+            "qpx.qvfcpsgn" => "__builtin_qpx_qvfcpsgn",
+            "qpx.qvfctid" => "__builtin_qpx_qvfctid",
+            "qpx.qvfctidu" => "__builtin_qpx_qvfctidu",
+            "qpx.qvfctiduz" => "__builtin_qpx_qvfctiduz",
+            "qpx.qvfctidz" => "__builtin_qpx_qvfctidz",
+            "qpx.qvfctiw" => "__builtin_qpx_qvfctiw",
+            "qpx.qvfctiwu" => "__builtin_qpx_qvfctiwu",
+            "qpx.qvfctiwuz" => "__builtin_qpx_qvfctiwuz",
+            "qpx.qvfctiwz" => "__builtin_qpx_qvfctiwz",
+            "qpx.qvflogical" => "__builtin_qpx_qvflogical",
+            "qpx.qvfmadd" => "__builtin_qpx_qvfmadd",
+            "qpx.qvfmadds" => "__builtin_qpx_qvfmadds",
+            "qpx.qvfmsub" => "__builtin_qpx_qvfmsub",
+            "qpx.qvfmsubs" => "__builtin_qpx_qvfmsubs",
+            "qpx.qvfmul" => "__builtin_qpx_qvfmul",
+            "qpx.qvfmuls" => "__builtin_qpx_qvfmuls",
+            "qpx.qvfnabs" => "__builtin_qpx_qvfnabs",
+            "qpx.qvfneg" => "__builtin_qpx_qvfneg",
+            "qpx.qvfnmadd" => "__builtin_qpx_qvfnmadd",
+            "qpx.qvfnmadds" => "__builtin_qpx_qvfnmadds",
+            "qpx.qvfnmsub" => "__builtin_qpx_qvfnmsub",
+            "qpx.qvfnmsubs" => "__builtin_qpx_qvfnmsubs",
+            "qpx.qvfperm" => "__builtin_qpx_qvfperm",
+            "qpx.qvfre" => "__builtin_qpx_qvfre",
+            "qpx.qvfres" => "__builtin_qpx_qvfres",
+            "qpx.qvfrim" => "__builtin_qpx_qvfrim",
+            "qpx.qvfrin" => "__builtin_qpx_qvfrin",
+            "qpx.qvfrip" => "__builtin_qpx_qvfrip",
+            "qpx.qvfriz" => "__builtin_qpx_qvfriz",
+            "qpx.qvfrsp" => "__builtin_qpx_qvfrsp",
+            "qpx.qvfrsqrte" => "__builtin_qpx_qvfrsqrte",
+            "qpx.qvfrsqrtes" => "__builtin_qpx_qvfrsqrtes",
+            "qpx.qvfsel" => "__builtin_qpx_qvfsel",
+            "qpx.qvfsub" => "__builtin_qpx_qvfsub",
+            "qpx.qvfsubs" => "__builtin_qpx_qvfsubs",
+            "qpx.qvftstnan" => "__builtin_qpx_qvftstnan",
+            "qpx.qvfxmadd" => "__builtin_qpx_qvfxmadd",
+            "qpx.qvfxmadds" => "__builtin_qpx_qvfxmadds",
+            "qpx.qvfxmul" => "__builtin_qpx_qvfxmul",
+            "qpx.qvfxmuls" => "__builtin_qpx_qvfxmuls",
+            "qpx.qvfxxcpnmadd" => "__builtin_qpx_qvfxxcpnmadd",
+            "qpx.qvfxxcpnmadds" => "__builtin_qpx_qvfxxcpnmadds",
+            "qpx.qvfxxmadd" => "__builtin_qpx_qvfxxmadd",
+            "qpx.qvfxxmadds" => "__builtin_qpx_qvfxxmadds",
+            "qpx.qvfxxnpmadd" => "__builtin_qpx_qvfxxnpmadd",
+            "qpx.qvfxxnpmadds" => "__builtin_qpx_qvfxxnpmadds",
+            "qpx.qvgpci" => "__builtin_qpx_qvgpci",
+            "qpx.qvlfcd" => "__builtin_qpx_qvlfcd",
+            "qpx.qvlfcda" => "__builtin_qpx_qvlfcda",
+            "qpx.qvlfcs" => "__builtin_qpx_qvlfcs",
+            "qpx.qvlfcsa" => "__builtin_qpx_qvlfcsa",
+            "qpx.qvlfd" => "__builtin_qpx_qvlfd",
+            "qpx.qvlfda" => "__builtin_qpx_qvlfda",
+            "qpx.qvlfiwa" => "__builtin_qpx_qvlfiwa",
+            "qpx.qvlfiwaa" => "__builtin_qpx_qvlfiwaa",
+            "qpx.qvlfiwz" => "__builtin_qpx_qvlfiwz",
+            "qpx.qvlfiwza" => "__builtin_qpx_qvlfiwza",
+            "qpx.qvlfs" => "__builtin_qpx_qvlfs",
+            "qpx.qvlfsa" => "__builtin_qpx_qvlfsa",
+            "qpx.qvlpcld" => "__builtin_qpx_qvlpcld",
+            "qpx.qvlpcls" => "__builtin_qpx_qvlpcls",
+            "qpx.qvlpcrd" => "__builtin_qpx_qvlpcrd",
+            "qpx.qvlpcrs" => "__builtin_qpx_qvlpcrs",
+            "qpx.qvstfcd" => "__builtin_qpx_qvstfcd",
+            "qpx.qvstfcda" => "__builtin_qpx_qvstfcda",
+            "qpx.qvstfcs" => "__builtin_qpx_qvstfcs",
+            "qpx.qvstfcsa" => "__builtin_qpx_qvstfcsa",
+            "qpx.qvstfd" => "__builtin_qpx_qvstfd",
+            "qpx.qvstfda" => "__builtin_qpx_qvstfda",
+            "qpx.qvstfiw" => "__builtin_qpx_qvstfiw",
+            "qpx.qvstfiwa" => "__builtin_qpx_qvstfiwa",
+            "qpx.qvstfs" => "__builtin_qpx_qvstfs",
+            "qpx.qvstfsa" => "__builtin_qpx_qvstfsa",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "ptx" => match name {
+            "bar.sync" => "__builtin_ptx_bar_sync",
+            "read.clock" => "__builtin_ptx_read_clock",
+            "read.clock64" => "__builtin_ptx_read_clock64",
+            "read.gridid" => "__builtin_ptx_read_gridid",
+            "read.laneid" => "__builtin_ptx_read_laneid",
+            "read.lanemask.eq" => "__builtin_ptx_read_lanemask_eq",
+            "read.lanemask.ge" => "__builtin_ptx_read_lanemask_ge",
+            "read.lanemask.gt" => "__builtin_ptx_read_lanemask_gt",
+            "read.lanemask.le" => "__builtin_ptx_read_lanemask_le",
+            "read.lanemask.lt" => "__builtin_ptx_read_lanemask_lt",
+            "read.nsmid" => "__builtin_ptx_read_nsmid",
+            "read.nwarpid" => "__builtin_ptx_read_nwarpid",
+            "read.pm0" => "__builtin_ptx_read_pm0",
+            "read.pm1" => "__builtin_ptx_read_pm1",
+            "read.pm2" => "__builtin_ptx_read_pm2",
+            "read.pm3" => "__builtin_ptx_read_pm3",
+            "read.smid" => "__builtin_ptx_read_smid",
+            "read.warpid" => "__builtin_ptx_read_warpid",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        "x86" => match name {
+            "addcarry.u32" => "__builtin_ia32_addcarry_u32",
+            "addcarry.u64" => "__builtin_ia32_addcarry_u64",
+            "addcarryx.u32" => "__builtin_ia32_addcarryx_u32",
+            "addcarryx.u64" => "__builtin_ia32_addcarryx_u64",
+            "avx.blend.pd.256" => "__builtin_ia32_blendpd256",
+            "avx.blend.ps.256" => "__builtin_ia32_blendps256",
+            "avx.cmp.pd.256" => "__builtin_ia32_cmppd256",
+            "avx.cmp.ps.256" => "__builtin_ia32_cmpps256",
+            "avx.cvt.ps2.pd.256" => "__builtin_ia32_cvtps2pd256",
+            "avx.cvtdq2.pd.256" => "__builtin_ia32_cvtdq2pd256",
+            "avx.cvtdq2.ps.256" => "__builtin_ia32_cvtdq2ps256",
+            "avx.sqrt.pd.256" => "__builtin_ia32_sqrtpd256",
+            "avx.sqrt.ps.256" => "__builtin_ia32_sqrtps256",
+            "avx.storeu.dq.256" => "__builtin_ia32_storedqu256",
+            "avx.storeu.pd.256" => "__builtin_ia32_storeupd256",
+            "avx.storeu.ps.256" => "__builtin_ia32_storeups256",
+            "avx.vbroadcastf128.pd.256" => "__builtin_ia32_vbroadcastf128_pd256",
+            "avx.vbroadcastf128.ps.256" => "__builtin_ia32_vbroadcastf128_ps256",
+            "avx.vextractf128.pd.256" => "__builtin_ia32_vextractf128_pd256",
+            "avx.vextractf128.ps.256" => "__builtin_ia32_vextractf128_ps256",
+            "avx.vextractf128.si.256" => "__builtin_ia32_vextractf128_si256",
+            "avx.vinsertf128.pd.256" => "__builtin_ia32_vinsertf128_pd256",
+            "avx.vinsertf128.ps.256" => "__builtin_ia32_vinsertf128_ps256",
+            "avx.vinsertf128.si.256" => "__builtin_ia32_vinsertf128_si256",
+            "avx.vperm2f128.pd.256" => "__builtin_ia32_vperm2f128_pd256",
+            "avx.vperm2f128.ps.256" => "__builtin_ia32_vperm2f128_ps256",
+            "avx.vperm2f128.si.256" => "__builtin_ia32_vperm2f128_si256",
+            "avx2.movntdqa" => "__builtin_ia32_movntdqa256",
+            "avx2.pabs.b" => "__builtin_ia32_pabsb256",
+            "avx2.pabs.d" => "__builtin_ia32_pabsd256",
+            "avx2.pabs.w" => "__builtin_ia32_pabsw256",
+            "avx2.padds.b" => "__builtin_ia32_paddsb256",
+            "avx2.padds.w" => "__builtin_ia32_paddsw256",
+            "avx2.paddus.b" => "__builtin_ia32_paddusb256",
+            "avx2.paddus.w" => "__builtin_ia32_paddusw256",
+            "avx2.pblendd.128" => "__builtin_ia32_pblendd128",
+            "avx2.pblendd.256" => "__builtin_ia32_pblendd256",
+            "avx2.pblendw" => "__builtin_ia32_pblendw256",
+            "avx2.pbroadcastb.128" => "__builtin_ia32_pbroadcastb128",
+            "avx2.pbroadcastb.256" => "__builtin_ia32_pbroadcastb256",
+            "avx2.pbroadcastd.128" => "__builtin_ia32_pbroadcastd128",
+            "avx2.pbroadcastd.256" => "__builtin_ia32_pbroadcastd256",
+            "avx2.pbroadcastq.128" => "__builtin_ia32_pbroadcastq128",
+            "avx2.pbroadcastq.256" => "__builtin_ia32_pbroadcastq256",
+            "avx2.pbroadcastw.128" => "__builtin_ia32_pbroadcastw128",
+            "avx2.pbroadcastw.256" => "__builtin_ia32_pbroadcastw256",
+            "avx2.pmaxs.b" => "__builtin_ia32_pmaxsb256",
+            "avx2.pmaxs.d" => "__builtin_ia32_pmaxsd256",
+            "avx2.pmaxs.w" => "__builtin_ia32_pmaxsw256",
+            "avx2.pmaxu.b" => "__builtin_ia32_pmaxub256",
+            "avx2.pmaxu.d" => "__builtin_ia32_pmaxud256",
+            "avx2.pmaxu.w" => "__builtin_ia32_pmaxuw256",
+            "avx2.pmins.b" => "__builtin_ia32_pminsb256",
+            "avx2.pmins.d" => "__builtin_ia32_pminsd256",
+            "avx2.pmins.w" => "__builtin_ia32_pminsw256",
+            "avx2.pminu.b" => "__builtin_ia32_pminub256",
+            "avx2.pminu.d" => "__builtin_ia32_pminud256",
+            "avx2.pminu.w" => "__builtin_ia32_pminuw256",
+            "avx2.pmovsxbd" => "__builtin_ia32_pmovsxbd256",
+            "avx2.pmovsxbq" => "__builtin_ia32_pmovsxbq256",
+            "avx2.pmovsxbw" => "__builtin_ia32_pmovsxbw256",
+            "avx2.pmovsxdq" => "__builtin_ia32_pmovsxdq256",
+            "avx2.pmovsxwd" => "__builtin_ia32_pmovsxwd256",
+            "avx2.pmovsxwq" => "__builtin_ia32_pmovsxwq256",
+            "avx2.pmovzxbd" => "__builtin_ia32_pmovzxbd256",
+            "avx2.pmovzxbq" => "__builtin_ia32_pmovzxbq256",
+            "avx2.pmovzxbw" => "__builtin_ia32_pmovzxbw256",
+            "avx2.pmovzxdq" => "__builtin_ia32_pmovzxdq256",
+            "avx2.pmovzxwd" => "__builtin_ia32_pmovzxwd256",
+            "avx2.pmovzxwq" => "__builtin_ia32_pmovzxwq256",
+            "avx2.pmul.dq" => "__builtin_ia32_pmuldq256",
+            "avx2.pmulu.dq" => "__builtin_ia32_pmuludq256",
+            "avx2.psll.dq" => "__builtin_ia32_pslldqi256",
+            "avx2.psll.dq.bs" => "__builtin_ia32_pslldqi256_byteshift",
+            "avx2.psrl.dq" => "__builtin_ia32_psrldqi256",
+            "avx2.psrl.dq.bs" => "__builtin_ia32_psrldqi256_byteshift",
+            "avx2.psubs.b" => "__builtin_ia32_psubsb256",
+            "avx2.psubs.w" => "__builtin_ia32_psubsw256",
+            "avx2.psubus.b" => "__builtin_ia32_psubusb256",
+            "avx2.psubus.w" => "__builtin_ia32_psubusw256",
+            "avx2.vbroadcast.sd.pd.256" => "__builtin_ia32_vbroadcastsd_pd256",
+            "avx2.vbroadcast.ss.ps" => "__builtin_ia32_vbroadcastss_ps",
+            "avx2.vbroadcast.ss.ps.256" => "__builtin_ia32_vbroadcastss_ps256",
+            "avx2.vextracti128" => "__builtin_ia32_extract128i256",
+            "avx2.vinserti128" => "__builtin_ia32_insert128i256",
+            "avx2.vperm2i128" => "__builtin_ia32_permti256",
+            "avx512.cvtb2mask.128" => "__builtin_ia32_cvtb2mask128",
+            "avx512.cvtb2mask.256" => "__builtin_ia32_cvtb2mask256",
+            "avx512.cvtb2mask.512" => "__builtin_ia32_cvtb2mask512",
+            "avx512.cvtd2mask.128" => "__builtin_ia32_cvtd2mask128",
+            "avx512.cvtd2mask.256" => "__builtin_ia32_cvtd2mask256",
+            "avx512.cvtd2mask.512" => "__builtin_ia32_cvtd2mask512",
+            "avx512.cvtmask2b.128" => "__builtin_ia32_cvtmask2b128",
+            "avx512.cvtmask2b.256" => "__builtin_ia32_cvtmask2b256",
+            "avx512.cvtmask2b.512" => "__builtin_ia32_cvtmask2b512",
+            "avx512.cvtmask2d.128" => "__builtin_ia32_cvtmask2d128",
+            "avx512.cvtmask2d.256" => "__builtin_ia32_cvtmask2d256",
+            "avx512.cvtmask2d.512" => "__builtin_ia32_cvtmask2d512",
+            "avx512.cvtmask2q.128" => "__builtin_ia32_cvtmask2q128",
+            "avx512.cvtmask2q.256" => "__builtin_ia32_cvtmask2q256",
+            "avx512.cvtmask2q.512" => "__builtin_ia32_cvtmask2q512",
+            "avx512.cvtmask2w.128" => "__builtin_ia32_cvtmask2w128",
+            "avx512.cvtmask2w.256" => "__builtin_ia32_cvtmask2w256",
+            "avx512.cvtmask2w.512" => "__builtin_ia32_cvtmask2w512",
+            "avx512.cvtq2mask.128" => "__builtin_ia32_cvtq2mask128",
+            "avx512.cvtq2mask.256" => "__builtin_ia32_cvtq2mask256",
+            "avx512.cvtq2mask.512" => "__builtin_ia32_cvtq2mask512",
+            "avx512.cvtsd2usi" => "__builtin_ia32_cvtsd2usi",
+            "avx512.cvtsd2usi64" => "__builtin_ia32_cvtsd2usi64",
+            "avx512.cvtsi2sd32" => "__builtin_ia32_cvtsi2sd32",
+            "avx512.cvtss2usi" => "__builtin_ia32_cvtss2usi",
+            "avx512.cvtss2usi64" => "__builtin_ia32_cvtss2usi64",
+            "avx512.cvtw2mask.128" => "__builtin_ia32_cvtw2mask128",
+            "avx512.cvtw2mask.256" => "__builtin_ia32_cvtw2mask256",
+            "avx512.cvtw2mask.512" => "__builtin_ia32_cvtw2mask512",
+            "avx512.exp2.pd" => "__builtin_ia32_exp2pd_mask",
+            "avx512.exp2.ps" => "__builtin_ia32_exp2ps_mask",
+            "avx512.gather.dpd.512" => "__builtin_ia32_gathersiv8df",
+            "avx512.gather.dpi.512" => "__builtin_ia32_gathersiv16si",
+            "avx512.gather.dpq.512" => "__builtin_ia32_gathersiv8di",
+            "avx512.gather.dps.512" => "__builtin_ia32_gathersiv16sf",
+            "avx512.gather.qpd.512" => "__builtin_ia32_gatherdiv8df",
+            "avx512.gather.qpi.512" => "__builtin_ia32_gatherdiv16si",
+            "avx512.gather.qpq.512" => "__builtin_ia32_gatherdiv8di",
+            "avx512.gather.qps.512" => "__builtin_ia32_gatherdiv16sf",
+            "avx512.gather3div2.df" => "__builtin_ia32_gather3div2df",
+            "avx512.gather3div2.di" => "__builtin_ia32_gather3div2di",
+            "avx512.gather3div4.df" => "__builtin_ia32_gather3div4df",
+            "avx512.gather3div4.di" => "__builtin_ia32_gather3div4di",
+            "avx512.gather3div4.sf" => "__builtin_ia32_gather3div4sf",
+            "avx512.gather3div4.si" => "__builtin_ia32_gather3div4si",
+            "avx512.gather3div8.sf" => "__builtin_ia32_gather3div8sf",
+            "avx512.gather3div8.si" => "__builtin_ia32_gather3div8si",
+            "avx512.gather3siv2.df" => "__builtin_ia32_gather3siv2df",
+            "avx512.gather3siv2.di" => "__builtin_ia32_gather3siv2di",
+            "avx512.gather3siv4.df" => "__builtin_ia32_gather3siv4df",
+            "avx512.gather3siv4.di" => "__builtin_ia32_gather3siv4di",
+            "avx512.gather3siv4.sf" => "__builtin_ia32_gather3siv4sf",
+            "avx512.gather3siv4.si" => "__builtin_ia32_gather3siv4si",
+            "avx512.gather3siv8.sf" => "__builtin_ia32_gather3siv8sf",
+            "avx512.gather3siv8.si" => "__builtin_ia32_gather3siv8si",
+            "avx512.gatherpf.dpd.512" => "__builtin_ia32_gatherpfdpd",
+            "avx512.gatherpf.dps.512" => "__builtin_ia32_gatherpfdps",
+            "avx512.gatherpf.qpd.512" => "__builtin_ia32_gatherpfqpd",
+            "avx512.gatherpf.qps.512" => "__builtin_ia32_gatherpfqps",
+            "avx512.kand.w" => "__builtin_ia32_kandhi",
+            "avx512.kandn.w" => "__builtin_ia32_kandnhi",
+            "avx512.knot.w" => "__builtin_ia32_knothi",
+            "avx512.kor.w" => "__builtin_ia32_korhi",
+            "avx512.kortestc.w" => "__builtin_ia32_kortestchi",
+            "avx512.kortestz.w" => "__builtin_ia32_kortestzhi",
+            "avx512.kunpck.bw" => "__builtin_ia32_kunpckhi",
+            "avx512.kunpck.dq" => "__builtin_ia32_kunpckdi",
+            "avx512.kunpck.wd" => "__builtin_ia32_kunpcksi",
+            "avx512.kxnor.w" => "__builtin_ia32_kxnorhi",
+            "avx512.kxor.w" => "__builtin_ia32_kxorhi",
+            "avx512.mask.add.pd.128" => "__builtin_ia32_addpd128_mask",
+            "avx512.mask.add.pd.256" => "__builtin_ia32_addpd256_mask",
+            "avx512.mask.add.pd.512" => "__builtin_ia32_addpd512_mask",
+            "avx512.mask.add.ps.128" => "__builtin_ia32_addps128_mask",
+            "avx512.mask.add.ps.256" => "__builtin_ia32_addps256_mask",
+            "avx512.mask.add.ps.512" => "__builtin_ia32_addps512_mask",
+            "avx512.mask.and.pd.128" => "__builtin_ia32_andpd128_mask",
+            "avx512.mask.and.pd.256" => "__builtin_ia32_andpd256_mask",
+            "avx512.mask.and.pd.512" => "__builtin_ia32_andpd512_mask",
+            "avx512.mask.and.ps.128" => "__builtin_ia32_andps128_mask",
+            "avx512.mask.and.ps.256" => "__builtin_ia32_andps256_mask",
+            "avx512.mask.and.ps.512" => "__builtin_ia32_andps512_mask",
+            "avx512.mask.andn.pd.128" => "__builtin_ia32_andnpd128_mask",
+            "avx512.mask.andn.pd.256" => "__builtin_ia32_andnpd256_mask",
+            "avx512.mask.andn.pd.512" => "__builtin_ia32_andnpd512_mask",
+            "avx512.mask.andn.ps.128" => "__builtin_ia32_andnps128_mask",
+            "avx512.mask.andn.ps.256" => "__builtin_ia32_andnps256_mask",
+            "avx512.mask.andn.ps.512" => "__builtin_ia32_andnps512_mask",
+            "avx512.mask.blend.d.512" => "__builtin_ia32_blendmd_512_mask",
+            "avx512.mask.blend.pd.512" => "__builtin_ia32_blendmpd_512_mask",
+            "avx512.mask.blend.ps.512" => "__builtin_ia32_blendmps_512_mask",
+            "avx512.mask.blend.q.512" => "__builtin_ia32_blendmq_512_mask",
+            "avx512.mask.broadcastf32x2.256" => "__builtin_ia32_broadcastf32x2_256_mask",
+            "avx512.mask.broadcastf32x2.512" => "__builtin_ia32_broadcastf32x2_512_mask",
+            "avx512.mask.broadcastf32x4.256" => "__builtin_ia32_broadcastf32x4_256_mask",
+            "avx512.mask.broadcastf32x4.512" => "__builtin_ia32_broadcastf32x4_512",
+            "avx512.mask.broadcastf32x8.512" => "__builtin_ia32_broadcastf32x8_512_mask",
+            "avx512.mask.broadcastf64x2.256" => "__builtin_ia32_broadcastf64x2_256_mask",
+            "avx512.mask.broadcastf64x2.512" => "__builtin_ia32_broadcastf64x2_512_mask",
+            "avx512.mask.broadcastf64x4.512" => "__builtin_ia32_broadcastf64x4_512",
+            "avx512.mask.broadcasti32x2.128" => "__builtin_ia32_broadcasti32x2_128_mask",
+            "avx512.mask.broadcasti32x2.256" => "__builtin_ia32_broadcasti32x2_256_mask",
+            "avx512.mask.broadcasti32x2.512" => "__builtin_ia32_broadcasti32x2_512_mask",
+            "avx512.mask.broadcasti32x4.256" => "__builtin_ia32_broadcasti32x4_256_mask",
+            "avx512.mask.broadcasti32x4.512" => "__builtin_ia32_broadcasti32x4_512",
+            "avx512.mask.broadcasti32x8.512" => "__builtin_ia32_broadcasti32x8_512_mask",
+            "avx512.mask.broadcasti64x2.256" => "__builtin_ia32_broadcasti64x2_256_mask",
+            "avx512.mask.broadcasti64x2.512" => "__builtin_ia32_broadcasti64x2_512_mask",
+            "avx512.mask.broadcasti64x4.512" => "__builtin_ia32_broadcasti64x4_512",
+            "avx512.mask.cmp.pd.128" => "__builtin_ia32_cmppd128_mask",
+            "avx512.mask.cmp.pd.256" => "__builtin_ia32_cmppd256_mask",
+            "avx512.mask.cmp.pd.512" => "__builtin_ia32_cmppd512_mask",
+            "avx512.mask.cmp.ps.128" => "__builtin_ia32_cmpps128_mask",
+            "avx512.mask.cmp.ps.256" => "__builtin_ia32_cmpps256_mask",
+            "avx512.mask.cmp.ps.512" => "__builtin_ia32_cmpps512_mask",
+            "avx512.mask.compress.d.128" => "__builtin_ia32_compresssi128_mask",
+            "avx512.mask.compress.d.256" => "__builtin_ia32_compresssi256_mask",
+            "avx512.mask.compress.d.512" => "__builtin_ia32_compresssi512_mask",
+            "avx512.mask.compress.pd.128" => "__builtin_ia32_compressdf128_mask",
+            "avx512.mask.compress.pd.256" => "__builtin_ia32_compressdf256_mask",
+            "avx512.mask.compress.pd.512" => "__builtin_ia32_compressdf512_mask",
+            "avx512.mask.compress.ps.128" => "__builtin_ia32_compresssf128_mask",
+            "avx512.mask.compress.ps.256" => "__builtin_ia32_compresssf256_mask",
+            "avx512.mask.compress.ps.512" => "__builtin_ia32_compresssf512_mask",
+            "avx512.mask.compress.q.128" => "__builtin_ia32_compressdi128_mask",
+            "avx512.mask.compress.q.256" => "__builtin_ia32_compressdi256_mask",
+            "avx512.mask.compress.q.512" => "__builtin_ia32_compressdi512_mask",
+            "avx512.mask.compress.store.d.128" => "__builtin_ia32_compressstoresi128_mask",
+            "avx512.mask.compress.store.d.256" => "__builtin_ia32_compressstoresi256_mask",
+            "avx512.mask.compress.store.d.512" => "__builtin_ia32_compressstoresi512_mask",
+            "avx512.mask.compress.store.pd.128" => "__builtin_ia32_compressstoredf128_mask",
+            "avx512.mask.compress.store.pd.256" => "__builtin_ia32_compressstoredf256_mask",
+            "avx512.mask.compress.store.pd.512" => "__builtin_ia32_compressstoredf512_mask",
+            "avx512.mask.compress.store.ps.128" => "__builtin_ia32_compressstoresf128_mask",
+            "avx512.mask.compress.store.ps.256" => "__builtin_ia32_compressstoresf256_mask",
+            "avx512.mask.compress.store.ps.512" => "__builtin_ia32_compressstoresf512_mask",
+            "avx512.mask.compress.store.q.128" => "__builtin_ia32_compressstoredi128_mask",
+            "avx512.mask.compress.store.q.256" => "__builtin_ia32_compressstoredi256_mask",
+            "avx512.mask.compress.store.q.512" => "__builtin_ia32_compressstoredi512_mask",
+            "avx512.mask.conflict.d.128" => "__builtin_ia32_vpconflictsi_128_mask",
+            "avx512.mask.conflict.d.256" => "__builtin_ia32_vpconflictsi_256_mask",
+            "avx512.mask.conflict.d.512" => "__builtin_ia32_vpconflictsi_512_mask",
+            "avx512.mask.conflict.q.128" => "__builtin_ia32_vpconflictdi_128_mask",
+            "avx512.mask.conflict.q.256" => "__builtin_ia32_vpconflictdi_256_mask",
+            "avx512.mask.conflict.q.512" => "__builtin_ia32_vpconflictdi_512_mask",
+            "avx512.mask.cvtdq2pd.128" => "__builtin_ia32_cvtdq2pd128_mask",
+            "avx512.mask.cvtdq2pd.256" => "__builtin_ia32_cvtdq2pd256_mask",
+            "avx512.mask.cvtdq2pd.512" => "__builtin_ia32_cvtdq2pd512_mask",
+            "avx512.mask.cvtdq2ps.128" => "__builtin_ia32_cvtdq2ps128_mask",
+            "avx512.mask.cvtdq2ps.256" => "__builtin_ia32_cvtdq2ps256_mask",
+            "avx512.mask.cvtdq2ps.512" => "__builtin_ia32_cvtdq2ps512_mask",
+            "avx512.mask.cvtpd2dq.256" => "__builtin_ia32_cvtpd2dq256_mask",
+            "avx512.mask.cvtps2pd.128" => "__builtin_ia32_cvtps2pd128_mask",
+            "avx512.mask.cvtps2pd.256" => "__builtin_ia32_cvtps2pd256_mask",
+            "avx512.mask.cvtqq2pd.128" => "__builtin_ia32_cvtqq2pd128_mask",
+            "avx512.mask.cvtqq2pd.256" => "__builtin_ia32_cvtqq2pd256_mask",
+            "avx512.mask.cvtqq2pd.512" => "__builtin_ia32_cvtqq2pd512_mask",
+            "avx512.mask.cvtqq2ps.256" => "__builtin_ia32_cvtqq2ps256_mask",
+            "avx512.mask.cvtqq2ps.512" => "__builtin_ia32_cvtqq2ps512_mask",
+            "avx512.mask.cvttpd2dq.256" => "__builtin_ia32_cvttpd2dq256_mask",
+            "avx512.mask.cvttps2dq.128" => "__builtin_ia32_cvttps2dq128_mask",
+            "avx512.mask.cvttps2dq.256" => "__builtin_ia32_cvttps2dq256_mask",
+            "avx512.mask.cvtudq2pd.128" => "__builtin_ia32_cvtudq2pd128_mask",
+            "avx512.mask.cvtudq2pd.256" => "__builtin_ia32_cvtudq2pd256_mask",
+            "avx512.mask.cvtudq2pd.512" => "__builtin_ia32_cvtudq2pd512_mask",
+            "avx512.mask.cvtudq2ps.128" => "__builtin_ia32_cvtudq2ps128_mask",
+            "avx512.mask.cvtudq2ps.256" => "__builtin_ia32_cvtudq2ps256_mask",
+            "avx512.mask.cvtudq2ps.512" => "__builtin_ia32_cvtudq2ps512_mask",
+            "avx512.mask.cvtuqq2pd.128" => "__builtin_ia32_cvtuqq2pd128_mask",
+            "avx512.mask.cvtuqq2pd.256" => "__builtin_ia32_cvtuqq2pd256_mask",
+            "avx512.mask.cvtuqq2pd.512" => "__builtin_ia32_cvtuqq2pd512_mask",
+            "avx512.mask.cvtuqq2ps.256" => "__builtin_ia32_cvtuqq2ps256_mask",
+            "avx512.mask.cvtuqq2ps.512" => "__builtin_ia32_cvtuqq2ps512_mask",
+            "avx512.mask.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128_mask",
+            "avx512.mask.dbpsadbw.256" => "__builtin_ia32_dbpsadbw256_mask",
+            "avx512.mask.dbpsadbw.512" => "__builtin_ia32_dbpsadbw512_mask",
+            "avx512.mask.div.pd.128" => "__builtin_ia32_divpd_mask",
+            "avx512.mask.div.pd.256" => "__builtin_ia32_divpd256_mask",
+            "avx512.mask.div.pd.512" => "__builtin_ia32_divpd512_mask",
+            "avx512.mask.div.ps.128" => "__builtin_ia32_divps_mask",
+            "avx512.mask.div.ps.256" => "__builtin_ia32_divps256_mask",
+            "avx512.mask.div.ps.512" => "__builtin_ia32_divps512_mask",
+            "avx512.mask.expand.d.128" => "__builtin_ia32_expandsi128_mask",
+            "avx512.mask.expand.d.256" => "__builtin_ia32_expandsi256_mask",
+            "avx512.mask.expand.d.512" => "__builtin_ia32_expandsi512_mask",
+            "avx512.mask.expand.load.d.128" => "__builtin_ia32_expandloadsi128_mask",
+            "avx512.mask.expand.load.d.256" => "__builtin_ia32_expandloadsi256_mask",
+            "avx512.mask.expand.load.d.512" => "__builtin_ia32_expandloadsi512_mask",
+            "avx512.mask.expand.load.pd.128" => "__builtin_ia32_expandloaddf128_mask",
+            "avx512.mask.expand.load.pd.256" => "__builtin_ia32_expandloaddf256_mask",
+            "avx512.mask.expand.load.pd.512" => "__builtin_ia32_expandloaddf512_mask",
+            "avx512.mask.expand.load.ps.128" => "__builtin_ia32_expandloadsf128_mask",
+            "avx512.mask.expand.load.ps.256" => "__builtin_ia32_expandloadsf256_mask",
+            "avx512.mask.expand.load.ps.512" => "__builtin_ia32_expandloadsf512_mask",
+            "avx512.mask.expand.load.q.128" => "__builtin_ia32_expandloaddi128_mask",
+            "avx512.mask.expand.load.q.256" => "__builtin_ia32_expandloaddi256_mask",
+            "avx512.mask.expand.load.q.512" => "__builtin_ia32_expandloaddi512_mask",
+            "avx512.mask.expand.pd.128" => "__builtin_ia32_expanddf128_mask",
+            "avx512.mask.expand.pd.256" => "__builtin_ia32_expanddf256_mask",
+            "avx512.mask.expand.pd.512" => "__builtin_ia32_expanddf512_mask",
+            "avx512.mask.expand.ps.128" => "__builtin_ia32_expandsf128_mask",
+            "avx512.mask.expand.ps.256" => "__builtin_ia32_expandsf256_mask",
+            "avx512.mask.expand.ps.512" => "__builtin_ia32_expandsf512_mask",
+            "avx512.mask.expand.q.128" => "__builtin_ia32_expanddi128_mask",
+            "avx512.mask.expand.q.256" => "__builtin_ia32_expanddi256_mask",
+            "avx512.mask.expand.q.512" => "__builtin_ia32_expanddi512_mask",
+            "avx512.mask.fpclass.pd.128" => "__builtin_ia32_fpclasspd128_mask",
+            "avx512.mask.fpclass.pd.256" => "__builtin_ia32_fpclasspd256_mask",
+            "avx512.mask.fpclass.pd.512" => "__builtin_ia32_fpclasspd512_mask",
+            "avx512.mask.fpclass.ps.128" => "__builtin_ia32_fpclassps128_mask",
+            "avx512.mask.fpclass.ps.256" => "__builtin_ia32_fpclassps256_mask",
+            "avx512.mask.fpclass.ps.512" => "__builtin_ia32_fpclassps512_mask",
+            "avx512.mask.insertf32x4.256" => "__builtin_ia32_insertf32x4_256_mask",
+            "avx512.mask.insertf32x4.512" => "__builtin_ia32_insertf32x4_mask",
+            "avx512.mask.insertf32x8.512" => "__builtin_ia32_insertf32x8_mask",
+            "avx512.mask.insertf64x2.256" => "__builtin_ia32_insertf64x2_256_mask",
+            "avx512.mask.insertf64x2.512" => "__builtin_ia32_insertf64x2_512_mask",
+            "avx512.mask.insertf64x4.512" => "__builtin_ia32_insertf64x4_mask",
+            "avx512.mask.inserti32x4.256" => "__builtin_ia32_inserti32x4_256_mask",
+            "avx512.mask.inserti32x4.512" => "__builtin_ia32_inserti32x4_mask",
+            "avx512.mask.inserti32x8.512" => "__builtin_ia32_inserti32x8_mask",
+            "avx512.mask.inserti64x2.256" => "__builtin_ia32_inserti64x2_256_mask",
+            "avx512.mask.inserti64x2.512" => "__builtin_ia32_inserti64x2_512_mask",
+            "avx512.mask.inserti64x4.512" => "__builtin_ia32_inserti64x4_mask",
+            "avx512.mask.loadu.d.512" => "__builtin_ia32_loaddqusi512_mask",
+            "avx512.mask.loadu.pd.512" => "__builtin_ia32_loadupd512_mask",
+            "avx512.mask.loadu.ps.512" => "__builtin_ia32_loadups512_mask",
+            "avx512.mask.loadu.q.512" => "__builtin_ia32_loaddqudi512_mask",
+            "avx512.mask.lzcnt.d.512" => "__builtin_ia32_vplzcntd_512_mask",
+            "avx512.mask.lzcnt.q.512" => "__builtin_ia32_vplzcntq_512_mask",
+            "avx512.mask.max.pd.128" => "__builtin_ia32_maxpd_mask",
+            "avx512.mask.max.pd.256" => "__builtin_ia32_maxpd256_mask",
+            "avx512.mask.max.pd.512" => "__builtin_ia32_maxpd512_mask",
+            "avx512.mask.max.ps.128" => "__builtin_ia32_maxps_mask",
+            "avx512.mask.max.ps.256" => "__builtin_ia32_maxps256_mask",
+            "avx512.mask.max.ps.512" => "__builtin_ia32_maxps512_mask",
+            "avx512.mask.min.pd.128" => "__builtin_ia32_minpd_mask",
+            "avx512.mask.min.pd.256" => "__builtin_ia32_minpd256_mask",
+            "avx512.mask.min.pd.512" => "__builtin_ia32_minpd512_mask",
+            "avx512.mask.min.ps.128" => "__builtin_ia32_minps_mask",
+            "avx512.mask.min.ps.256" => "__builtin_ia32_minps256_mask",
+            "avx512.mask.min.ps.512" => "__builtin_ia32_minps512_mask",
+            "avx512.mask.move.sd" => "__builtin_ia32_movsd_mask",
+            "avx512.mask.move.ss" => "__builtin_ia32_movss_mask",
+            "avx512.mask.mul.pd.128" => "__builtin_ia32_mulpd_mask",
+            "avx512.mask.mul.pd.256" => "__builtin_ia32_mulpd256_mask",
+            "avx512.mask.mul.pd.512" => "__builtin_ia32_mulpd512_mask",
+            "avx512.mask.mul.ps.128" => "__builtin_ia32_mulps_mask",
+            "avx512.mask.mul.ps.256" => "__builtin_ia32_mulps256_mask",
+            "avx512.mask.mul.ps.512" => "__builtin_ia32_mulps512_mask",
+            "avx512.mask.or.pd.128" => "__builtin_ia32_orpd128_mask",
+            "avx512.mask.or.pd.256" => "__builtin_ia32_orpd256_mask",
+            "avx512.mask.or.pd.512" => "__builtin_ia32_orpd512_mask",
+            "avx512.mask.or.ps.128" => "__builtin_ia32_orps128_mask",
+            "avx512.mask.or.ps.256" => "__builtin_ia32_orps256_mask",
+            "avx512.mask.or.ps.512" => "__builtin_ia32_orps512_mask",
+            "avx512.mask.pabs.b.128" => "__builtin_ia32_pabsb128_mask",
+            "avx512.mask.pabs.b.256" => "__builtin_ia32_pabsb256_mask",
+            "avx512.mask.pabs.b.512" => "__builtin_ia32_pabsb512_mask",
+            "avx512.mask.pabs.d.128" => "__builtin_ia32_pabsd128_mask",
+            "avx512.mask.pabs.d.256" => "__builtin_ia32_pabsd256_mask",
+            "avx512.mask.pabs.d.512" => "__builtin_ia32_pabsd512_mask",
+            "avx512.mask.pabs.q.128" => "__builtin_ia32_pabsq128_mask",
+            "avx512.mask.pabs.q.256" => "__builtin_ia32_pabsq256_mask",
+            "avx512.mask.pabs.q.512" => "__builtin_ia32_pabsq512_mask",
+            "avx512.mask.pabs.w.128" => "__builtin_ia32_pabsw128_mask",
+            "avx512.mask.pabs.w.256" => "__builtin_ia32_pabsw256_mask",
+            "avx512.mask.pabs.w.512" => "__builtin_ia32_pabsw512_mask",
+            "avx512.mask.packssdw.128" => "__builtin_ia32_packssdw128_mask",
+            "avx512.mask.packssdw.256" => "__builtin_ia32_packssdw256_mask",
+            "avx512.mask.packssdw.512" => "__builtin_ia32_packssdw512_mask",
+            "avx512.mask.packsswb.128" => "__builtin_ia32_packsswb128_mask",
+            "avx512.mask.packsswb.256" => "__builtin_ia32_packsswb256_mask",
+            "avx512.mask.packsswb.512" => "__builtin_ia32_packsswb512_mask",
+            "avx512.mask.packusdw.128" => "__builtin_ia32_packusdw128_mask",
+            "avx512.mask.packusdw.256" => "__builtin_ia32_packusdw256_mask",
+            "avx512.mask.packusdw.512" => "__builtin_ia32_packusdw512_mask",
+            "avx512.mask.packuswb.128" => "__builtin_ia32_packuswb128_mask",
+            "avx512.mask.packuswb.256" => "__builtin_ia32_packuswb256_mask",
+            "avx512.mask.packuswb.512" => "__builtin_ia32_packuswb512_mask",
+            "avx512.mask.padd.b.128" => "__builtin_ia32_paddb128_mask",
+            "avx512.mask.padd.b.256" => "__builtin_ia32_paddb256_mask",
+            "avx512.mask.padd.b.512" => "__builtin_ia32_paddb512_mask",
+            "avx512.mask.padd.d.128" => "__builtin_ia32_paddd128_mask",
+            "avx512.mask.padd.d.256" => "__builtin_ia32_paddd256_mask",
+            "avx512.mask.padd.d.512" => "__builtin_ia32_paddd512_mask",
+            "avx512.mask.padd.q.128" => "__builtin_ia32_paddq128_mask",
+            "avx512.mask.padd.q.256" => "__builtin_ia32_paddq256_mask",
+            "avx512.mask.padd.q.512" => "__builtin_ia32_paddq512_mask",
+            "avx512.mask.padd.w.128" => "__builtin_ia32_paddw128_mask",
+            "avx512.mask.padd.w.256" => "__builtin_ia32_paddw256_mask",
+            "avx512.mask.padd.w.512" => "__builtin_ia32_paddw512_mask",
+            "avx512.mask.padds.b.128" => "__builtin_ia32_paddsb128_mask",
+            "avx512.mask.padds.b.256" => "__builtin_ia32_paddsb256_mask",
+            "avx512.mask.padds.b.512" => "__builtin_ia32_paddsb512_mask",
+            "avx512.mask.padds.w.128" => "__builtin_ia32_paddsw128_mask",
+            "avx512.mask.padds.w.256" => "__builtin_ia32_paddsw256_mask",
+            "avx512.mask.padds.w.512" => "__builtin_ia32_paddsw512_mask",
+            "avx512.mask.paddus.b.128" => "__builtin_ia32_paddusb128_mask",
+            "avx512.mask.paddus.b.256" => "__builtin_ia32_paddusb256_mask",
+            "avx512.mask.paddus.b.512" => "__builtin_ia32_paddusb512_mask",
+            "avx512.mask.paddus.w.128" => "__builtin_ia32_paddusw128_mask",
+            "avx512.mask.paddus.w.256" => "__builtin_ia32_paddusw256_mask",
+            "avx512.mask.paddus.w.512" => "__builtin_ia32_paddusw512_mask",
+            "avx512.mask.pand.d.512" => "__builtin_ia32_pandd512_mask",
+            "avx512.mask.pand.q.512" => "__builtin_ia32_pandq512_mask",
+            "avx512.mask.pavg.b.128" => "__builtin_ia32_pavgb128_mask",
+            "avx512.mask.pavg.b.256" => "__builtin_ia32_pavgb256_mask",
+            "avx512.mask.pavg.b.512" => "__builtin_ia32_pavgb512_mask",
+            "avx512.mask.pavg.w.128" => "__builtin_ia32_pavgw128_mask",
+            "avx512.mask.pavg.w.256" => "__builtin_ia32_pavgw256_mask",
+            "avx512.mask.pavg.w.512" => "__builtin_ia32_pavgw512_mask",
+            "avx512.mask.pbroadcast.b.gpr.128" => "__builtin_ia32_pbroadcastb128_gpr_mask",
+            "avx512.mask.pbroadcast.b.gpr.256" => "__builtin_ia32_pbroadcastb256_gpr_mask",
+            "avx512.mask.pbroadcast.b.gpr.512" => "__builtin_ia32_pbroadcastb512_gpr_mask",
+            "avx512.mask.pbroadcast.d.gpr.128" => "__builtin_ia32_pbroadcastd128_gpr_mask",
+            "avx512.mask.pbroadcast.d.gpr.256" => "__builtin_ia32_pbroadcastd256_gpr_mask",
+            "avx512.mask.pbroadcast.d.gpr.512" => "__builtin_ia32_pbroadcastd512_gpr_mask",
+            "avx512.mask.pbroadcast.q.gpr.128" => "__builtin_ia32_pbroadcastq128_gpr_mask",
+            "avx512.mask.pbroadcast.q.gpr.256" => "__builtin_ia32_pbroadcastq256_gpr_mask",
+            "avx512.mask.pbroadcast.q.gpr.512" => "__builtin_ia32_pbroadcastq512_gpr_mask",
+            "avx512.mask.pbroadcast.q.mem.512" => "__builtin_ia32_pbroadcastq512_mem_mask",
+            "avx512.mask.pbroadcast.w.gpr.128" => "__builtin_ia32_pbroadcastw128_gpr_mask",
+            "avx512.mask.pbroadcast.w.gpr.256" => "__builtin_ia32_pbroadcastw256_gpr_mask",
+            "avx512.mask.pbroadcast.w.gpr.512" => "__builtin_ia32_pbroadcastw512_gpr_mask",
+            "avx512.mask.pcmpeq.b.128" => "__builtin_ia32_pcmpeqb128_mask",
+            "avx512.mask.pcmpeq.b.256" => "__builtin_ia32_pcmpeqb256_mask",
+            "avx512.mask.pcmpeq.b.512" => "__builtin_ia32_pcmpeqb512_mask",
+            "avx512.mask.pcmpeq.d.128" => "__builtin_ia32_pcmpeqd128_mask",
+            "avx512.mask.pcmpeq.d.256" => "__builtin_ia32_pcmpeqd256_mask",
+            "avx512.mask.pcmpeq.d.512" => "__builtin_ia32_pcmpeqd512_mask",
+            "avx512.mask.pcmpeq.q.128" => "__builtin_ia32_pcmpeqq128_mask",
+            "avx512.mask.pcmpeq.q.256" => "__builtin_ia32_pcmpeqq256_mask",
+            "avx512.mask.pcmpeq.q.512" => "__builtin_ia32_pcmpeqq512_mask",
+            "avx512.mask.pcmpeq.w.128" => "__builtin_ia32_pcmpeqw128_mask",
+            "avx512.mask.pcmpeq.w.256" => "__builtin_ia32_pcmpeqw256_mask",
+            "avx512.mask.pcmpeq.w.512" => "__builtin_ia32_pcmpeqw512_mask",
+            "avx512.mask.pcmpgt.b.128" => "__builtin_ia32_pcmpgtb128_mask",
+            "avx512.mask.pcmpgt.b.256" => "__builtin_ia32_pcmpgtb256_mask",
+            "avx512.mask.pcmpgt.b.512" => "__builtin_ia32_pcmpgtb512_mask",
+            "avx512.mask.pcmpgt.d.128" => "__builtin_ia32_pcmpgtd128_mask",
+            "avx512.mask.pcmpgt.d.256" => "__builtin_ia32_pcmpgtd256_mask",
+            "avx512.mask.pcmpgt.d.512" => "__builtin_ia32_pcmpgtd512_mask",
+            "avx512.mask.pcmpgt.q.128" => "__builtin_ia32_pcmpgtq128_mask",
+            "avx512.mask.pcmpgt.q.256" => "__builtin_ia32_pcmpgtq256_mask",
+            "avx512.mask.pcmpgt.q.512" => "__builtin_ia32_pcmpgtq512_mask",
+            "avx512.mask.pcmpgt.w.128" => "__builtin_ia32_pcmpgtw128_mask",
+            "avx512.mask.pcmpgt.w.256" => "__builtin_ia32_pcmpgtw256_mask",
+            "avx512.mask.pcmpgt.w.512" => "__builtin_ia32_pcmpgtw512_mask",
+            "avx512.mask.permvar.df.256" => "__builtin_ia32_permvardf256_mask",
+            "avx512.mask.permvar.df.512" => "__builtin_ia32_permvardf512_mask",
+            "avx512.mask.permvar.di.256" => "__builtin_ia32_permvardi256_mask",
+            "avx512.mask.permvar.di.512" => "__builtin_ia32_permvardi512_mask",
+            "avx512.mask.permvar.hi.128" => "__builtin_ia32_permvarhi128_mask",
+            "avx512.mask.permvar.hi.256" => "__builtin_ia32_permvarhi256_mask",
+            "avx512.mask.permvar.hi.512" => "__builtin_ia32_permvarhi512_mask",
+            "avx512.mask.permvar.qi.128" => "__builtin_ia32_permvarqi128_mask",
+            "avx512.mask.permvar.qi.256" => "__builtin_ia32_permvarqi256_mask",
+            "avx512.mask.permvar.qi.512" => "__builtin_ia32_permvarqi512_mask",
+            "avx512.mask.permvar.sf.256" => "__builtin_ia32_permvarsf256_mask",
+            "avx512.mask.permvar.sf.512" => "__builtin_ia32_permvarsf512_mask",
+            "avx512.mask.permvar.si.256" => "__builtin_ia32_permvarsi256_mask",
+            "avx512.mask.permvar.si.512" => "__builtin_ia32_permvarsi512_mask",
+            "avx512.mask.pmaddubs.w.128" => "__builtin_ia32_pmaddubsw128_mask",
+            "avx512.mask.pmaddubs.w.256" => "__builtin_ia32_pmaddubsw256_mask",
+            "avx512.mask.pmaddubs.w.512" => "__builtin_ia32_pmaddubsw512_mask",
+            "avx512.mask.pmaddw.d.128" => "__builtin_ia32_pmaddwd128_mask",
+            "avx512.mask.pmaddw.d.256" => "__builtin_ia32_pmaddwd256_mask",
+            "avx512.mask.pmaddw.d.512" => "__builtin_ia32_pmaddwd512_mask",
+            "avx512.mask.pmaxs.b.128" => "__builtin_ia32_pmaxsb128_mask",
+            "avx512.mask.pmaxs.b.256" => "__builtin_ia32_pmaxsb256_mask",
+            "avx512.mask.pmaxs.b.512" => "__builtin_ia32_pmaxsb512_mask",
+            "avx512.mask.pmaxs.d.128" => "__builtin_ia32_pmaxsd128_mask",
+            "avx512.mask.pmaxs.d.256" => "__builtin_ia32_pmaxsd256_mask",
+            "avx512.mask.pmaxs.d.512" => "__builtin_ia32_pmaxsd512_mask",
+            "avx512.mask.pmaxs.q.128" => "__builtin_ia32_pmaxsq128_mask",
+            "avx512.mask.pmaxs.q.256" => "__builtin_ia32_pmaxsq256_mask",
+            "avx512.mask.pmaxs.q.512" => "__builtin_ia32_pmaxsq512_mask",
+            "avx512.mask.pmaxs.w.128" => "__builtin_ia32_pmaxsw128_mask",
+            "avx512.mask.pmaxs.w.256" => "__builtin_ia32_pmaxsw256_mask",
+            "avx512.mask.pmaxs.w.512" => "__builtin_ia32_pmaxsw512_mask",
+            "avx512.mask.pmaxu.b.128" => "__builtin_ia32_pmaxub128_mask",
+            "avx512.mask.pmaxu.b.256" => "__builtin_ia32_pmaxub256_mask",
+            "avx512.mask.pmaxu.b.512" => "__builtin_ia32_pmaxub512_mask",
+            "avx512.mask.pmaxu.d.128" => "__builtin_ia32_pmaxud128_mask",
+            "avx512.mask.pmaxu.d.256" => "__builtin_ia32_pmaxud256_mask",
+            "avx512.mask.pmaxu.d.512" => "__builtin_ia32_pmaxud512_mask",
+            "avx512.mask.pmaxu.q.128" => "__builtin_ia32_pmaxuq128_mask",
+            "avx512.mask.pmaxu.q.256" => "__builtin_ia32_pmaxuq256_mask",
+            "avx512.mask.pmaxu.q.512" => "__builtin_ia32_pmaxuq512_mask",
+            "avx512.mask.pmaxu.w.128" => "__builtin_ia32_pmaxuw128_mask",
+            "avx512.mask.pmaxu.w.256" => "__builtin_ia32_pmaxuw256_mask",
+            "avx512.mask.pmaxu.w.512" => "__builtin_ia32_pmaxuw512_mask",
+            "avx512.mask.pmins.b.128" => "__builtin_ia32_pminsb128_mask",
+            "avx512.mask.pmins.b.256" => "__builtin_ia32_pminsb256_mask",
+            "avx512.mask.pmins.b.512" => "__builtin_ia32_pminsb512_mask",
+            "avx512.mask.pmins.d.128" => "__builtin_ia32_pminsd128_mask",
+            "avx512.mask.pmins.d.256" => "__builtin_ia32_pminsd256_mask",
+            "avx512.mask.pmins.d.512" => "__builtin_ia32_pminsd512_mask",
+            "avx512.mask.pmins.q.128" => "__builtin_ia32_pminsq128_mask",
+            "avx512.mask.pmins.q.256" => "__builtin_ia32_pminsq256_mask",
+            "avx512.mask.pmins.q.512" => "__builtin_ia32_pminsq512_mask",
+            "avx512.mask.pmins.w.128" => "__builtin_ia32_pminsw128_mask",
+            "avx512.mask.pmins.w.256" => "__builtin_ia32_pminsw256_mask",
+            "avx512.mask.pmins.w.512" => "__builtin_ia32_pminsw512_mask",
+            "avx512.mask.pminu.b.128" => "__builtin_ia32_pminub128_mask",
+            "avx512.mask.pminu.b.256" => "__builtin_ia32_pminub256_mask",
+            "avx512.mask.pminu.b.512" => "__builtin_ia32_pminub512_mask",
+            "avx512.mask.pminu.d.128" => "__builtin_ia32_pminud128_mask",
+            "avx512.mask.pminu.d.256" => "__builtin_ia32_pminud256_mask",
+            "avx512.mask.pminu.d.512" => "__builtin_ia32_pminud512_mask",
+            "avx512.mask.pminu.q.128" => "__builtin_ia32_pminuq128_mask",
+            "avx512.mask.pminu.q.256" => "__builtin_ia32_pminuq256_mask",
+            "avx512.mask.pminu.q.512" => "__builtin_ia32_pminuq512_mask",
+            "avx512.mask.pminu.w.128" => "__builtin_ia32_pminuw128_mask",
+            "avx512.mask.pminu.w.256" => "__builtin_ia32_pminuw256_mask",
+            "avx512.mask.pminu.w.512" => "__builtin_ia32_pminuw512_mask",
+            "avx512.mask.pmov.db.512" => "__builtin_ia32_pmovdb512_mask",
+            "avx512.mask.pmov.dw.512" => "__builtin_ia32_pmovdw512_mask",
+            "avx512.mask.pmov.qd.256" => "__builtin_ia32_pmovqd256_mask",
+            "avx512.mask.pmov.qd.512" => "__builtin_ia32_pmovqd512_mask",
+            "avx512.mask.pmov.qw.512" => "__builtin_ia32_pmovqw512_mask",
+            "avx512.mask.pmov.wb.256" => "__builtin_ia32_pmovwb256_mask",
+            "avx512.mask.pmov.wb.512" => "__builtin_ia32_pmovwb512_mask",
+            "avx512.mask.pmovsxb.d.128" => "__builtin_ia32_pmovsxbd128_mask",
+            "avx512.mask.pmovsxb.d.256" => "__builtin_ia32_pmovsxbd256_mask",
+            "avx512.mask.pmovsxb.d.512" => "__builtin_ia32_pmovsxbd512_mask",
+            "avx512.mask.pmovsxb.q.128" => "__builtin_ia32_pmovsxbq128_mask",
+            "avx512.mask.pmovsxb.q.256" => "__builtin_ia32_pmovsxbq256_mask",
+            "avx512.mask.pmovsxb.q.512" => "__builtin_ia32_pmovsxbq512_mask",
+            "avx512.mask.pmovsxb.w.128" => "__builtin_ia32_pmovsxbw128_mask",
+            "avx512.mask.pmovsxb.w.256" => "__builtin_ia32_pmovsxbw256_mask",
+            "avx512.mask.pmovsxb.w.512" => "__builtin_ia32_pmovsxbw512_mask",
+            "avx512.mask.pmovsxd.q.128" => "__builtin_ia32_pmovsxdq128_mask",
+            "avx512.mask.pmovsxd.q.256" => "__builtin_ia32_pmovsxdq256_mask",
+            "avx512.mask.pmovsxd.q.512" => "__builtin_ia32_pmovsxdq512_mask",
+            "avx512.mask.pmovsxw.d.128" => "__builtin_ia32_pmovsxwd128_mask",
+            "avx512.mask.pmovsxw.d.256" => "__builtin_ia32_pmovsxwd256_mask",
+            "avx512.mask.pmovsxw.d.512" => "__builtin_ia32_pmovsxwd512_mask",
+            "avx512.mask.pmovsxw.q.128" => "__builtin_ia32_pmovsxwq128_mask",
+            "avx512.mask.pmovsxw.q.256" => "__builtin_ia32_pmovsxwq256_mask",
+            "avx512.mask.pmovsxw.q.512" => "__builtin_ia32_pmovsxwq512_mask",
+            "avx512.mask.pmovzxb.d.128" => "__builtin_ia32_pmovzxbd128_mask",
+            "avx512.mask.pmovzxb.d.256" => "__builtin_ia32_pmovzxbd256_mask",
+            "avx512.mask.pmovzxb.d.512" => "__builtin_ia32_pmovzxbd512_mask",
+            "avx512.mask.pmovzxb.q.128" => "__builtin_ia32_pmovzxbq128_mask",
+            "avx512.mask.pmovzxb.q.256" => "__builtin_ia32_pmovzxbq256_mask",
+            "avx512.mask.pmovzxb.q.512" => "__builtin_ia32_pmovzxbq512_mask",
+            "avx512.mask.pmovzxb.w.128" => "__builtin_ia32_pmovzxbw128_mask",
+            "avx512.mask.pmovzxb.w.256" => "__builtin_ia32_pmovzxbw256_mask",
+            "avx512.mask.pmovzxb.w.512" => "__builtin_ia32_pmovzxbw512_mask",
+            "avx512.mask.pmovzxd.q.128" => "__builtin_ia32_pmovzxdq128_mask",
+            "avx512.mask.pmovzxd.q.256" => "__builtin_ia32_pmovzxdq256_mask",
+            "avx512.mask.pmovzxd.q.512" => "__builtin_ia32_pmovzxdq512_mask",
+            "avx512.mask.pmovzxw.d.128" => "__builtin_ia32_pmovzxwd128_mask",
+            "avx512.mask.pmovzxw.d.256" => "__builtin_ia32_pmovzxwd256_mask",
+            "avx512.mask.pmovzxw.d.512" => "__builtin_ia32_pmovzxwd512_mask",
+            "avx512.mask.pmovzxw.q.128" => "__builtin_ia32_pmovzxwq128_mask",
+            "avx512.mask.pmovzxw.q.256" => "__builtin_ia32_pmovzxwq256_mask",
+            "avx512.mask.pmovzxw.q.512" => "__builtin_ia32_pmovzxwq512_mask",
+            "avx512.mask.pmul.dq.128" => "__builtin_ia32_pmuldq128_mask",
+            "avx512.mask.pmul.dq.256" => "__builtin_ia32_pmuldq256_mask",
+            "avx512.mask.pmul.dq.512" => "__builtin_ia32_pmuldq512_mask",
+            "avx512.mask.pmul.hr.sw.128" => "__builtin_ia32_pmulhrsw128_mask",
+            "avx512.mask.pmul.hr.sw.256" => "__builtin_ia32_pmulhrsw256_mask",
+            "avx512.mask.pmul.hr.sw.512" => "__builtin_ia32_pmulhrsw512_mask",
+            "avx512.mask.pmulh.w.128" => "__builtin_ia32_pmulhw128_mask",
+            "avx512.mask.pmulh.w.256" => "__builtin_ia32_pmulhw256_mask",
+            "avx512.mask.pmulh.w.512" => "__builtin_ia32_pmulhw512_mask",
+            "avx512.mask.pmulhu.w.128" => "__builtin_ia32_pmulhuw128_mask",
+            "avx512.mask.pmulhu.w.256" => "__builtin_ia32_pmulhuw256_mask",
+            "avx512.mask.pmulhu.w.512" => "__builtin_ia32_pmulhuw512_mask",
+            "avx512.mask.pmull.d.128" => "__builtin_ia32_pmulld128_mask",
+            "avx512.mask.pmull.d.256" => "__builtin_ia32_pmulld256_mask",
+            "avx512.mask.pmull.d.512" => "__builtin_ia32_pmulld512_mask",
+            "avx512.mask.pmull.q.128" => "__builtin_ia32_pmullq128_mask",
+            "avx512.mask.pmull.q.256" => "__builtin_ia32_pmullq256_mask",
+            "avx512.mask.pmull.q.512" => "__builtin_ia32_pmullq512_mask",
+            "avx512.mask.pmull.w.128" => "__builtin_ia32_pmullw128_mask",
+            "avx512.mask.pmull.w.256" => "__builtin_ia32_pmullw256_mask",
+            "avx512.mask.pmull.w.512" => "__builtin_ia32_pmullw512_mask",
+            "avx512.mask.pmultishift.qb.128" => "__builtin_ia32_vpmultishiftqb128_mask",
+            "avx512.mask.pmultishift.qb.256" => "__builtin_ia32_vpmultishiftqb256_mask",
+            "avx512.mask.pmultishift.qb.512" => "__builtin_ia32_vpmultishiftqb512_mask",
+            "avx512.mask.pmulu.dq.128" => "__builtin_ia32_pmuludq128_mask",
+            "avx512.mask.pmulu.dq.256" => "__builtin_ia32_pmuludq256_mask",
+            "avx512.mask.pmulu.dq.512" => "__builtin_ia32_pmuludq512_mask",
+            "avx512.mask.prol.d.128" => "__builtin_ia32_prold128_mask",
+            "avx512.mask.prol.d.256" => "__builtin_ia32_prold256_mask",
+            "avx512.mask.prol.d.512" => "__builtin_ia32_prold512_mask",
+            "avx512.mask.prol.q.128" => "__builtin_ia32_prolq128_mask",
+            "avx512.mask.prol.q.256" => "__builtin_ia32_prolq256_mask",
+            "avx512.mask.prol.q.512" => "__builtin_ia32_prolq512_mask",
+            "avx512.mask.prolv.d.128" => "__builtin_ia32_prolvd128_mask",
+            "avx512.mask.prolv.d.256" => "__builtin_ia32_prolvd256_mask",
+            "avx512.mask.prolv.d.512" => "__builtin_ia32_prolvd512_mask",
+            "avx512.mask.prolv.q.128" => "__builtin_ia32_prolvq128_mask",
+            "avx512.mask.prolv.q.256" => "__builtin_ia32_prolvq256_mask",
+            "avx512.mask.prolv.q.512" => "__builtin_ia32_prolvq512_mask",
+            "avx512.mask.pror.d.128" => "__builtin_ia32_prord128_mask",
+            "avx512.mask.pror.d.256" => "__builtin_ia32_prord256_mask",
+            "avx512.mask.pror.d.512" => "__builtin_ia32_prord512_mask",
+            "avx512.mask.pror.q.128" => "__builtin_ia32_prorq128_mask",
+            "avx512.mask.pror.q.256" => "__builtin_ia32_prorq256_mask",
+            "avx512.mask.pror.q.512" => "__builtin_ia32_prorq512_mask",
+            "avx512.mask.prorv.d.128" => "__builtin_ia32_prorvd128_mask",
+            "avx512.mask.prorv.d.256" => "__builtin_ia32_prorvd256_mask",
+            "avx512.mask.prorv.d.512" => "__builtin_ia32_prorvd512_mask",
+            "avx512.mask.prorv.q.128" => "__builtin_ia32_prorvq128_mask",
+            "avx512.mask.prorv.q.256" => "__builtin_ia32_prorvq256_mask",
+            "avx512.mask.prorv.q.512" => "__builtin_ia32_prorvq512_mask",
+            "avx512.mask.pshuf.b.128" => "__builtin_ia32_pshufb128_mask",
+            "avx512.mask.pshuf.b.256" => "__builtin_ia32_pshufb256_mask",
+            "avx512.mask.pshuf.b.512" => "__builtin_ia32_pshufb512_mask",
+            "avx512.mask.psll.d" => "__builtin_ia32_pslld512_mask",
+            "avx512.mask.psll.d.128" => "__builtin_ia32_pslld128_mask",
+            "avx512.mask.psll.d.256" => "__builtin_ia32_pslld256_mask",
+            "avx512.mask.psll.di.128" => "__builtin_ia32_pslldi128_mask",
+            "avx512.mask.psll.di.256" => "__builtin_ia32_pslldi256_mask",
+            "avx512.mask.psll.di.512" => "__builtin_ia32_pslldi512_mask",
+            "avx512.mask.psll.q" => "__builtin_ia32_psllq512_mask",
+            "avx512.mask.psll.q.128" => "__builtin_ia32_psllq128_mask",
+            "avx512.mask.psll.q.256" => "__builtin_ia32_psllq256_mask",
+            "avx512.mask.psll.qi.128" => "__builtin_ia32_psllqi128_mask",
+            "avx512.mask.psll.qi.256" => "__builtin_ia32_psllqi256_mask",
+            "avx512.mask.psll.qi.512" => "__builtin_ia32_psllqi512_mask",
+            "avx512.mask.psll.w.128" => "__builtin_ia32_psllw128_mask",
+            "avx512.mask.psll.w.256" => "__builtin_ia32_psllw256_mask",
+            "avx512.mask.psll.w.512" => "__builtin_ia32_psllw512_mask",
+            "avx512.mask.psll.wi.128" => "__builtin_ia32_psllwi128_mask",
+            "avx512.mask.psll.wi.256" => "__builtin_ia32_psllwi256_mask",
+            "avx512.mask.psll.wi.512" => "__builtin_ia32_psllwi512_mask",
+            "avx512.mask.psllv.d" => "__builtin_ia32_psllv16si_mask",
+            "avx512.mask.psllv.q" => "__builtin_ia32_psllv8di_mask",
+            "avx512.mask.psllv16.hi" => "__builtin_ia32_psllv16hi_mask",
+            "avx512.mask.psllv2.di" => "__builtin_ia32_psllv2di_mask",
+            "avx512.mask.psllv32hi" => "__builtin_ia32_psllv32hi_mask",
+            "avx512.mask.psllv4.di" => "__builtin_ia32_psllv4di_mask",
+            "avx512.mask.psllv4.si" => "__builtin_ia32_psllv4si_mask",
+            "avx512.mask.psllv8.hi" => "__builtin_ia32_psllv8hi_mask",
+            "avx512.mask.psllv8.si" => "__builtin_ia32_psllv8si_mask",
+            "avx512.mask.psra.d" => "__builtin_ia32_psrad512_mask",
+            "avx512.mask.psra.d.128" => "__builtin_ia32_psrad128_mask",
+            "avx512.mask.psra.d.256" => "__builtin_ia32_psrad256_mask",
+            "avx512.mask.psra.di.128" => "__builtin_ia32_psradi128_mask",
+            "avx512.mask.psra.di.256" => "__builtin_ia32_psradi256_mask",
+            "avx512.mask.psra.di.512" => "__builtin_ia32_psradi512_mask",
+            "avx512.mask.psra.q" => "__builtin_ia32_psraq512_mask",
+            "avx512.mask.psra.q.128" => "__builtin_ia32_psraq128_mask",
+            "avx512.mask.psra.q.256" => "__builtin_ia32_psraq256_mask",
+            "avx512.mask.psra.qi.128" => "__builtin_ia32_psraqi128_mask",
+            "avx512.mask.psra.qi.256" => "__builtin_ia32_psraqi256_mask",
+            "avx512.mask.psra.qi.512" => "__builtin_ia32_psraqi512_mask",
+            "avx512.mask.psra.w.128" => "__builtin_ia32_psraw128_mask",
+            "avx512.mask.psra.w.256" => "__builtin_ia32_psraw256_mask",
+            "avx512.mask.psra.w.512" => "__builtin_ia32_psraw512_mask",
+            "avx512.mask.psra.wi.128" => "__builtin_ia32_psrawi128_mask",
+            "avx512.mask.psra.wi.256" => "__builtin_ia32_psrawi256_mask",
+            "avx512.mask.psra.wi.512" => "__builtin_ia32_psrawi512_mask",
+            "avx512.mask.psrav.d" => "__builtin_ia32_psrav16si_mask",
+            "avx512.mask.psrav.q" => "__builtin_ia32_psrav8di_mask",
+            "avx512.mask.psrav.q.128" => "__builtin_ia32_psravq128_mask",
+            "avx512.mask.psrav.q.256" => "__builtin_ia32_psravq256_mask",
+            "avx512.mask.psrav16.hi" => "__builtin_ia32_psrav16hi_mask",
+            "avx512.mask.psrav32.hi" => "__builtin_ia32_psrav32hi_mask",
+            "avx512.mask.psrav4.si" => "__builtin_ia32_psrav4si_mask",
+            "avx512.mask.psrav8.hi" => "__builtin_ia32_psrav8hi_mask",
+            "avx512.mask.psrav8.si" => "__builtin_ia32_psrav8si_mask",
+            "avx512.mask.psrl.d" => "__builtin_ia32_psrld512_mask",
+            "avx512.mask.psrl.d.128" => "__builtin_ia32_psrld128_mask",
+            "avx512.mask.psrl.d.256" => "__builtin_ia32_psrld256_mask",
+            "avx512.mask.psrl.di.128" => "__builtin_ia32_psrldi128_mask",
+            "avx512.mask.psrl.di.256" => "__builtin_ia32_psrldi256_mask",
+            "avx512.mask.psrl.di.512" => "__builtin_ia32_psrldi512_mask",
+            "avx512.mask.psrl.q" => "__builtin_ia32_psrlq512_mask",
+            "avx512.mask.psrl.q.128" => "__builtin_ia32_psrlq128_mask",
+            "avx512.mask.psrl.q.256" => "__builtin_ia32_psrlq256_mask",
+            "avx512.mask.psrl.qi.128" => "__builtin_ia32_psrlqi128_mask",
+            "avx512.mask.psrl.qi.256" => "__builtin_ia32_psrlqi256_mask",
+            "avx512.mask.psrl.qi.512" => "__builtin_ia32_psrlqi512_mask",
+            "avx512.mask.psrl.w.128" => "__builtin_ia32_psrlw128_mask",
+            "avx512.mask.psrl.w.256" => "__builtin_ia32_psrlw256_mask",
+            "avx512.mask.psrl.w.512" => "__builtin_ia32_psrlw512_mask",
+            "avx512.mask.psrl.wi.128" => "__builtin_ia32_psrlwi128_mask",
+            "avx512.mask.psrl.wi.256" => "__builtin_ia32_psrlwi256_mask",
+            "avx512.mask.psrl.wi.512" => "__builtin_ia32_psrlwi512_mask",
+            "avx512.mask.psrlv.d" => "__builtin_ia32_psrlv16si_mask",
+            "avx512.mask.psrlv.q" => "__builtin_ia32_psrlv8di_mask",
+            "avx512.mask.psrlv16.hi" => "__builtin_ia32_psrlv16hi_mask",
+            "avx512.mask.psrlv2.di" => "__builtin_ia32_psrlv2di_mask",
+            "avx512.mask.psrlv32hi" => "__builtin_ia32_psrlv32hi_mask",
+            "avx512.mask.psrlv4.di" => "__builtin_ia32_psrlv4di_mask",
+            "avx512.mask.psrlv4.si" => "__builtin_ia32_psrlv4si_mask",
+            "avx512.mask.psrlv8.hi" => "__builtin_ia32_psrlv8hi_mask",
+            "avx512.mask.psrlv8.si" => "__builtin_ia32_psrlv8si_mask",
+            "avx512.mask.psub.b.128" => "__builtin_ia32_psubb128_mask",
+            "avx512.mask.psub.b.256" => "__builtin_ia32_psubb256_mask",
+            "avx512.mask.psub.b.512" => "__builtin_ia32_psubb512_mask",
+            "avx512.mask.psub.d.128" => "__builtin_ia32_psubd128_mask",
+            "avx512.mask.psub.d.256" => "__builtin_ia32_psubd256_mask",
+            "avx512.mask.psub.d.512" => "__builtin_ia32_psubd512_mask",
+            "avx512.mask.psub.q.128" => "__builtin_ia32_psubq128_mask",
+            "avx512.mask.psub.q.256" => "__builtin_ia32_psubq256_mask",
+            "avx512.mask.psub.q.512" => "__builtin_ia32_psubq512_mask",
+            "avx512.mask.psub.w.128" => "__builtin_ia32_psubw128_mask",
+            "avx512.mask.psub.w.256" => "__builtin_ia32_psubw256_mask",
+            "avx512.mask.psub.w.512" => "__builtin_ia32_psubw512_mask",
+            "avx512.mask.psubs.b.128" => "__builtin_ia32_psubsb128_mask",
+            "avx512.mask.psubs.b.256" => "__builtin_ia32_psubsb256_mask",
+            "avx512.mask.psubs.b.512" => "__builtin_ia32_psubsb512_mask",
+            "avx512.mask.psubs.w.128" => "__builtin_ia32_psubsw128_mask",
+            "avx512.mask.psubs.w.256" => "__builtin_ia32_psubsw256_mask",
+            "avx512.mask.psubs.w.512" => "__builtin_ia32_psubsw512_mask",
+            "avx512.mask.psubus.b.128" => "__builtin_ia32_psubusb128_mask",
+            "avx512.mask.psubus.b.256" => "__builtin_ia32_psubusb256_mask",
+            "avx512.mask.psubus.b.512" => "__builtin_ia32_psubusb512_mask",
+            "avx512.mask.psubus.w.128" => "__builtin_ia32_psubusw128_mask",
+            "avx512.mask.psubus.w.256" => "__builtin_ia32_psubusw256_mask",
+            "avx512.mask.psubus.w.512" => "__builtin_ia32_psubusw512_mask",
+            "avx512.mask.pternlog.d.128" => "__builtin_ia32_pternlogd128_mask",
+            "avx512.mask.pternlog.d.256" => "__builtin_ia32_pternlogd256_mask",
+            "avx512.mask.pternlog.d.512" => "__builtin_ia32_pternlogd512_mask",
+            "avx512.mask.pternlog.q.128" => "__builtin_ia32_pternlogq128_mask",
+            "avx512.mask.pternlog.q.256" => "__builtin_ia32_pternlogq256_mask",
+            "avx512.mask.pternlog.q.512" => "__builtin_ia32_pternlogq512_mask",
+            "avx512.mask.ptestm.d.512" => "__builtin_ia32_ptestmd512",
+            "avx512.mask.ptestm.q.512" => "__builtin_ia32_ptestmq512",
+            "avx512.mask.shuf.f32x4" => "__builtin_ia32_shuf_f32x4_mask",
+            "avx512.mask.shuf.f32x4.256" => "__builtin_ia32_shuf_f32x4_256_mask",
+            "avx512.mask.shuf.f64x2" => "__builtin_ia32_shuf_f64x2_mask",
+            "avx512.mask.shuf.f64x2.256" => "__builtin_ia32_shuf_f64x2_256_mask",
+            "avx512.mask.shuf.i32x4" => "__builtin_ia32_shuf_i32x4_mask",
+            "avx512.mask.shuf.i32x4.256" => "__builtin_ia32_shuf_i32x4_256_mask",
+            "avx512.mask.shuf.i64x2" => "__builtin_ia32_shuf_i64x2_mask",
+            "avx512.mask.shuf.i64x2.256" => "__builtin_ia32_shuf_i64x2_256_mask",
+            "avx512.mask.shuf.pd.128" => "__builtin_ia32_shufpd128_mask",
+            "avx512.mask.shuf.pd.256" => "__builtin_ia32_shufpd256_mask",
+            "avx512.mask.shuf.pd.512" => "__builtin_ia32_shufpd512_mask",
+            "avx512.mask.shuf.ps.128" => "__builtin_ia32_shufps128_mask",
+            "avx512.mask.shuf.ps.256" => "__builtin_ia32_shufps256_mask",
+            "avx512.mask.shuf.ps.512" => "__builtin_ia32_shufps512_mask",
+            "avx512.mask.sqrt.pd.128" => "__builtin_ia32_sqrtpd128_mask",
+            "avx512.mask.sqrt.pd.256" => "__builtin_ia32_sqrtpd256_mask",
+            "avx512.mask.sqrt.pd.512" => "__builtin_ia32_sqrtpd512_mask",
+            "avx512.mask.sqrt.ps.128" => "__builtin_ia32_sqrtps128_mask",
+            "avx512.mask.sqrt.ps.256" => "__builtin_ia32_sqrtps256_mask",
+            "avx512.mask.sqrt.ps.512" => "__builtin_ia32_sqrtps512_mask",
+            "avx512.mask.store.ss" => "__builtin_ia32_storess_mask",
+            "avx512.mask.storeu.d.512" => "__builtin_ia32_storedqusi512_mask",
+            "avx512.mask.storeu.pd.512" => "__builtin_ia32_storeupd512_mask",
+            "avx512.mask.storeu.ps.512" => "__builtin_ia32_storeups512_mask",
+            "avx512.mask.storeu.q.512" => "__builtin_ia32_storedqudi512_mask",
+            "avx512.mask.sub.pd.128" => "__builtin_ia32_subpd128_mask",
+            "avx512.mask.sub.pd.256" => "__builtin_ia32_subpd256_mask",
+            "avx512.mask.sub.pd.512" => "__builtin_ia32_subpd512_mask",
+            "avx512.mask.sub.ps.128" => "__builtin_ia32_subps128_mask",
+            "avx512.mask.sub.ps.256" => "__builtin_ia32_subps256_mask",
+            "avx512.mask.sub.ps.512" => "__builtin_ia32_subps512_mask",
+            "avx512.mask.valign.d.128" => "__builtin_ia32_alignd128_mask",
+            "avx512.mask.valign.d.256" => "__builtin_ia32_alignd256_mask",
+            "avx512.mask.valign.d.512" => "__builtin_ia32_alignd512_mask",
+            "avx512.mask.valign.q.128" => "__builtin_ia32_alignq128_mask",
+            "avx512.mask.valign.q.256" => "__builtin_ia32_alignq256_mask",
+            "avx512.mask.valign.q.512" => "__builtin_ia32_alignq512_mask",
+            "avx512.mask.vcvtph2ps.128" => "__builtin_ia32_vcvtph2ps_mask",
+            "avx512.mask.vcvtph2ps.256" => "__builtin_ia32_vcvtph2ps256_mask",
+            "avx512.mask.vcvtph2ps.512" => "__builtin_ia32_vcvtph2ps512_mask",
+            "avx512.mask.vextractf32x4.256" => "__builtin_ia32_extractf32x4_256_mask",
+            "avx512.mask.vextractf32x4.512" => "__builtin_ia32_extractf32x4_mask",
+            "avx512.mask.vextractf32x8.512" => "__builtin_ia32_extractf32x8_mask",
+            "avx512.mask.vextractf64x2.256" => "__builtin_ia32_extractf64x2_256_mask",
+            "avx512.mask.vextractf64x2.512" => "__builtin_ia32_extractf64x2_512_mask",
+            "avx512.mask.vextractf64x4.512" => "__builtin_ia32_extractf64x4_mask",
+            "avx512.mask.vextracti32x4.256" => "__builtin_ia32_extracti32x4_256_mask",
+            "avx512.mask.vextracti32x4.512" => "__builtin_ia32_extracti32x4_mask",
+            "avx512.mask.vextracti32x8.512" => "__builtin_ia32_extracti32x8_mask",
+            "avx512.mask.vextracti64x2.256" => "__builtin_ia32_extracti64x2_256_mask",
+            "avx512.mask.vextracti64x2.512" => "__builtin_ia32_extracti64x2_512_mask",
+            "avx512.mask.vextracti64x4.512" => "__builtin_ia32_extracti64x4_mask",
+            "avx512.mask.vfmadd.pd.128" => "__builtin_ia32_vfmaddpd128_mask",
+            "avx512.mask.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256_mask",
+            "avx512.mask.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
+            "avx512.mask.vfmadd.ps.128" => "__builtin_ia32_vfmaddps128_mask",
+            "avx512.mask.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256_mask",
+            "avx512.mask.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
+            "avx512.mask.vfmadd.sd" => "__builtin_ia32_vfmaddsd3_mask",
+            "avx512.mask.vfmadd.ss" => "__builtin_ia32_vfmaddss3_mask",
+            "avx512.mask.vfmaddsub.pd.128" => "__builtin_ia32_vfmaddsubpd128_mask",
+            "avx512.mask.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256_mask",
+            "avx512.mask.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask",
+            "avx512.mask.vfmaddsub.ps.128" => "__builtin_ia32_vfmaddsubps128_mask",
+            "avx512.mask.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256_mask",
+            "avx512.mask.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
+            "avx512.mask.vfnmadd.pd.128" => "__builtin_ia32_vfnmaddpd128_mask",
+            "avx512.mask.vfnmadd.pd.256" => "__builtin_ia32_vfnmaddpd256_mask",
+            "avx512.mask.vfnmadd.pd.512" => "__builtin_ia32_vfnmaddpd512_mask",
+            "avx512.mask.vfnmadd.ps.128" => "__builtin_ia32_vfnmaddps128_mask",
+            "avx512.mask.vfnmadd.ps.256" => "__builtin_ia32_vfnmaddps256_mask",
+            "avx512.mask.vfnmadd.ps.512" => "__builtin_ia32_vfnmaddps512_mask",
+            "avx512.mask.vfnmsub.pd.128" => "__builtin_ia32_vfnmsubpd128_mask",
+            "avx512.mask.vfnmsub.pd.256" => "__builtin_ia32_vfnmsubpd256_mask",
+            "avx512.mask.vfnmsub.pd.512" => "__builtin_ia32_vfnmsubpd512_mask",
+            "avx512.mask.vfnmsub.ps.128" => "__builtin_ia32_vfnmsubps128_mask",
+            "avx512.mask.vfnmsub.ps.256" => "__builtin_ia32_vfnmsubps256_mask",
+            "avx512.mask.vfnmsub.ps.512" => "__builtin_ia32_vfnmsubps512_mask",
+            "avx512.mask.vpermi2var.d.128" => "__builtin_ia32_vpermi2vard128_mask",
+            "avx512.mask.vpermi2var.d.256" => "__builtin_ia32_vpermi2vard256_mask",
+            "avx512.mask.vpermi2var.d.512" => "__builtin_ia32_vpermi2vard512_mask",
+            "avx512.mask.vpermi2var.hi.128" => "__builtin_ia32_vpermi2varhi128_mask",
+            "avx512.mask.vpermi2var.hi.256" => "__builtin_ia32_vpermi2varhi256_mask",
+            "avx512.mask.vpermi2var.hi.512" => "__builtin_ia32_vpermi2varhi512_mask",
+            "avx512.mask.vpermi2var.pd.128" => "__builtin_ia32_vpermi2varpd128_mask",
+            "avx512.mask.vpermi2var.pd.256" => "__builtin_ia32_vpermi2varpd256_mask",
+            "avx512.mask.vpermi2var.pd.512" => "__builtin_ia32_vpermi2varpd512_mask",
+            "avx512.mask.vpermi2var.ps.128" => "__builtin_ia32_vpermi2varps128_mask",
+            "avx512.mask.vpermi2var.ps.256" => "__builtin_ia32_vpermi2varps256_mask",
+            "avx512.mask.vpermi2var.ps.512" => "__builtin_ia32_vpermi2varps512_mask",
+            "avx512.mask.vpermi2var.q.128" => "__builtin_ia32_vpermi2varq128_mask",
+            "avx512.mask.vpermi2var.q.256" => "__builtin_ia32_vpermi2varq256_mask",
+            "avx512.mask.vpermi2var.q.512" => "__builtin_ia32_vpermi2varq512_mask",
+            "avx512.mask.vpermi2var.qi.128" => "__builtin_ia32_vpermi2varqi128_mask",
+            "avx512.mask.vpermi2var.qi.256" => "__builtin_ia32_vpermi2varqi256_mask",
+            "avx512.mask.vpermi2var.qi.512" => "__builtin_ia32_vpermi2varqi512_mask",
+            "avx512.mask.vpermilvar.pd.128" => "__builtin_ia32_vpermilvarpd_mask",
+            "avx512.mask.vpermilvar.pd.256" => "__builtin_ia32_vpermilvarpd256_mask",
+            "avx512.mask.vpermilvar.pd.512" => "__builtin_ia32_vpermilvarpd512_mask",
+            "avx512.mask.vpermilvar.ps.128" => "__builtin_ia32_vpermilvarps_mask",
+            "avx512.mask.vpermilvar.ps.256" => "__builtin_ia32_vpermilvarps256_mask",
+            "avx512.mask.vpermilvar.ps.512" => "__builtin_ia32_vpermilvarps512_mask",
+            "avx512.mask.vpermt.d.512" => "__builtin_ia32_vpermt2vard512_mask",
+            "avx512.mask.vpermt.pd.512" => "__builtin_ia32_vpermt2varpd512_mask",
+            "avx512.mask.vpermt.ps.512" => "__builtin_ia32_vpermt2varps512_mask",
+            "avx512.mask.vpermt.q.512" => "__builtin_ia32_vpermt2varq512_mask",
+            "avx512.mask.vpermt2var.d.128" => "__builtin_ia32_vpermt2vard128_mask",
+            "avx512.mask.vpermt2var.d.256" => "__builtin_ia32_vpermt2vard256_mask",
+            "avx512.mask.vpermt2var.d.512" => "__builtin_ia32_vpermt2vard512_mask",
+            "avx512.mask.vpermt2var.hi.128" => "__builtin_ia32_vpermt2varhi128_mask",
+            "avx512.mask.vpermt2var.hi.256" => "__builtin_ia32_vpermt2varhi256_mask",
+            "avx512.mask.vpermt2var.hi.512" => "__builtin_ia32_vpermt2varhi512_mask",
+            "avx512.mask.vpermt2var.pd.128" => "__builtin_ia32_vpermt2varpd128_mask",
+            "avx512.mask.vpermt2var.pd.256" => "__builtin_ia32_vpermt2varpd256_mask",
+            "avx512.mask.vpermt2var.pd.512" => "__builtin_ia32_vpermt2varpd512_mask",
+            "avx512.mask.vpermt2var.ps.128" => "__builtin_ia32_vpermt2varps128_mask",
+            "avx512.mask.vpermt2var.ps.256" => "__builtin_ia32_vpermt2varps256_mask",
+            "avx512.mask.vpermt2var.ps.512" => "__builtin_ia32_vpermt2varps512_mask",
+            "avx512.mask.vpermt2var.q.128" => "__builtin_ia32_vpermt2varq128_mask",
+            "avx512.mask.vpermt2var.q.256" => "__builtin_ia32_vpermt2varq256_mask",
+            "avx512.mask.vpermt2var.q.512" => "__builtin_ia32_vpermt2varq512_mask",
+            "avx512.mask.vpermt2var.qi.128" => "__builtin_ia32_vpermt2varqi128_mask",
+            "avx512.mask.vpermt2var.qi.256" => "__builtin_ia32_vpermt2varqi256_mask",
+            "avx512.mask.vpermt2var.qi.512" => "__builtin_ia32_vpermt2varqi512_mask",
+            "avx512.mask.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_mask",
+            "avx512.mask.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_mask",
+            "avx512.mask.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_mask",
+            "avx512.mask.vpmadd52l.uq.128" => "__builtin_ia32_vpmadd52luq128_mask",
+            "avx512.mask.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_mask",
+            "avx512.mask.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_mask",
+            "avx512.mask.xor.pd.128" => "__builtin_ia32_xorpd128_mask",
+            "avx512.mask.xor.pd.256" => "__builtin_ia32_xorpd256_mask",
+            "avx512.mask.xor.pd.512" => "__builtin_ia32_xorpd512_mask",
+            "avx512.mask.xor.ps.128" => "__builtin_ia32_xorps128_mask",
+            "avx512.mask.xor.ps.256" => "__builtin_ia32_xorps256_mask",
+            "avx512.mask.xor.ps.512" => "__builtin_ia32_xorps512_mask",
+            "avx512.mask3.vfmadd.pd.128" => "__builtin_ia32_vfmaddpd128_mask3",
+            "avx512.mask3.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256_mask3",
+            "avx512.mask3.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask3",
+            "avx512.mask3.vfmadd.ps.128" => "__builtin_ia32_vfmaddps128_mask3",
+            "avx512.mask3.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256_mask3",
+            "avx512.mask3.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask3",
+            "avx512.mask3.vfmadd.sd" => "__builtin_ia32_vfmaddsd3_mask3",
+            "avx512.mask3.vfmadd.ss" => "__builtin_ia32_vfmaddss3_mask3",
+            "avx512.mask3.vfmaddsub.pd.128" => "__builtin_ia32_vfmaddsubpd128_mask3",
+            "avx512.mask3.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256_mask3",
+            "avx512.mask3.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask3",
+            "avx512.mask3.vfmaddsub.ps.128" => "__builtin_ia32_vfmaddsubps128_mask3",
+            "avx512.mask3.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256_mask3",
+            "avx512.mask3.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask3",
+            "avx512.mask3.vfmsub.pd.128" => "__builtin_ia32_vfmsubpd128_mask3",
+            "avx512.mask3.vfmsub.pd.256" => "__builtin_ia32_vfmsubpd256_mask3",
+            "avx512.mask3.vfmsub.pd.512" => "__builtin_ia32_vfmsubpd512_mask3",
+            "avx512.mask3.vfmsub.ps.128" => "__builtin_ia32_vfmsubps128_mask3",
+            "avx512.mask3.vfmsub.ps.256" => "__builtin_ia32_vfmsubps256_mask3",
+            "avx512.mask3.vfmsub.ps.512" => "__builtin_ia32_vfmsubps512_mask3",
+            "avx512.mask3.vfmsubadd.pd.128" => "__builtin_ia32_vfmsubaddpd128_mask3",
+            "avx512.mask3.vfmsubadd.pd.256" => "__builtin_ia32_vfmsubaddpd256_mask3",
+            "avx512.mask3.vfmsubadd.pd.512" => "__builtin_ia32_vfmsubaddpd512_mask3",
+            "avx512.mask3.vfmsubadd.ps.128" => "__builtin_ia32_vfmsubaddps128_mask3",
+            "avx512.mask3.vfmsubadd.ps.256" => "__builtin_ia32_vfmsubaddps256_mask3",
+            "avx512.mask3.vfmsubadd.ps.512" => "__builtin_ia32_vfmsubaddps512_mask3",
+            "avx512.mask3.vfnmsub.pd.128" => "__builtin_ia32_vfnmsubpd128_mask3",
+            "avx512.mask3.vfnmsub.pd.256" => "__builtin_ia32_vfnmsubpd256_mask3",
+            "avx512.mask3.vfnmsub.pd.512" => "__builtin_ia32_vfnmsubpd512_mask3",
+            "avx512.mask3.vfnmsub.ps.128" => "__builtin_ia32_vfnmsubps128_mask3",
+            "avx512.mask3.vfnmsub.ps.256" => "__builtin_ia32_vfnmsubps256_mask3",
+            "avx512.mask3.vfnmsub.ps.512" => "__builtin_ia32_vfnmsubps512_mask3",
+            "avx512.maskz.pternlog.d.128" => "__builtin_ia32_pternlogd128_maskz",
+            "avx512.maskz.pternlog.d.256" => "__builtin_ia32_pternlogd256_maskz",
+            "avx512.maskz.pternlog.d.512" => "__builtin_ia32_pternlogd512_maskz",
+            "avx512.maskz.pternlog.q.128" => "__builtin_ia32_pternlogq128_maskz",
+            "avx512.maskz.pternlog.q.256" => "__builtin_ia32_pternlogq256_maskz",
+            "avx512.maskz.pternlog.q.512" => "__builtin_ia32_pternlogq512_maskz",
+            "avx512.maskz.vfmadd.pd.128" => "__builtin_ia32_vfmaddpd128_maskz",
+            "avx512.maskz.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256_maskz",
+            "avx512.maskz.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_maskz",
+            "avx512.maskz.vfmadd.ps.128" => "__builtin_ia32_vfmaddps128_maskz",
+            "avx512.maskz.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256_maskz",
+            "avx512.maskz.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_maskz",
+            "avx512.maskz.vfmadd.sd" => "__builtin_ia32_vfmaddsd3_maskz",
+            "avx512.maskz.vfmadd.ss" => "__builtin_ia32_vfmaddss3_maskz",
+            "avx512.maskz.vfmaddsub.pd.128" => "__builtin_ia32_vfmaddsubpd128_maskz",
+            "avx512.maskz.vfmaddsub.pd.256" => "__builtin_ia32_vfmaddsubpd256_maskz",
+            "avx512.maskz.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_maskz",
+            "avx512.maskz.vfmaddsub.ps.128" => "__builtin_ia32_vfmaddsubps128_maskz",
+            "avx512.maskz.vfmaddsub.ps.256" => "__builtin_ia32_vfmaddsubps256_maskz",
+            "avx512.maskz.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_maskz",
+            "avx512.maskz.vpermt2var.d.128" => "__builtin_ia32_vpermt2vard128_maskz",
+            "avx512.maskz.vpermt2var.d.256" => "__builtin_ia32_vpermt2vard256_maskz",
+            "avx512.maskz.vpermt2var.d.512" => "__builtin_ia32_vpermt2vard512_maskz",
+            "avx512.maskz.vpermt2var.hi.128" => "__builtin_ia32_vpermt2varhi128_maskz",
+            "avx512.maskz.vpermt2var.hi.256" => "__builtin_ia32_vpermt2varhi256_maskz",
+            "avx512.maskz.vpermt2var.hi.512" => "__builtin_ia32_vpermt2varhi512_maskz",
+            "avx512.maskz.vpermt2var.pd.128" => "__builtin_ia32_vpermt2varpd128_maskz",
+            "avx512.maskz.vpermt2var.pd.256" => "__builtin_ia32_vpermt2varpd256_maskz",
+            "avx512.maskz.vpermt2var.pd.512" => "__builtin_ia32_vpermt2varpd512_maskz",
+            "avx512.maskz.vpermt2var.ps.128" => "__builtin_ia32_vpermt2varps128_maskz",
+            "avx512.maskz.vpermt2var.ps.256" => "__builtin_ia32_vpermt2varps256_maskz",
+            "avx512.maskz.vpermt2var.ps.512" => "__builtin_ia32_vpermt2varps512_maskz",
+            "avx512.maskz.vpermt2var.q.128" => "__builtin_ia32_vpermt2varq128_maskz",
+            "avx512.maskz.vpermt2var.q.256" => "__builtin_ia32_vpermt2varq256_maskz",
+            "avx512.maskz.vpermt2var.q.512" => "__builtin_ia32_vpermt2varq512_maskz",
+            "avx512.maskz.vpermt2var.qi.128" => "__builtin_ia32_vpermt2varqi128_maskz",
+            "avx512.maskz.vpermt2var.qi.256" => "__builtin_ia32_vpermt2varqi256_maskz",
+            "avx512.maskz.vpermt2var.qi.512" => "__builtin_ia32_vpermt2varqi512_maskz",
+            "avx512.maskz.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_maskz",
+            "avx512.maskz.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_maskz",
+            "avx512.maskz.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_maskz",
+            "avx512.maskz.vpmadd52l.uq.128" => "__builtin_ia32_vpmadd52luq128_maskz",
+            "avx512.maskz.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_maskz",
+            "avx512.maskz.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_maskz",
+            "avx512.movntdqa" => "__builtin_ia32_movntdqa512",
+            "avx512.pbroadcastd.512" => "__builtin_ia32_pbroadcastd512",
+            "avx512.pbroadcastq.512" => "__builtin_ia32_pbroadcastq512",
+            "avx512.pmovzxbd" => "__builtin_ia32_pmovzxbd512",
+            "avx512.pmovzxbq" => "__builtin_ia32_pmovzxbq512",
+            "avx512.pmovzxdq" => "__builtin_ia32_pmovzxdq512",
+            "avx512.pmovzxwd" => "__builtin_ia32_pmovzxwd512",
+            "avx512.pmovzxwq" => "__builtin_ia32_pmovzxwq512",
+            "avx512.psll.dq" => "__builtin_ia32_pslldqi512",
+            "avx512.psll.dq.bs" => "__builtin_ia32_pslldqi512_byteshift",
+            "avx512.psrl.dq" => "__builtin_ia32_psrldqi512",
+            "avx512.psrl.dq.bs" => "__builtin_ia32_psrldqi512_byteshift",
+            "avx512.ptestm.b.128" => "__builtin_ia32_ptestmb128",
+            "avx512.ptestm.b.256" => "__builtin_ia32_ptestmb256",
+            "avx512.ptestm.b.512" => "__builtin_ia32_ptestmb512",
+            "avx512.ptestm.d.128" => "__builtin_ia32_ptestmd128",
+            "avx512.ptestm.d.256" => "__builtin_ia32_ptestmd256",
+            "avx512.ptestm.d.512" => "__builtin_ia32_ptestmd512",
+            "avx512.ptestm.q.128" => "__builtin_ia32_ptestmq128",
+            "avx512.ptestm.q.256" => "__builtin_ia32_ptestmq256",
+            "avx512.ptestm.q.512" => "__builtin_ia32_ptestmq512",
+            "avx512.ptestm.w.128" => "__builtin_ia32_ptestmw128",
+            "avx512.ptestm.w.256" => "__builtin_ia32_ptestmw256",
+            "avx512.ptestm.w.512" => "__builtin_ia32_ptestmw512",
+            "avx512.ptestnm.b.128" => "__builtin_ia32_ptestnmb128",
+            "avx512.ptestnm.b.256" => "__builtin_ia32_ptestnmb256",
+            "avx512.ptestnm.b.512" => "__builtin_ia32_ptestnmb512",
+            "avx512.ptestnm.d.128" => "__builtin_ia32_ptestnmd128",
+            "avx512.ptestnm.d.256" => "__builtin_ia32_ptestnmd256",
+            "avx512.ptestnm.d.512" => "__builtin_ia32_ptestnmd512",
+            "avx512.ptestnm.q.128" => "__builtin_ia32_ptestnmq128",
+            "avx512.ptestnm.q.256" => "__builtin_ia32_ptestnmq256",
+            "avx512.ptestnm.q.512" => "__builtin_ia32_ptestnmq512",
+            "avx512.ptestnm.w.128" => "__builtin_ia32_ptestnmw128",
+            "avx512.ptestnm.w.256" => "__builtin_ia32_ptestnmw256",
+            "avx512.ptestnm.w.512" => "__builtin_ia32_ptestnmw512",
+            "avx512.rcp28.pd" => "__builtin_ia32_rcp28pd_mask",
+            "avx512.rcp28.ps" => "__builtin_ia32_rcp28ps_mask",
+            "avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
+            "avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
+            "avx512.rndscale.sd" => "__builtin_ia32_rndscalesd",
+            "avx512.rndscale.ss" => "__builtin_ia32_rndscaless",
+            "avx512.rsqrt28.pd" => "__builtin_ia32_rsqrt28pd_mask",
+            "avx512.rsqrt28.ps" => "__builtin_ia32_rsqrt28ps_mask",
+            "avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
+            "avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
+            "avx512.scatter.dpd.512" => "__builtin_ia32_scattersiv8df",
+            "avx512.scatter.dpi.512" => "__builtin_ia32_scattersiv16si",
+            "avx512.scatter.dpq.512" => "__builtin_ia32_scattersiv8di",
+            "avx512.scatter.dps.512" => "__builtin_ia32_scattersiv16sf",
+            "avx512.scatter.qpd.512" => "__builtin_ia32_scatterdiv8df",
+            "avx512.scatter.qpi.512" => "__builtin_ia32_scatterdiv16si",
+            "avx512.scatter.qpq.512" => "__builtin_ia32_scatterdiv8di",
+            "avx512.scatter.qps.512" => "__builtin_ia32_scatterdiv16sf",
+            "avx512.scatterdiv2.df" => "__builtin_ia32_scatterdiv2df",
+            "avx512.scatterdiv2.di" => "__builtin_ia32_scatterdiv2di",
+            "avx512.scatterdiv4.df" => "__builtin_ia32_scatterdiv4df",
+            "avx512.scatterdiv4.di" => "__builtin_ia32_scatterdiv4di",
+            "avx512.scatterdiv4.sf" => "__builtin_ia32_scatterdiv4sf",
+            "avx512.scatterdiv4.si" => "__builtin_ia32_scatterdiv4si",
+            "avx512.scatterdiv8.sf" => "__builtin_ia32_scatterdiv8sf",
+            "avx512.scatterdiv8.si" => "__builtin_ia32_scatterdiv8si",
+            "avx512.scatterpf.dpd.512" => "__builtin_ia32_scatterpfdpd",
+            "avx512.scatterpf.dps.512" => "__builtin_ia32_scatterpfdps",
+            "avx512.scatterpf.qpd.512" => "__builtin_ia32_scatterpfqpd",
+            "avx512.scatterpf.qps.512" => "__builtin_ia32_scatterpfqps",
+            "avx512.scattersiv2.df" => "__builtin_ia32_scattersiv2df",
+            "avx512.scattersiv2.di" => "__builtin_ia32_scattersiv2di",
+            "avx512.scattersiv4.df" => "__builtin_ia32_scattersiv4df",
+            "avx512.scattersiv4.di" => "__builtin_ia32_scattersiv4di",
+            "avx512.scattersiv4.sf" => "__builtin_ia32_scattersiv4sf",
+            "avx512.scattersiv4.si" => "__builtin_ia32_scattersiv4si",
+            "avx512.scattersiv8.sf" => "__builtin_ia32_scattersiv8sf",
+            "avx512.scattersiv8.si" => "__builtin_ia32_scattersiv8si",
+            "avx512.sqrt.pd.512" => "__builtin_ia32_sqrtpd512_mask",
+            "avx512.sqrt.ps.512" => "__builtin_ia32_sqrtps512_mask",
+            "avx512.sqrt.sd" => "__builtin_ia32_sqrtrndsd",
+            "avx512.sqrt.ss" => "__builtin_ia32_sqrtrndss",
+            "avx512.vbroadcast.sd.512" => "__builtin_ia32_vbroadcastsd512",
+            "avx512.vbroadcast.sd.pd.512" => "__builtin_ia32_vbroadcastsd_pd512",
+            "avx512.vbroadcast.ss.512" => "__builtin_ia32_vbroadcastss512",
+            "avx512.vbroadcast.ss.ps.512" => "__builtin_ia32_vbroadcastss_ps512",
+            "fma.mask.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
+            "fma.mask.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
+            "fma.mask.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask",
+            "fma.mask.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
+            "fma.mask.vfmsub.pd.512" => "__builtin_ia32_vfmsubpd512_mask",
+            "fma.mask.vfmsub.ps.512" => "__builtin_ia32_vfmsubps512_mask",
+            "fma.mask.vfmsubadd.pd.512" => "__builtin_ia32_vfmsubaddpd512_mask",
+            "fma.mask.vfmsubadd.ps.512" => "__builtin_ia32_vfmsubaddps512_mask",
+            "fma.mask.vfnmadd.pd.512" => "__builtin_ia32_vfnmaddpd512_mask",
+            "fma.mask.vfnmadd.ps.512" => "__builtin_ia32_vfnmaddps512_mask",
+            "fma.mask.vfnmsub.pd.512" => "__builtin_ia32_vfnmsubpd512_mask",
+            "fma.mask.vfnmsub.ps.512" => "__builtin_ia32_vfnmsubps512_mask",
+            "fma.vfmadd.pd" => "__builtin_ia32_vfmaddpd",
+            "fma.vfmadd.pd.256" => "__builtin_ia32_vfmaddpd256",
+            "fma.vfmadd.ps" => "__builtin_ia32_vfmaddps",
+            "fma.vfmadd.ps.256" => "__builtin_ia32_vfmaddps256",
+            "fma.vfmadd.sd" => "__builtin_ia32_vfmaddsd",
+            "fma.vfmadd.ss" => "__builtin_ia32_vfmaddss",
+            "fma.vfmsub.pd" => "__builtin_ia32_vfmsubpd",
+            "fma.vfmsub.pd.256" => "__builtin_ia32_vfmsubpd256",
+            "fma.vfmsub.ps" => "__builtin_ia32_vfmsubps",
+            "fma.vfmsub.ps.256" => "__builtin_ia32_vfmsubps256",
+            "fma.vfmsub.sd" => "__builtin_ia32_vfmsubsd",
+            "fma.vfmsub.ss" => "__builtin_ia32_vfmsubss",
+            "fma.vfmsubadd.pd" => "__builtin_ia32_vfmsubaddpd",
+            "fma.vfmsubadd.pd.256" => "__builtin_ia32_vfmsubaddpd256",
+            "fma.vfmsubadd.ps" => "__builtin_ia32_vfmsubaddps",
+            "fma.vfmsubadd.ps.256" => "__builtin_ia32_vfmsubaddps256",
+            "fma.vfnmadd.pd" => "__builtin_ia32_vfnmaddpd",
+            "fma.vfnmadd.pd.256" => "__builtin_ia32_vfnmaddpd256",
+            "fma.vfnmadd.ps" => "__builtin_ia32_vfnmaddps",
+            "fma.vfnmadd.ps.256" => "__builtin_ia32_vfnmaddps256",
+            "fma.vfnmadd.sd" => "__builtin_ia32_vfnmaddsd",
+            "fma.vfnmadd.ss" => "__builtin_ia32_vfnmaddss",
+            "fma.vfnmsub.pd" => "__builtin_ia32_vfnmsubpd",
+            "fma.vfnmsub.pd.256" => "__builtin_ia32_vfnmsubpd256",
+            "fma.vfnmsub.ps" => "__builtin_ia32_vfnmsubps",
+            "fma.vfnmsub.ps.256" => "__builtin_ia32_vfnmsubps256",
+            "fma.vfnmsub.sd" => "__builtin_ia32_vfnmsubsd",
+            "fma.vfnmsub.ss" => "__builtin_ia32_vfnmsubss",
+            "mmx.femms" => "__builtin_ia32_femms",
+            "rdtscp" => "__builtin_ia32_rdtscp",
+            "sse.add.ss" => "__builtin_ia32_addss",
+            "sse.cmp.ps" => "__builtin_ia32_cmpps",
+            "sse.cvtsi2ss" => "__builtin_ia32_cvtsi2ss",
+            "sse.cvtsi642ss" => "__builtin_ia32_cvtsi642ss",
+            "sse.div.ss" => "__builtin_ia32_divss",
+            "sse.mul.ss" => "__builtin_ia32_mulss",
+            "sse.sqrt.ps" => "__builtin_ia32_sqrtps",
+            "sse.sqrt.ss" => "__builtin_ia32_sqrtss",
+            "sse.storeu.ps" => "__builtin_ia32_storeups",
+            "sse.sub.ss" => "__builtin_ia32_subss",
+            "sse2.add.sd" => "__builtin_ia32_addsd",
+            "sse2.cmp.pd" => "__builtin_ia32_cmppd",
+            "sse2.cvtdq2pd" => "__builtin_ia32_cvtdq2pd",
+            "sse2.cvtdq2ps" => "__builtin_ia32_cvtdq2ps",
+            "sse2.cvtps2pd" => "__builtin_ia32_cvtps2pd",
+            "sse2.cvtsi2sd" => "__builtin_ia32_cvtsi2sd",
+            "sse2.cvtsi642sd" => "__builtin_ia32_cvtsi642sd",
+            "sse2.cvtss2sd" => "__builtin_ia32_cvtss2sd",
+            "sse2.div.sd" => "__builtin_ia32_divsd",
+            "sse2.mul.sd" => "__builtin_ia32_mulsd",
+            "sse2.padds.b" => "__builtin_ia32_paddsb128",
+            "sse2.padds.w" => "__builtin_ia32_paddsw128",
+            "sse2.paddus.b" => "__builtin_ia32_paddusb128",
+            "sse2.paddus.w" => "__builtin_ia32_paddusw128",
+            "sse2.pmaxs.w" => "__builtin_ia32_pmaxsw128",
+            "sse2.pmaxu.b" => "__builtin_ia32_pmaxub128",
+            "sse2.pmins.w" => "__builtin_ia32_pminsw128",
+            "sse2.pminu.b" => "__builtin_ia32_pminub128",
+            "sse2.pmulu.dq" => "__builtin_ia32_pmuludq128",
+            "sse2.pshuf.d" => "__builtin_ia32_pshufd",
+            "sse2.pshufh.w" => "__builtin_ia32_pshufhw",
+            "sse2.pshufl.w" => "__builtin_ia32_pshuflw",
+            "sse2.psll.dq" => "__builtin_ia32_pslldqi128",
+            "sse2.psll.dq.bs" => "__builtin_ia32_pslldqi128_byteshift",
+            "sse2.psrl.dq" => "__builtin_ia32_psrldqi128",
+            "sse2.psrl.dq.bs" => "__builtin_ia32_psrldqi128_byteshift",
+            "sse2.psubs.b" => "__builtin_ia32_psubsb128",
+            "sse2.psubs.w" => "__builtin_ia32_psubsw128",
+            "sse2.psubus.b" => "__builtin_ia32_psubusb128",
+            "sse2.psubus.w" => "__builtin_ia32_psubusw128",
+            "sse2.sqrt.pd" => "__builtin_ia32_sqrtpd",
+            "sse2.sqrt.sd" => "__builtin_ia32_sqrtsd",
+            "sse2.storel.dq" => "__builtin_ia32_storelv4si",
+            "sse2.storeu.dq" => "__builtin_ia32_storedqu",
+            "sse2.storeu.pd" => "__builtin_ia32_storeupd",
+            "sse2.sub.sd" => "__builtin_ia32_subsd",
+            "sse41.blendpd" => "__builtin_ia32_blendpd",
+            "sse41.blendps" => "__builtin_ia32_blendps",
+            "sse41.extractps" => "__builtin_ia32_extractps128",
+            "sse41.movntdqa" => "__builtin_ia32_movntdqa",
+            "sse41.pblendw" => "__builtin_ia32_pblendw128",
+            "sse41.pmaxsb" => "__builtin_ia32_pmaxsb128",
+            "sse41.pmaxsd" => "__builtin_ia32_pmaxsd128",
+            "sse41.pmaxud" => "__builtin_ia32_pmaxud128",
+            "sse41.pmaxuw" => "__builtin_ia32_pmaxuw128",
+            "sse41.pminsb" => "__builtin_ia32_pminsb128",
+            "sse41.pminsd" => "__builtin_ia32_pminsd128",
+            "sse41.pminud" => "__builtin_ia32_pminud128",
+            "sse41.pminuw" => "__builtin_ia32_pminuw128",
+            "sse41.pmovsxbd" => "__builtin_ia32_pmovsxbd128",
+            "sse41.pmovsxbq" => "__builtin_ia32_pmovsxbq128",
+            "sse41.pmovsxbw" => "__builtin_ia32_pmovsxbw128",
+            "sse41.pmovsxdq" => "__builtin_ia32_pmovsxdq128",
+            "sse41.pmovsxwd" => "__builtin_ia32_pmovsxwd128",
+            "sse41.pmovsxwq" => "__builtin_ia32_pmovsxwq128",
+            "sse41.pmovzxbd" => "__builtin_ia32_pmovzxbd128",
+            "sse41.pmovzxbq" => "__builtin_ia32_pmovzxbq128",
+            "sse41.pmovzxbw" => "__builtin_ia32_pmovzxbw128",
+            "sse41.pmovzxdq" => "__builtin_ia32_pmovzxdq128",
+            "sse41.pmovzxwd" => "__builtin_ia32_pmovzxwd128",
+            "sse41.pmovzxwq" => "__builtin_ia32_pmovzxwq128",
+            "sse41.pmuldq" => "__builtin_ia32_pmuldq128",
+            "sse4a.movnt.sd" => "__builtin_ia32_movntsd",
+            "sse4a.movnt.ss" => "__builtin_ia32_movntss",
+            "ssse3.pabs.b.128" => "__builtin_ia32_pabsb128",
+            "ssse3.pabs.d.128" => "__builtin_ia32_pabsd128",
+            "ssse3.pabs.w.128" => "__builtin_ia32_pabsw128",
+            "subborrow.u32" => "__builtin_ia32_subborrow_u32",
+            "subborrow.u64" => "__builtin_ia32_subborrow_u64",
+            "xop.vpcmov" => "__builtin_ia32_vpcmov",
+            "xop.vpcmov.256" => "__builtin_ia32_vpcmov_256",
+            "xop.vpcomb" => "__builtin_ia32_vpcomb",
+            "xop.vpcomd" => "__builtin_ia32_vpcomd",
+            "xop.vpcomq" => "__builtin_ia32_vpcomq",
+            "xop.vpcomub" => "__builtin_ia32_vpcomub",
+            "xop.vpcomud" => "__builtin_ia32_vpcomud",
+            "xop.vpcomuq" => "__builtin_ia32_vpcomuq",
+            "xop.vpcomuw" => "__builtin_ia32_vpcomuw",
+            "xop.vpcomw" => "__builtin_ia32_vpcomw",
+            "xop.vprotb" => "__builtin_ia32_vprotb",
+            "xop.vprotbi" => "__builtin_ia32_vprotbi",
+            "xop.vprotd" => "__builtin_ia32_vprotd",
+            "xop.vprotdi" => "__builtin_ia32_vprotdi",
+            "xop.vprotq" => "__builtin_ia32_vprotq",
+            "xop.vprotqi" => "__builtin_ia32_vprotqi",
+            "xop.vprotw" => "__builtin_ia32_vprotw",
+            "xop.vprotwi" => "__builtin_ia32_vprotwi",
+            _ => return ArchCheckResult::UnknownIntrinsic,
+        },
+        _ => return ArchCheckResult::UnknownArch,
+    })
+}
diff --git a/tools/generate_intrinsics.py b/tools/generate_intrinsics.py
index 767082c23cce..56b425cdb0cd 100644
--- a/tools/generate_intrinsics.py
+++ b/tools/generate_intrinsics.py
@@ -147,14 +147,12 @@ def fill_intrinsics(intrinsics, from_intrinsics, all_intrinsics):
                 all_intrinsics[entry[0]] = entry[1]
 
 
-def update_intrinsics(llvm_path, llvmint, llvmint2):
+def update_intrinsics(llvm_path):
     intrinsics_llvm = {}
     intrinsics_llvmint = {}
     all_intrinsics = {}
 
     extract_intrinsics_from_llvm(llvm_path, intrinsics_llvm)
-    extract_intrinsics_from_llvmint(llvmint, intrinsics_llvmint)
-    extract_intrinsics_from_llvmint(llvmint2, intrinsics_llvmint)
 
     intrinsics = {}
     # We give priority to translations from LLVM over the ones from llvmint.
@@ -173,13 +171,17 @@ def update_intrinsics(llvm_path, llvmint, llvmint2):
     # Since all intrinsic names start with "llvm.", we skip that prefix.
     print("Updating content of `{}`...".format(output_file))
     with open(output_file, "w", encoding="utf8") as out:
-        out.write("// File generated by `rustc_codegen_gcc/tools/generate_intrinsics.py`\n")
-        out.write("// DO NOT EDIT IT!\n")
-        out.write("/// Translate a given LLVM intrinsic name to an equivalent GCC one.\n")
-        out.write("fn map_arch_intrinsic(full_name:&str)->&'static str{\n")
-        out.write('let Some(name) = full_name.strip_prefix("llvm.") else { unimplemented!("***** unsupported LLVM intrinsic {}", full_name) };\n')
-        out.write('let Some((arch, name)) = name.split_once(\'.\') else { unimplemented!("***** unsupported LLVM intrinsic {}", name) };\n')
-        out.write("match arch {\n")
+        out.write("""// File generated by `rustc_codegen_gcc/tools/generate_intrinsics.py`
+// DO NOT EDIT IT!
+/// Translate a given LLVM intrinsic name to an equivalent GCC one.
+fn map_arch_intrinsic(full_name:&str)-> &'static str {
+    let Some(name) = full_name.strip_prefix("llvm.") else { unimplemented!("***** unsupported LLVM intrinsic {}", full_name) };
+    let Some((arch, name)) = name.split_once('.') else { unimplemented!("***** unsupported LLVM intrinsic llvm.{}", name) };
+    let old_arch_res = old_archs(arch, name);
+    if let ArchCheckResult::Ok(res) = old_arch_res {
+        return res;
+    }
+match arch {""")
         for arch in archs:
             if len(intrinsics[arch]) == 0:
                 continue
@@ -199,7 +201,13 @@ def update_intrinsics(llvm_path, llvmint, llvmint2):
                     out.write('    "{}" => "{}",\n'.format(llvm_name, entry[1]))
             out.write('    _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),\n')
             out.write("}} }} {}(name,full_name) }}\n,".format(arch))
-        out.write('    _ => unimplemented!("***** unsupported LLVM architecture {arch}, intrinsic:{full_name}"),\n')
+        out.write("""    _ => {
+        match old_arch_res {
+            ArchCheckResult::UnknownIntrinsic => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),
+            ArchCheckResult::UnknownArch => unimplemented!("***** unsupported LLVM architecture {arch}, intrinsic: {full_name}"),
+            ArchCheckResult::Ok(_) => unreachable!(),
+        }
+    }""")
         out.write("}\n}")
     subprocess.call(["rustfmt", output_file])
     print("Done!")
@@ -210,14 +218,6 @@ def main():
         os.path.dirname(os.path.abspath(__file__)),
         "llvm-project",
     )
-    llvmint_path = os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        "llvmint",
-    )
-    llvmint2_path = os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        "llvmint-2",
-    )
 
     # First, we clone the LLVM repository if it's not already here.
     clone_repository(
@@ -227,17 +227,7 @@ def main():
         branch="main",
         sub_paths=["llvm/include/llvm/IR", "llvm/include/llvm/CodeGen/"],
     )
-    clone_repository(
-        "llvmint",
-        llvmint_path,
-        "https://github.com/GuillaumeGomez/llvmint",
-    )
-    clone_repository(
-        "llvmint2",
-        llvmint2_path,
-        "https://github.com/antoyo/llvmint",
-    )
-    update_intrinsics(llvm_path, llvmint_path, llvmint2_path)
+    update_intrinsics(llvm_path)
 
 
 if __name__ == "__main__":

From 3dc60d0bf14638cbc2dcaf70e720c6f417608365 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Mon, 12 Jan 2026 16:48:14 +0100
Subject: [PATCH 0285/1843] Simplify intrinsics translation generation script

---
 tools/generate_intrinsics.py | 150 ++++++++---------------------------
 1 file changed, 33 insertions(+), 117 deletions(-)

diff --git a/tools/generate_intrinsics.py b/tools/generate_intrinsics.py
index 56b425cdb0cd..539032340777 100644
--- a/tools/generate_intrinsics.py
+++ b/tools/generate_intrinsics.py
@@ -12,7 +12,7 @@ def run_command(command, cwd=None):
         sys.exit(1)
 
 
-def clone_repository(repo_name, path, repo_url, branch="master", sub_paths=None):
+def clone_repository(repo_name, path, repo_url, sub_paths):
     if os.path.exists(path):
         while True:
             choice = input("There is already a `{}` folder, do you want to update it? [y/N]".format(path))
@@ -21,18 +21,15 @@ def clone_repository(repo_name, path, repo_url, branch="master", sub_paths=None)
                 return
             elif choice.lower() == "y":
                 print("Updating repository...")
-                run_command(["git", "pull", "origin", branch], cwd=path)
+                run_command(["git", "pull", "origin", "main"], cwd=path)
                 return
             else:
                 print("Didn't understand answer...")
     print("Cloning {} repository...".format(repo_name))
-    if sub_paths is None:
-        run_command(["git", "clone", repo_url, "--depth", "1", path])
-    else:
-        run_command(["git", "clone", repo_url, "--filter=tree:0", "--no-checkout", path])
-        run_command(["git", "sparse-checkout", "init"], cwd=path)
-        run_command(["git", "sparse-checkout", "set", *sub_paths], cwd=path)
-        run_command(["git", "checkout"], cwd=path)
+    run_command(["git", "clone", repo_url, "--filter=tree:0", "--no-checkout", path])
+    run_command(["git", "sparse-checkout", "init"], cwd=path)
+    run_command(["git", "sparse-checkout", "set", *sub_paths], cwd=path)
+    run_command(["git", "checkout"], cwd=path)
 
 
 def append_intrinsic(array, intrinsic_name, translation):
@@ -45,119 +42,36 @@ def convert_to_string(content):
     return content
 
 
-def extract_intrinsics_from_llvm(llvm_path, intrinsics):
-    command = ["llvm-tblgen", "llvm/IR/Intrinsics.td"]
+def extract_intrinsics_from_llvm(llvm_path):
+    intrinsics = {}
+    command = ["llvm-tblgen", "llvm/IR/Intrinsics.td", "--dump-json"]
     cwd = os.path.join(llvm_path, "llvm/include")
     print("=> Running command `{}` from `{}`".format(command, cwd))
     p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE)
     output, err = p.communicate()
-    lines = convert_to_string(output).splitlines()
-    pos = 0
-    while pos < len(lines):
-        line = lines[pos]
-        if not line.startswith("def "):
-            pos += 1
+    content = json.loads(convert_to_string(output))
+    for intrinsic in content:
+        data = content[intrinsic]
+        if not isinstance(data, dict):
             continue
-        intrinsic = line.split(" ")[1].strip()
-        content = line
-        while pos < len(lines):
-            line = lines[pos].split(" // ")[0].strip()
-            content += line
-            pos += 1
-            if line == "}":
-                break
-        entries = re.findall('string ClangBuiltinName = "(\\w+)";', content)
-        current_arch = re.findall('string TargetPrefix = "(\\w+)";', content)
-        if len(entries) == 1 and len(current_arch) == 1:
-            current_arch = current_arch[0]
-            intrinsic = intrinsic.split("_")
-            if len(intrinsic) < 2 or intrinsic[0] != "int":
-                continue
-            intrinsic[0] = "llvm"
-            intrinsic = ".".join(intrinsic)
-            if current_arch not in intrinsics:
-                intrinsics[current_arch] = []
-            append_intrinsic(intrinsics[current_arch], intrinsic, entries[0])
-
-
-def append_translation(json_data, p, array):
-    it = json_data["index"][p]
-    content = it["docs"].split('`')
-    if len(content) != 5:
-        return
-    append_intrinsic(array, content[1], content[3])
-
-
-def extract_intrinsics_from_llvmint(llvmint, intrinsics):
-    archs = [
-        "AMDGPU",
-        "aarch64",
-        "arm",
-        "cuda",
-        "hexagon",
-        "mips",
-        "nvvm",
-        "ppc",
-        "ptx",
-        "x86",
-        "xcore",
-    ]
-
-    json_file = os.path.join(llvmint, "target/doc/llvmint.json")
-    # We need to regenerate the documentation!
-    run_command(
-        ["cargo", "rustdoc", "--", "-Zunstable-options", "--output-format", "json"],
-        cwd=llvmint,
-    )
-    with open(json_file, "r", encoding="utf8") as f:
-        json_data = json.loads(f.read())
-    for p in json_data["paths"]:
-        it = json_data["paths"][p]
-        if it["crate_id"] != 0:
-            # This is from an external crate.
+        current_arch = data.get("TargetPrefix")
+        builtin_name = data.get("ClangBuiltinName")
+        if current_arch is None or current_arch == "" or builtin_name is None:
             continue
-        if it["kind"] != "function":
-            # We're only looking for functions.
+        intrinsic = intrinsic.split("_")
+        if len(intrinsic) < 2 or intrinsic[0] != "int":
             continue
-        # if len(it["path"]) == 2:
-        #   # This is a "general" intrinsic, not bound to a specific arch.
-        #   append_translation(json_data, p, general)
-        #   continue
-        if len(it["path"]) != 3 or it["path"][1] not in archs:
-            continue
-        arch = it["path"][1]
-        if arch not in intrinsics:
-            intrinsics[arch] = []
-        append_translation(json_data, p, intrinsics[arch])
+        intrinsic[0] = "llvm"
+        intrinsic = ".".join(intrinsic)
+        if current_arch not in intrinsics:
+            intrinsics[current_arch] = []
+        append_intrinsic(intrinsics[current_arch], intrinsic, builtin_name)
 
-
-def fill_intrinsics(intrinsics, from_intrinsics, all_intrinsics):
-    for arch in from_intrinsics:
-        if arch not in intrinsics:
-            intrinsics[arch] = []
-        for entry in from_intrinsics[arch]:
-            if entry[0] in all_intrinsics:
-                if all_intrinsics[entry[0]] == entry[1]:
-                    # This is a "full" duplicate, both the LLVM instruction and the GCC
-                    # translation are the same.
-                    continue
-                intrinsics[arch].append((entry[0], entry[1], True))
-            else:
-                intrinsics[arch].append((entry[0], entry[1], False))
-                all_intrinsics[entry[0]] = entry[1]
+    return intrinsics
 
 
 def update_intrinsics(llvm_path):
-    intrinsics_llvm = {}
-    intrinsics_llvmint = {}
-    all_intrinsics = {}
-
-    extract_intrinsics_from_llvm(llvm_path, intrinsics_llvm)
-
-    intrinsics = {}
-    # We give priority to translations from LLVM over the ones from llvmint.
-    fill_intrinsics(intrinsics, intrinsics_llvm, all_intrinsics)
-    fill_intrinsics(intrinsics, intrinsics_llvmint, all_intrinsics)
+    intrinsics = extract_intrinsics_from_llvm(llvm_path)
 
     archs = [arch for arch in intrinsics]
     archs.sort()
@@ -187,15 +101,13 @@ match arch {""")
                 continue
             attribute = "#[expect(non_snake_case)]" if arch[0].isupper() else ""
             out.write("\"{}\" => {{ {} fn {}(name: &str,full_name:&str) -> &'static str {{ match name {{".format(arch, attribute, arch))
-            intrinsics[arch].sort(key=lambda x: (x[0], x[2]))
+            intrinsics[arch].sort(key=lambda x: (x[0], x[1]))
             out.write('    // {}\n'.format(arch))
             for entry in intrinsics[arch]:
                 llvm_name = entry[0].removeprefix("llvm.");
                 llvm_name = llvm_name.removeprefix(arch);
                 llvm_name = llvm_name.removeprefix(".");
-                if entry[2] is True: # if it is a duplicate
-                    out.write('    // [DUPLICATE]: "{}" => "{}",\n'.format(llvm_name, entry[1]))
-                elif "_round_mask" in entry[1]:
+                if "_round_mask" in entry[1]:
                     out.write('    // [INVALID CONVERSION]: "{}" => "{}",\n'.format(llvm_name, entry[1]))
                 else:
                     out.write('    "{}" => "{}",\n'.format(llvm_name, entry[1]))
@@ -224,11 +136,15 @@ def main():
         "llvm-project",
         llvm_path,
         "https://github.com/llvm/llvm-project",
-        branch="main",
-        sub_paths=["llvm/include/llvm/IR", "llvm/include/llvm/CodeGen/"],
+        ["llvm/include/llvm/IR", "llvm/include/llvm/CodeGen/"],
     )
     update_intrinsics(llvm_path)
 
+# llvm-tblgen can be built with:
+#
+# mkdir llvm-tblgen-build && cd llvm-tblgen-build
+# cmake -G Ninja  -DLLVM_ENABLE_PROJECTS="llvm"  -DCMAKE_BUILD_TYPE=Release  ../llvm
+# ninja llvm-tblgen
 
 if __name__ == "__main__":
     sys.exit(main())

From c848b28a5d20b512e7edcd03da5ba616fabf0a8d Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Mon, 12 Jan 2026 17:05:01 +0100
Subject: [PATCH 0286/1843] Ignore `src/intrinsic/old_archs.rs` for typo checks

---
 .cspell.json | 1 +
 _typos.toml  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.cspell.json b/.cspell.json
index 388ccce2b091..556432d69a41 100644
--- a/.cspell.json
+++ b/.cspell.json
@@ -18,6 +18,7 @@
     ],
     "ignorePaths": [
         "src/intrinsic/archs.rs",
+        "src/intrinsic/old_archs.rs",
         "src/intrinsic/llvm.rs"
     ],
     "ignoreRegExpList": [
diff --git a/_typos.toml b/_typos.toml
index 4a6a506a981a..c4918b114621 100644
--- a/_typos.toml
+++ b/_typos.toml
@@ -6,4 +6,4 @@ seh = "seh"
 typ = "typ"
 
 [files]
-extend-exclude = ["src/intrinsic/archs.rs"]
+extend-exclude = ["src/intrinsic/archs.rs", "src/intrinsic/old_archs.rs"]

From 4f59819c26c72f86baa98a2805d411cd5b665759 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Fri, 16 Jan 2026 17:46:34 +0100
Subject: [PATCH 0287/1843] Regenerate intrinsics

---
 src/intrinsic/archs.rs | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/src/intrinsic/archs.rs b/src/intrinsic/archs.rs
index 89a6cf7f3d6e..3c1698df6dec 100644
--- a/src/intrinsic/archs.rs
+++ b/src/intrinsic/archs.rs
@@ -24,6 +24,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "gcsss" => "__builtin_arm_gcsss",
                     "isb" => "__builtin_arm_isb",
                     "prefetch" => "__builtin_arm_prefetch",
+                    "range.prefetch" => "__builtin_arm_range_prefetch",
                     "sme.in.streaming.mode" => "__builtin_arm_in_streaming_mode",
                     "sve.aesd" => "__builtin_sve_svaesd_u8",
                     "sve.aese" => "__builtin_sve_svaese_u8",
@@ -414,6 +415,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready",
                     "s.wait.tensorcnt" => "__builtin_amdgcn_s_wait_tensorcnt",
                     "s.waitcnt" => "__builtin_amdgcn_s_waitcnt",
+                    "s.wakeup.barrier" => "__builtin_amdgcn_s_wakeup_barrier",
                     "sad.hi.u8" => "__builtin_amdgcn_sad_hi_u8",
                     "sad.u16" => "__builtin_amdgcn_sad_u16",
                     "sad.u8" => "__builtin_amdgcn_sad_u8",
@@ -4836,19 +4838,24 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "add.rm.d" => "__nvvm_add_rm_d",
                     "add.rm.f" => "__nvvm_add_rm_f",
                     "add.rm.ftz.f" => "__nvvm_add_rm_ftz_f",
+                    "add.rm.ftz.sat.f" => "__nvvm_add_rm_ftz_sat_f",
+                    "add.rm.sat.f" => "__nvvm_add_rm_sat_f",
                     "add.rn.d" => "__nvvm_add_rn_d",
                     "add.rn.f" => "__nvvm_add_rn_f",
                     "add.rn.ftz.f" => "__nvvm_add_rn_ftz_f",
+                    "add.rn.ftz.sat.f" => "__nvvm_add_rn_ftz_sat_f",
+                    "add.rn.sat.f" => "__nvvm_add_rn_sat_f",
                     "add.rp.d" => "__nvvm_add_rp_d",
                     "add.rp.f" => "__nvvm_add_rp_f",
                     "add.rp.ftz.f" => "__nvvm_add_rp_ftz_f",
+                    "add.rp.ftz.sat.f" => "__nvvm_add_rp_ftz_sat_f",
+                    "add.rp.sat.f" => "__nvvm_add_rp_sat_f",
                     "add.rz.d" => "__nvvm_add_rz_d",
                     "add.rz.f" => "__nvvm_add_rz_f",
                     "add.rz.ftz.f" => "__nvvm_add_rz_ftz_f",
+                    "add.rz.ftz.sat.f" => "__nvvm_add_rz_ftz_sat_f",
+                    "add.rz.sat.f" => "__nvvm_add_rz_sat_f",
                     "bar.warp.sync" => "__nvvm_bar_warp_sync",
-                    "barrier0.and" => "__nvvm_bar0_and",
-                    "barrier0.or" => "__nvvm_bar0_or",
-                    "barrier0.popc" => "__nvvm_bar0_popc",
                     "bf16x2.to.ue8m0x2.rp" => "__nvvm_bf16x2_to_ue8m0x2_rp",
                     "bf16x2.to.ue8m0x2.rp.satfinite" => "__nvvm_bf16x2_to_ue8m0x2_rp_satfinite",
                     "bf16x2.to.ue8m0x2.rz" => "__nvvm_bf16x2_to_ue8m0x2_rz",
@@ -5050,6 +5057,8 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "fma.rm.d" => "__nvvm_fma_rm_d",
                     "fma.rm.f" => "__nvvm_fma_rm_f",
                     "fma.rm.ftz.f" => "__nvvm_fma_rm_ftz_f",
+                    "fma.rm.ftz.sat.f" => "__nvvm_fma_rm_ftz_sat_f",
+                    "fma.rm.sat.f" => "__nvvm_fma_rm_sat_f",
                     "fma.rn.bf16" => "__nvvm_fma_rn_bf16",
                     "fma.rn.bf16x2" => "__nvvm_fma_rn_bf16x2",
                     "fma.rn.d" => "__nvvm_fma_rn_d",
@@ -5061,16 +5070,22 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "fma.rn.ftz.relu.bf16x2" => "__nvvm_fma_rn_ftz_relu_bf16x2",
                     "fma.rn.ftz.sat.bf16" => "__nvvm_fma_rn_ftz_sat_bf16",
                     "fma.rn.ftz.sat.bf16x2" => "__nvvm_fma_rn_ftz_sat_bf16x2",
+                    "fma.rn.ftz.sat.f" => "__nvvm_fma_rn_ftz_sat_f",
                     "fma.rn.relu.bf16" => "__nvvm_fma_rn_relu_bf16",
                     "fma.rn.relu.bf16x2" => "__nvvm_fma_rn_relu_bf16x2",
                     "fma.rn.sat.bf16" => "__nvvm_fma_rn_sat_bf16",
                     "fma.rn.sat.bf16x2" => "__nvvm_fma_rn_sat_bf16x2",
+                    "fma.rn.sat.f" => "__nvvm_fma_rn_sat_f",
                     "fma.rp.d" => "__nvvm_fma_rp_d",
                     "fma.rp.f" => "__nvvm_fma_rp_f",
                     "fma.rp.ftz.f" => "__nvvm_fma_rp_ftz_f",
+                    "fma.rp.ftz.sat.f" => "__nvvm_fma_rp_ftz_sat_f",
+                    "fma.rp.sat.f" => "__nvvm_fma_rp_sat_f",
                     "fma.rz.d" => "__nvvm_fma_rz_d",
                     "fma.rz.f" => "__nvvm_fma_rz_f",
                     "fma.rz.ftz.f" => "__nvvm_fma_rz_ftz_f",
+                    "fma.rz.ftz.sat.f" => "__nvvm_fma_rz_ftz_sat_f",
+                    "fma.rz.sat.f" => "__nvvm_fma_rz_sat_f",
                     "fmax.bf16" => "__nvvm_fmax_bf16",
                     "fmax.bf16x2" => "__nvvm_fmax_bf16x2",
                     "fmax.d" => "__nvvm_fmax_d",
@@ -5274,6 +5289,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "read.ptx.sreg.pm1" => "__nvvm_read_ptx_sreg_pm1",
                     "read.ptx.sreg.pm2" => "__nvvm_read_ptx_sreg_pm2",
                     "read.ptx.sreg.pm3" => "__nvvm_read_ptx_sreg_pm3",
+                    "read.ptx.sreg.pm4" => "__nvvm_read_ptx_sreg_pm4",
                     "read.ptx.sreg.smid" => "__nvvm_read_ptx_sreg_smid",
                     "read.ptx.sreg.tid.w" => "__nvvm_read_ptx_sreg_tid_w",
                     "read.ptx.sreg.tid.x" => "__nvvm_read_ptx_sreg_tid_x",
@@ -6370,6 +6386,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
             fn spv(name: &str, full_name: &str) -> &'static str {
                 match name {
                     // spv
+                    "group.memory.barrier.with.group.sync" => "__builtin_spirv_group_barrier",
                     "num.subgroups" => "__builtin_spirv_num_subgroups",
                     "subgroup.id" => "__builtin_spirv_subgroup_id",
                     "subgroup.local.invocation.id" => {
@@ -6377,6 +6394,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     }
                     "subgroup.max.size" => "__builtin_spirv_subgroup_max_size",
                     "subgroup.size" => "__builtin_spirv_subgroup_size",
+                    "wave.ballot" => "__builtin_spirv_subgroup_ballot",
                     _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),
                 }
             }
@@ -7711,8 +7729,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "avx.ptestnzc.256" => "__builtin_ia32_ptestnzc256",
                     "avx.ptestz.256" => "__builtin_ia32_ptestz256",
                     "avx.rcp.ps.256" => "__builtin_ia32_rcpps256",
-                    "avx.round.pd.256" => "__builtin_ia32_roundpd256",
-                    "avx.round.ps.256" => "__builtin_ia32_roundps256",
                     "avx.rsqrt.ps.256" => "__builtin_ia32_rsqrtps256",
                     "avx.vpermilvar.pd" => "__builtin_ia32_vpermilvarpd",
                     "avx.vpermilvar.pd.256" => "__builtin_ia32_vpermilvarpd256",
@@ -8829,10 +8845,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "sse41.ptestc" => "__builtin_ia32_ptestc128",
                     "sse41.ptestnzc" => "__builtin_ia32_ptestnzc128",
                     "sse41.ptestz" => "__builtin_ia32_ptestz128",
-                    "sse41.round.pd" => "__builtin_ia32_roundpd",
-                    "sse41.round.ps" => "__builtin_ia32_roundps",
-                    "sse41.round.sd" => "__builtin_ia32_roundsd",
-                    "sse41.round.ss" => "__builtin_ia32_roundss",
                     "sse42.crc32.32.16" => "__builtin_ia32_crc32hi",
                     "sse42.crc32.32.32" => "__builtin_ia32_crc32si",
                     "sse42.crc32.32.8" => "__builtin_ia32_crc32qi",
@@ -8869,10 +8881,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "ssse3.psign.w.128" => "__builtin_ia32_psignw128",
                     "sttilecfg" => "__builtin_ia32_tile_storeconfig",
                     "stui" => "__builtin_ia32_stui",
-                    "t2rpntlvwz0rs" => "__builtin_ia32_t2rpntlvwz0rs",
-                    "t2rpntlvwz0rst1" => "__builtin_ia32_t2rpntlvwz0rst1",
-                    "t2rpntlvwz1rs" => "__builtin_ia32_t2rpntlvwz1rs",
-                    "t2rpntlvwz1rst1" => "__builtin_ia32_t2rpntlvwz1rst1",
                     "tbm.bextri.u32" => "__builtin_ia32_bextri_u32",
                     "tbm.bextri.u64" => "__builtin_ia32_bextri_u64",
                     "tcmmimfp16ps" => "__builtin_ia32_tcmmimfp16ps",
@@ -8881,14 +8889,19 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "tcmmrlfp16ps.internal" => "__builtin_ia32_tcmmrlfp16ps_internal",
                     "tcvtrowd2ps" => "__builtin_ia32_tcvtrowd2ps",
                     "tcvtrowd2ps.internal" => "__builtin_ia32_tcvtrowd2ps_internal",
+                    "tcvtrowd2psi" => "__builtin_ia32_tcvtrowd2psi",
                     "tcvtrowps2bf16h" => "__builtin_ia32_tcvtrowps2bf16h",
                     "tcvtrowps2bf16h.internal" => "__builtin_ia32_tcvtrowps2bf16h_internal",
+                    "tcvtrowps2bf16hi" => "__builtin_ia32_tcvtrowps2bf16hi",
                     "tcvtrowps2bf16l" => "__builtin_ia32_tcvtrowps2bf16l",
                     "tcvtrowps2bf16l.internal" => "__builtin_ia32_tcvtrowps2bf16l_internal",
+                    "tcvtrowps2bf16li" => "__builtin_ia32_tcvtrowps2bf16li",
                     "tcvtrowps2phh" => "__builtin_ia32_tcvtrowps2phh",
                     "tcvtrowps2phh.internal" => "__builtin_ia32_tcvtrowps2phh_internal",
+                    "tcvtrowps2phhi" => "__builtin_ia32_tcvtrowps2phhi",
                     "tcvtrowps2phl" => "__builtin_ia32_tcvtrowps2phl",
                     "tcvtrowps2phl.internal" => "__builtin_ia32_tcvtrowps2phl_internal",
+                    "tcvtrowps2phli" => "__builtin_ia32_tcvtrowps2phli",
                     "tdpbf16ps" => "__builtin_ia32_tdpbf16ps",
                     "tdpbf16ps.internal" => "__builtin_ia32_tdpbf16ps_internal",
                     "tdpbf8ps" => "__builtin_ia32_tdpbf8ps",
@@ -8920,6 +8933,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
                     "tileloaddt164.internal" => "__builtin_ia32_tileloaddt164_internal",
                     "tilemovrow" => "__builtin_ia32_tilemovrow",
                     "tilemovrow.internal" => "__builtin_ia32_tilemovrow_internal",
+                    "tilemovrowi" => "__builtin_ia32_tilemovrowi",
                     "tilerelease" => "__builtin_ia32_tilerelease",
                     "tilestored64" => "__builtin_ia32_tilestored64",
                     "tilestored64.internal" => "__builtin_ia32_tilestored64_internal",

From 4cb70c8e4a18c073703d3ccda8adeb2931fe4fc6 Mon Sep 17 00:00:00 2001
From: linshuy2 
Date: Fri, 16 Jan 2026 03:46:38 +0000
Subject: [PATCH 0288/1843] fix: `unnecessary_sort_by` FN on field access

---
 .../src/methods/unnecessary_sort_by.rs        | 24 ++++++++++-----
 tests/ui/unnecessary_sort_by.fixed            | 30 +++++++++++++++++++
 tests/ui/unnecessary_sort_by.rs               | 30 +++++++++++++++++++
 tests/ui/unnecessary_sort_by.stderr           | 20 ++++++++++++-
 4 files changed, 96 insertions(+), 8 deletions(-)

diff --git a/clippy_lints/src/methods/unnecessary_sort_by.rs b/clippy_lints/src/methods/unnecessary_sort_by.rs
index 9dddbe814317..c7339a3bdd05 100644
--- a/clippy_lints/src/methods/unnecessary_sort_by.rs
+++ b/clippy_lints/src/methods/unnecessary_sort_by.rs
@@ -2,12 +2,11 @@ use clippy_utils::diagnostics::span_lint_and_then;
 use clippy_utils::res::{MaybeDef, MaybeTypeckRes};
 use clippy_utils::std_or_core;
 use clippy_utils::sugg::Sugg;
-use clippy_utils::ty::implements_trait;
+use clippy_utils::ty::{implements_trait, is_copy};
 use rustc_errors::Applicability;
 use rustc_hir::{Closure, Expr, ExprKind, Mutability, Param, Pat, PatKind, Path, PathSegment, QPath};
 use rustc_lint::LateContext;
-use rustc_middle::ty;
-use rustc_middle::ty::GenericArgKind;
+use rustc_middle::ty::{self, GenericArgKind, Ty};
 use rustc_span::sym;
 use rustc_span::symbol::{Ident, Symbol};
 use std::iter;
@@ -70,7 +69,7 @@ fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: Ident, b_expr: &Expr<'_>, b_ident:
             mirrored_exprs(left_block, a_ident, right_block, b_ident)
         },
         (ExprKind::Field(left_expr, left_ident), ExprKind::Field(right_expr, right_ident)) => {
-            left_ident.name == right_ident.name && mirrored_exprs(left_expr, a_ident, right_expr, right_ident)
+            left_ident.name == right_ident.name && mirrored_exprs(left_expr, a_ident, right_expr, b_ident)
         },
         // Two paths: either one is a and the other is b, or they're identical to each other
         (
@@ -159,7 +158,11 @@ fn detect_lint<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, arg: &Expr<'_>) ->
             return Some(LintTrigger::Sort);
         }
 
-        if !expr_borrows(cx, left_expr) {
+        let left_expr_ty = cx.typeck_results().expr_ty(left_expr);
+        if !expr_borrows(left_expr_ty)
+            // Don't lint if the closure is accessing non-Copy fields
+            && (!expr_is_field_access(left_expr) || is_copy(cx, left_expr_ty))
+        {
             return Some(LintTrigger::SortByKey(SortByKeyDetection {
                 closure_arg,
                 closure_body,
@@ -171,11 +174,18 @@ fn detect_lint<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, arg: &Expr<'_>) ->
     None
 }
 
-fn expr_borrows(cx: &LateContext<'_>, expr: &Expr<'_>) -> bool {
-    let ty = cx.typeck_results().expr_ty(expr);
+fn expr_borrows(ty: Ty<'_>) -> bool {
     matches!(ty.kind(), ty::Ref(..)) || ty.walk().any(|arg| matches!(arg.kind(), GenericArgKind::Lifetime(_)))
 }
 
+fn expr_is_field_access(expr: &Expr<'_>) -> bool {
+    match expr.kind {
+        ExprKind::Field(_, _) => true,
+        ExprKind::AddrOf(_, Mutability::Not, inner) => expr_is_field_access(inner),
+        _ => false,
+    }
+}
+
 pub(super) fn check<'tcx>(
     cx: &LateContext<'tcx>,
     expr: &'tcx Expr<'_>,
diff --git a/tests/ui/unnecessary_sort_by.fixed b/tests/ui/unnecessary_sort_by.fixed
index 5255ab173efb..372cf683f223 100644
--- a/tests/ui/unnecessary_sort_by.fixed
+++ b/tests/ui/unnecessary_sort_by.fixed
@@ -111,3 +111,33 @@ fn main() {
     issue_5754::test();
     issue_6001::test();
 }
+
+fn issue16405() {
+    let mut v: Vec<(i32, &str)> = vec![(1, "foo"), (2, "bar")];
+
+    v.sort_by_key(|a| a.0);
+    //~^ unnecessary_sort_by
+
+    struct Item {
+        key: i32,
+        value: String,
+    }
+
+    let mut items = vec![
+        Item {
+            key: 2,
+            value: "b".to_string(),
+        },
+        Item {
+            key: 1,
+            value: "a".to_string(),
+        },
+    ];
+    items.sort_by_key(|item1| item1.key);
+    //~^ unnecessary_sort_by
+
+    items.sort_by(|item1, item2| item1.value.cmp(&item2.value));
+
+    items.sort_by_key(|item1| item1.value.clone());
+    //~^ unnecessary_sort_by
+}
diff --git a/tests/ui/unnecessary_sort_by.rs b/tests/ui/unnecessary_sort_by.rs
index 65db7ca3f137..7dd274623b35 100644
--- a/tests/ui/unnecessary_sort_by.rs
+++ b/tests/ui/unnecessary_sort_by.rs
@@ -111,3 +111,33 @@ fn main() {
     issue_5754::test();
     issue_6001::test();
 }
+
+fn issue16405() {
+    let mut v: Vec<(i32, &str)> = vec![(1, "foo"), (2, "bar")];
+
+    v.sort_by(|a, b| a.0.cmp(&b.0));
+    //~^ unnecessary_sort_by
+
+    struct Item {
+        key: i32,
+        value: String,
+    }
+
+    let mut items = vec![
+        Item {
+            key: 2,
+            value: "b".to_string(),
+        },
+        Item {
+            key: 1,
+            value: "a".to_string(),
+        },
+    ];
+    items.sort_by(|item1, item2| item1.key.cmp(&item2.key));
+    //~^ unnecessary_sort_by
+
+    items.sort_by(|item1, item2| item1.value.cmp(&item2.value));
+
+    items.sort_by(|item1, item2| item1.value.clone().cmp(&item2.value.clone()));
+    //~^ unnecessary_sort_by
+}
diff --git a/tests/ui/unnecessary_sort_by.stderr b/tests/ui/unnecessary_sort_by.stderr
index a066554037fe..b555245b0d01 100644
--- a/tests/ui/unnecessary_sort_by.stderr
+++ b/tests/ui/unnecessary_sort_by.stderr
@@ -73,5 +73,23 @@ error: consider using `sort_unstable_by_key`
 LL |         args.sort_unstable_by(|a, b| b.name().cmp(&a.name()));
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `args.sort_unstable_by_key(|b| std::cmp::Reverse(b.name()))`
 
-error: aborting due to 12 previous errors
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:118:5
+   |
+LL |     v.sort_by(|a, b| a.0.cmp(&b.0));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|a| a.0)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:136:5
+   |
+LL |     items.sort_by(|item1, item2| item1.key.cmp(&item2.key));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `items.sort_by_key(|item1| item1.key)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:141:5
+   |
+LL |     items.sort_by(|item1, item2| item1.value.clone().cmp(&item2.value.clone()));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `items.sort_by_key(|item1| item1.value.clone())`
+
+error: aborting due to 15 previous errors
 

From 03fd408af5657404b1d165f7c7679918ed0f536a Mon Sep 17 00:00:00 2001
From: linshuy2 
Date: Fri, 16 Jan 2026 05:08:40 +0000
Subject: [PATCH 0289/1843] Apply `unnecessary_sort_by` to Clippy itself

---
 clippy_utils/src/ty/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clippy_utils/src/ty/mod.rs b/clippy_utils/src/ty/mod.rs
index a90d64e972c1..979170d72006 100644
--- a/clippy_utils/src/ty/mod.rs
+++ b/clippy_utils/src/ty/mod.rs
@@ -824,7 +824,7 @@ impl AdtVariantInfo {
                 }
             })
             .collect::>();
-        variants_size.sort_by(|a, b| b.size.cmp(&a.size));
+        variants_size.sort_by_key(|b| std::cmp::Reverse(b.size));
         variants_size
     }
 }

From 4870a5fb692e0fb796a4db6a8259a22b047e14e9 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Thu, 1 Jan 2026 09:41:09 +0530
Subject: [PATCH 0290/1843] add worker abstraction

---
 .../src/bidirectional_protocol.rs             | 12 +--
 .../proc-macro-api/src/legacy_protocol.rs     |  4 +-
 .../crates/proc-macro-api/src/lib.rs          | 17 ++--
 .../crates/proc-macro-api/src/process.rs      | 94 +++++++++++++++++--
 4 files changed, 103 insertions(+), 24 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index 5996f882981c..643ba98f5176 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -21,7 +21,7 @@ use crate::{
             serialize_span_data_index_map,
         },
     },
-    process::ProcMacroServerProcess,
+    process::{ProcMacroWorker, SynIO},
     transport::codec::postcard::PostcardProtocol,
     version,
 };
@@ -84,7 +84,7 @@ fn wrap_decode(err: io::Error) -> ServerError {
 }
 
 pub(crate) fn version_check(
-    srv: &ProcMacroServerProcess,
+    srv: &dyn ProcMacroWorker,
     callback: SubCallback<'_>,
 ) -> Result {
     let request = BidirectionalMessage::Request(Request::ApiVersionCheck {});
@@ -101,7 +101,7 @@ pub(crate) fn version_check(
 
 /// Enable support for rust-analyzer span mode if the server supports it.
 pub(crate) fn enable_rust_analyzer_spans(
-    srv: &ProcMacroServerProcess,
+    srv: &dyn ProcMacroWorker,
     callback: SubCallback<'_>,
 ) -> Result {
     let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig {
@@ -120,7 +120,7 @@ pub(crate) fn enable_rust_analyzer_spans(
 
 /// Finds proc-macros in a given dynamic library.
 pub(crate) fn find_proc_macros(
-    srv: &ProcMacroServerProcess,
+    srv: &dyn ProcMacroWorker,
     dylib_path: &AbsPath,
     callback: SubCallback<'_>,
 ) -> Result, String>, ServerError> {
@@ -175,7 +175,7 @@ pub(crate) fn expand(
         current_dir: Some(current_dir),
     })));
 
-    let response_payload = run_request(&proc_macro.process, task, callback)?;
+    let response_payload = run_request(proc_macro.process.as_ref(), task, callback)?;
 
     match response_payload {
         BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it
@@ -205,7 +205,7 @@ pub(crate) fn expand(
 }
 
 fn run_request(
-    srv: &ProcMacroServerProcess,
+    srv: &dyn ProcMacroWorker,
     msg: BidirectionalMessage,
     callback: SubCallback<'_>,
 ) -> Result {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
index aabe5a011851..56bf863a88e9 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
@@ -113,7 +113,7 @@ pub(crate) fn expand(
         current_dir: Some(current_dir),
     };
 
-    let response = send_task(&proc_macro.process, Request::ExpandMacro(Box::new(task)))?;
+    let response = send_task(proc_macro.process.as_ref(), Request::ExpandMacro(Box::new(task)))?;
 
     match response {
         Response::ExpandMacro(it) => Ok(it
@@ -143,7 +143,7 @@ pub(crate) fn expand(
 }
 
 /// Sends a request to the proc-macro server and waits for a response.
-fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result {
+fn send_task(srv: &dyn ProcMacroWorker, req: Request) -> Result {
     if let Some(server_error) = srv.exited() {
         return Err(server_error.clone());
     }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index 01195c10feef..dd0c89103a2b 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -27,7 +27,10 @@ use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span};
 use std::{fmt, io, sync::Arc, time::SystemTime};
 
 pub use crate::transport::codec::Codec;
-use crate::{bidirectional_protocol::SubCallback, process::ProcMacroServerProcess};
+use crate::{
+    bidirectional_protocol::SubCallback,
+    process::{ProcMacroServerProcess, ProcMacroWorker},
+};
 
 /// The versions of the server protocol
 pub mod version {
@@ -85,7 +88,7 @@ pub struct ProcMacroClient {
     ///
     /// That means that concurrent salsa requests may block each other when expanding proc macros,
     /// which is unfortunate, but simple and good enough for the time being.
-    process: Arc,
+    worker: Arc,
     path: AbsPathBuf,
 }
 
@@ -107,7 +110,7 @@ impl MacroDylib {
 /// we share a single expander process for all macros within a workspace.
 #[derive(Debug, Clone)]
 pub struct ProcMacro {
-    process: Arc,
+    process: Arc,
     dylib_path: Arc,
     name: Box,
     kind: ProcMacroKind,
@@ -171,7 +174,7 @@ impl ProcMacroClient {
         version: Option<&Version>,
     ) -> io::Result {
         let process = ProcMacroServerProcess::run(spawn, version, || "".to_owned())?;
-        Ok(ProcMacroClient { process: Arc::new(process), path: process_path.to_owned() })
+        Ok(ProcMacroClient { worker: Arc::new(process), path: process_path.to_owned() })
     }
 
     /// Returns the absolute path to the proc-macro server.
@@ -186,7 +189,7 @@ impl ProcMacroClient {
         callback: Option>,
     ) -> Result, ServerError> {
         let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered();
-        let macros = self.process.find_proc_macros(&dylib.path, callback)?;
+        let macros = self.worker.find_proc_macros(&dylib.path, callback)?;
 
         let dylib_path = Arc::new(dylib.path);
         let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
@@ -196,7 +199,7 @@ impl ProcMacroClient {
             Ok(macros) => Ok(macros
                 .into_iter()
                 .map(|(name, kind)| ProcMacro {
-                    process: self.process.clone(),
+                    process: self.worker.clone(),
                     name: name.into(),
                     kind,
                     dylib_path: dylib_path.clone(),
@@ -209,7 +212,7 @@ impl ProcMacroClient {
 
     /// Checks if the proc-macro server has exited.
     pub fn exited(&self) -> Option<&ServerError> {
-        self.process.exited()
+        self.worker.exited()
     }
 }
 
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index cd387dad0d0b..a206e9fc5d90 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -1,8 +1,9 @@
 //! Handle process life-time and message passing for proc-macro client
 
 use std::{
+    fmt::Debug,
     io::{self, BufRead, BufReader, Read, Write},
-    panic::AssertUnwindSafe,
+    panic::{AssertUnwindSafe, RefUnwindSafe},
     process::{Child, ChildStdin, ChildStdout, Command, Stdio},
     sync::{Arc, Mutex, OnceLock},
 };
@@ -74,12 +75,79 @@ impl ProcessExit for Process {
 }
 
 /// Maintains the state of the proc-macro server process.
-struct ProcessSrvState {
+pub(crate) struct ProcessSrvState {
     process: Box,
     stdin: Box,
     stdout: Box,
 }
 
+impl ProcMacroWorker for ProcMacroServerProcess {
+    fn find_proc_macros(
+        &self,
+        dylib_path: &AbsPath,
+        callback: Option>,
+    ) -> Result, String>, ServerError> {
+        ProcMacroServerProcess::find_proc_macros(self, dylib_path, callback)
+    }
+
+    fn expand(
+        &self,
+        proc_macro: &ProcMacro,
+        subtree: tt::SubtreeView<'_>,
+        attr: Option>,
+        env: Vec<(String, String)>,
+        def_site: Span,
+        call_site: Span,
+        mixed_site: Span,
+        current_dir: String,
+        callback: Option>,
+    ) -> Result, ServerError> {
+        ProcMacroServerProcess::expand(
+            self,
+            proc_macro,
+            subtree,
+            attr,
+            env,
+            def_site,
+            call_site,
+            mixed_site,
+            current_dir,
+            callback,
+        )
+    }
+
+    fn exited(&self) -> Option<&ServerError> {
+        ProcMacroServerProcess::exited(self)
+    }
+
+    fn version(&self) -> u32 {
+        ProcMacroServerProcess::version(self)
+    }
+
+    fn rust_analyzer_spans(&self) -> bool {
+        ProcMacroServerProcess::rust_analyzer_spans(self)
+    }
+
+    fn enable_rust_analyzer_spans(
+        &self,
+        callback: Option>,
+    ) -> Result {
+        ProcMacroServerProcess::enable_rust_analyzer_spans(self, callback)
+    }
+
+    fn use_postcard(&self) -> bool {
+        ProcMacroServerProcess::use_postcard(self)
+    }
+
+    fn state(&self) -> &Mutex {
+        &self.state
+    }
+
+    fn get_exited(&self) -> &OnceLock> {
+        &self.exited
+    }
+}
+
 impl ProcMacroServerProcess {
     /// Starts the proc-macro server and performs a version check
     pub(crate) fn spawn<'a>(
@@ -291,9 +359,13 @@ impl ProcMacroServerProcess {
             ),
         }
     }
+}
 
+pub(crate) struct SynIO;
+
+impl SynIO {
     pub(crate) fn send_task(
-        &self,
+        proc_macro_worker: &dyn ProcMacroWorker,
         send: impl FnOnce(
             &mut dyn Write,
             &mut dyn BufRead,
@@ -302,7 +374,7 @@ impl ProcMacroServerProcess {
         ) -> Result, ServerError>,
         req: Request,
     ) -> Result {
-        self.with_locked_io::(|writer, reader, buf| {
+        SynIO::with_locked_io::(proc_macro_worker, |writer, reader, buf| {
             send(writer, reader, req, buf).and_then(|res| {
                 res.ok_or_else(|| {
                     let message = "proc-macro server did not respond with data".to_owned();
@@ -319,10 +391,10 @@ impl ProcMacroServerProcess {
     }
 
     pub(crate) fn with_locked_io(
-        &self,
+        proc_macro_worker: &dyn ProcMacroWorker,
         f: impl FnOnce(&mut dyn Write, &mut dyn BufRead, &mut C::Buf) -> Result,
     ) -> Result {
-        let state = &mut *self.state.lock().unwrap();
+        let state = &mut *proc_macro_worker.state().lock().unwrap();
         let mut buf = C::Buf::default();
 
         f(&mut state.stdin, &mut state.stdout, &mut buf).map_err(|e| {
@@ -330,7 +402,11 @@ impl ProcMacroServerProcess {
                 match state.process.exit_err() {
                     None => e,
                     Some(server_error) => {
-                        self.exited.get_or_init(|| AssertUnwindSafe(server_error)).0.clone()
+                        proc_macro_worker
+                            .get_exited()
+                            .get_or_init(|| AssertUnwindSafe(server_error))
+                            .0
+                            .clone()
                     }
                 }
             } else {
@@ -340,11 +416,11 @@ impl ProcMacroServerProcess {
     }
 
     pub(crate) fn run_bidirectional(
-        &self,
+        proc_macro_worker: &dyn ProcMacroWorker,
         initial: BidirectionalMessage,
         callback: SubCallback<'_>,
     ) -> Result {
-        self.with_locked_io::(|writer, reader, buf| {
+        SynIO::with_locked_io::(proc_macro_worker, |writer, reader, buf| {
             bidirectional_protocol::run_conversation::(writer, reader, buf, initial, callback)
         })
     }

From 98d4496a1e9f03a5193e85304f069757d8032768 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Thu, 1 Jan 2026 13:39:04 +0530
Subject: [PATCH 0291/1843] add termination flag to procmacroserverprocess

---
 .../crates/proc-macro-api/src/process.rs      | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index a206e9fc5d90..efb8e0e84a1f 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -29,6 +29,7 @@ pub(crate) struct ProcMacroServerProcess {
     protocol: Protocol,
     /// Populated when the server exits.
     exited: OnceLock>,
+    single_use: bool,
 }
 
 impl std::fmt::Debug for ProcMacroServerProcess {
@@ -146,6 +147,10 @@ impl ProcMacroWorker for ProcMacroServerProcess {
     fn get_exited(&self) -> &OnceLock> {
         &self.exited
     }
+
+    fn is_reusable(&self) -> bool {
+        !self.single_use
+    }
 }
 
 impl ProcMacroServerProcess {
@@ -226,6 +231,7 @@ impl ProcMacroServerProcess {
                         }
                     },
                     exited: OnceLock::new(),
+                    single_use,
                 })
             };
             let mut srv = create_srv()?;
@@ -335,7 +341,7 @@ impl ProcMacroServerProcess {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
-        match self.protocol {
+        let result = match self.protocol {
             Protocol::LegacyJson { .. } => legacy_protocol::expand(
                 proc_macro,
                 subtree,
@@ -357,6 +363,18 @@ impl ProcMacroServerProcess {
                 current_dir,
                 callback.expect("callback required for bidirectional protocol"),
             ),
+        };
+
+        if self.is_reusable() {
+            self.terminate();
+        }
+
+        result
+    }
+
+    fn terminate(&self) {
+        if let Ok(mut state) = self.state.lock() {
+            let _ = state.process.child.kill();
         }
     }
 }

From 9fb5d34626d2ce137167e556d53c01f04e410329 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Thu, 1 Jan 2026 16:05:33 +0530
Subject: [PATCH 0292/1843] add pool of processes

---
 .../src/bidirectional_protocol.rs             |  10 +-
 .../proc-macro-api/src/legacy_protocol.rs     |   2 +-
 .../crates/proc-macro-api/src/lib.rs          |  31 +---
 .../crates/proc-macro-api/src/pool.rs         |  61 ++++++++
 .../crates/proc-macro-api/src/process.rs      | 147 ++++++------------
 5 files changed, 122 insertions(+), 129 deletions(-)
 create mode 100644 src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index 643ba98f5176..137f2dafc0de 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -21,7 +21,7 @@ use crate::{
             serialize_span_data_index_map,
         },
     },
-    process::{ProcMacroWorker, SynIO},
+    process::ProcMacroServerProcess,
     transport::codec::postcard::PostcardProtocol,
     version,
 };
@@ -84,7 +84,7 @@ fn wrap_decode(err: io::Error) -> ServerError {
 }
 
 pub(crate) fn version_check(
-    srv: &dyn ProcMacroWorker,
+    srv: &ProcMacroServerProcess,
     callback: SubCallback<'_>,
 ) -> Result {
     let request = BidirectionalMessage::Request(Request::ApiVersionCheck {});
@@ -101,7 +101,7 @@ pub(crate) fn version_check(
 
 /// Enable support for rust-analyzer span mode if the server supports it.
 pub(crate) fn enable_rust_analyzer_spans(
-    srv: &dyn ProcMacroWorker,
+    srv: &ProcMacroServerProcess,
     callback: SubCallback<'_>,
 ) -> Result {
     let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig {
@@ -120,7 +120,7 @@ pub(crate) fn enable_rust_analyzer_spans(
 
 /// Finds proc-macros in a given dynamic library.
 pub(crate) fn find_proc_macros(
-    srv: &dyn ProcMacroWorker,
+    srv: &ProcMacroServerProcess,
     dylib_path: &AbsPath,
     callback: SubCallback<'_>,
 ) -> Result, String>, ServerError> {
@@ -205,7 +205,7 @@ pub(crate) fn expand(
 }
 
 fn run_request(
-    srv: &dyn ProcMacroWorker,
+    srv: &ProcMacroServerProcess,
     msg: BidirectionalMessage,
     callback: SubCallback<'_>,
 ) -> Result {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
index 56bf863a88e9..7b546cf7aef6 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
@@ -143,7 +143,7 @@ pub(crate) fn expand(
 }
 
 /// Sends a request to the proc-macro server and waits for a response.
-fn send_task(srv: &dyn ProcMacroWorker, req: Request) -> Result {
+fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result {
     if let Some(server_error) = srv.exited() {
         return Err(server_error.clone());
     }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index dd0c89103a2b..ffae28f92c05 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -18,6 +18,7 @@ extern crate rustc_driver as _;
 
 pub mod bidirectional_protocol;
 pub mod legacy_protocol;
+pub mod pool;
 pub mod process;
 pub mod transport;
 
@@ -29,7 +30,8 @@ use std::{fmt, io, sync::Arc, time::SystemTime};
 pub use crate::transport::codec::Codec;
 use crate::{
     bidirectional_protocol::SubCallback,
-    process::{ProcMacroServerProcess, ProcMacroWorker},
+    pool::{ProcMacroServerPool, default_pool_size},
+    process::ProcMacroServerProcess,
 };
 
 /// The versions of the server protocol
@@ -88,7 +90,7 @@ pub struct ProcMacroClient {
     ///
     /// That means that concurrent salsa requests may block each other when expanding proc macros,
     /// which is unfortunate, but simple and good enough for the time being.
-    worker: Arc,
+    pool: Arc,
     path: AbsPathBuf,
 }
 
@@ -110,7 +112,7 @@ impl MacroDylib {
 /// we share a single expander process for all macros within a workspace.
 #[derive(Debug, Clone)]
 pub struct ProcMacro {
-    process: Arc,
+    process: Arc,
     dylib_path: Arc,
     name: Box,
     kind: ProcMacroKind,
@@ -188,31 +190,12 @@ impl ProcMacroClient {
         dylib: MacroDylib,
         callback: Option>,
     ) -> Result, ServerError> {
-        let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered();
-        let macros = self.worker.find_proc_macros(&dylib.path, callback)?;
-
-        let dylib_path = Arc::new(dylib.path);
-        let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
-            .ok()
-            .and_then(|metadata| metadata.modified().ok());
-        match macros {
-            Ok(macros) => Ok(macros
-                .into_iter()
-                .map(|(name, kind)| ProcMacro {
-                    process: self.worker.clone(),
-                    name: name.into(),
-                    kind,
-                    dylib_path: dylib_path.clone(),
-                    dylib_last_modified,
-                })
-                .collect()),
-            Err(message) => Err(ServerError { message, io: None }),
-        }
+        self.pool.load_dylib(&dylib, callback)
     }
 
     /// Checks if the proc-macro server has exited.
     pub fn exited(&self) -> Option<&ServerError> {
-        self.worker.exited()
+        self.pool.exited()
     }
 }
 
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
new file mode 100644
index 000000000000..685bc05be62a
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -0,0 +1,61 @@
+use std::sync::Arc;
+
+use crate::{
+    MacroDylib, ProcMacro, ServerError, bidirectional_protocol::SubCallback,
+    process::ProcMacroServerProcess,
+};
+
+#[derive(Debug)]
+pub(crate) struct ProcMacroServerPool {
+    workers: Vec>,
+}
+
+impl ProcMacroServerPool {
+    pub(crate) fn new(workers: Vec>) -> Self {
+        Self { workers }
+    }
+}
+
+impl ProcMacroServerPool {
+    pub(crate) fn exited(&self) -> Option<&ServerError> {
+        for worker in &self.workers {
+            if let Some(e) = worker.exited() {
+                return Some(e);
+            }
+        }
+        None
+    }
+
+    pub(crate) fn load_dylib(
+        &self,
+        dylib: &MacroDylib,
+        _callback: Option>,
+    ) -> Result, ServerError> {
+        let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered();
+        let mut all_macros = Vec::new();
+
+        for worker in &self.workers {
+            let dylib_path = Arc::new(dylib.path.clone());
+            let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
+                .ok()
+                .and_then(|metadata| metadata.modified().ok());
+            let macros = worker.load_dylib(&dylib.path, None)?;
+
+            for (name, kind) in macros {
+                all_macros.push(ProcMacro {
+                    process: worker.clone(),
+                    name: name.into(),
+                    kind,
+                    dylib_path: Arc::new(dylib.path.clone()),
+                    dylib_last_modified,
+                });
+            }
+        }
+
+        Ok(all_macros)
+    }
+}
+
+pub(crate) fn default_pool_size() -> usize {
+    std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1).min(4)
+}
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index efb8e0e84a1f..30877c5cf491 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -3,7 +3,7 @@
 use std::{
     fmt::Debug,
     io::{self, BufRead, BufReader, Read, Write},
-    panic::{AssertUnwindSafe, RefUnwindSafe},
+    panic::AssertUnwindSafe,
     process::{Child, ChildStdin, ChildStdout, Command, Stdio},
     sync::{Arc, Mutex, OnceLock},
 };
@@ -82,77 +82,6 @@ pub(crate) struct ProcessSrvState {
     stdout: Box,
 }
 
-impl ProcMacroWorker for ProcMacroServerProcess {
-    fn find_proc_macros(
-        &self,
-        dylib_path: &AbsPath,
-        callback: Option>,
-    ) -> Result, String>, ServerError> {
-        ProcMacroServerProcess::find_proc_macros(self, dylib_path, callback)
-    }
-
-    fn expand(
-        &self,
-        proc_macro: &ProcMacro,
-        subtree: tt::SubtreeView<'_>,
-        attr: Option>,
-        env: Vec<(String, String)>,
-        def_site: Span,
-        call_site: Span,
-        mixed_site: Span,
-        current_dir: String,
-        callback: Option>,
-    ) -> Result, ServerError> {
-        ProcMacroServerProcess::expand(
-            self,
-            proc_macro,
-            subtree,
-            attr,
-            env,
-            def_site,
-            call_site,
-            mixed_site,
-            current_dir,
-            callback,
-        )
-    }
-
-    fn exited(&self) -> Option<&ServerError> {
-        ProcMacroServerProcess::exited(self)
-    }
-
-    fn version(&self) -> u32 {
-        ProcMacroServerProcess::version(self)
-    }
-
-    fn rust_analyzer_spans(&self) -> bool {
-        ProcMacroServerProcess::rust_analyzer_spans(self)
-    }
-
-    fn enable_rust_analyzer_spans(
-        &self,
-        callback: Option>,
-    ) -> Result {
-        ProcMacroServerProcess::enable_rust_analyzer_spans(self, callback)
-    }
-
-    fn use_postcard(&self) -> bool {
-        ProcMacroServerProcess::use_postcard(self)
-    }
-
-    fn state(&self) -> &Mutex {
-        &self.state
-    }
-
-    fn get_exited(&self) -> &OnceLock> {
-        &self.exited
-    }
-
-    fn is_reusable(&self) -> bool {
-        !self.single_use
-    }
-}
-
 impl ProcMacroServerProcess {
     /// Starts the proc-macro server and performs a version check
     pub(crate) fn spawn<'a>(
@@ -220,7 +149,11 @@ impl ProcMacroServerProcess {
                 let (process, stdin, stdout) = spawn(format)?;
 
                 io::Result::Ok(ProcMacroServerProcess {
-                    state: Mutex::new(ProcessSrvState { process, stdin, stdout }),
+                    state: Mutex::new(ProcessSrvState {
+                        process,
+                        stdin,
+                        stdout,
+                    }),
                     version: 0,
                     protocol: match format {
                         Some(ProtocolFormat::BidirectionalPostcardPrototype) => {
@@ -271,6 +204,37 @@ impl ProcMacroServerProcess {
         Err(err.unwrap())
     }
 
+    pub(crate) fn load_dylib(
+        &self,
+        dylib_path: &AbsPath,
+        callback: Option>,
+    ) -> Result, ServerError> {
+        let _state = self.state.lock().unwrap();
+
+        // if state.loaded_dylibs.contains(dylib_path) {
+        //     // Already loaded in this worker
+        //     return Ok(Vec::new());
+        // }
+
+        let result = match self.protocol {
+            Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
+                legacy_protocol::find_proc_macros(self, dylib_path)?
+            }
+            Protocol::BidirectionalPostcardPrototype { .. } => {
+                let cb = callback.expect("callback required");
+                bidirectional_protocol::find_proc_macros(self, dylib_path, cb)?
+            }
+        };
+
+        match result {
+            Ok(macros) => {
+                // state.loaded_dylibs.insert(dylib_path.to_owned());
+                Ok(macros)
+            }
+            Err(message) => Err(ServerError { message, io: None }),
+        }
+    }
+
     /// Returns the server error if the process has exited.
     pub(crate) fn exited(&self) -> Option<&ServerError> {
         self.exited.get().map(|it| &it.0)
@@ -314,21 +278,6 @@ impl ProcMacroServerProcess {
         }
     }
 
-    /// Finds proc-macros in a given dynamic library.
-    pub(crate) fn find_proc_macros(
-        &self,
-        dylib_path: &AbsPath,
-        callback: Option>,
-    ) -> Result, String>, ServerError> {
-        match self.protocol {
-            Protocol::LegacyJson { .. } => legacy_protocol::find_proc_macros(self, dylib_path),
-            Protocol::BidirectionalPostcardPrototype { .. } => {
-                let cb = callback.expect("callback required for bidirectional protocol");
-                bidirectional_protocol::find_proc_macros(self, dylib_path, cb)
-            }
-        }
-    }
-
     pub(crate) fn expand(
         &self,
         proc_macro: &ProcMacro,
@@ -365,25 +314,25 @@ impl ProcMacroServerProcess {
             ),
         };
 
-        if self.is_reusable() {
+        if !self.is_reusable() {
             self.terminate();
         }
 
         result
     }
 
+    fn is_reusable(&self) -> bool {
+        self.single_use
+    }
+
     fn terminate(&self) {
         if let Ok(mut state) = self.state.lock() {
             let _ = state.process.child.kill();
         }
     }
-}
 
-pub(crate) struct SynIO;
-
-impl SynIO {
     pub(crate) fn send_task(
-        proc_macro_worker: &dyn ProcMacroWorker,
+        &self,
         send: impl FnOnce(
             &mut dyn Write,
             &mut dyn BufRead,
@@ -392,7 +341,7 @@ impl SynIO {
         ) -> Result, ServerError>,
         req: Request,
     ) -> Result {
-        SynIO::with_locked_io::(proc_macro_worker, |writer, reader, buf| {
+        self.with_locked_io::(|writer, reader, buf| {
             send(writer, reader, req, buf).and_then(|res| {
                 res.ok_or_else(|| {
                     let message = "proc-macro server did not respond with data".to_owned();
@@ -409,10 +358,10 @@ impl SynIO {
     }
 
     pub(crate) fn with_locked_io(
-        proc_macro_worker: &dyn ProcMacroWorker,
+        &self,
         f: impl FnOnce(&mut dyn Write, &mut dyn BufRead, &mut C::Buf) -> Result,
     ) -> Result {
-        let state = &mut *proc_macro_worker.state().lock().unwrap();
+        let state = &mut *self.state.lock().unwrap();
         let mut buf = C::Buf::default();
 
         f(&mut state.stdin, &mut state.stdout, &mut buf).map_err(|e| {
@@ -434,11 +383,11 @@ impl SynIO {
     }
 
     pub(crate) fn run_bidirectional(
-        proc_macro_worker: &dyn ProcMacroWorker,
+        &self,
         initial: BidirectionalMessage,
         callback: SubCallback<'_>,
     ) -> Result {
-        SynIO::with_locked_io::(proc_macro_worker, |writer, reader, buf| {
+        self.with_locked_io::(|writer, reader, buf| {
             bidirectional_protocol::run_conversation::(writer, reader, buf, initial, callback)
         })
     }

From c685aa912349b96f21485c434d03ee288cfceb9e Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Fri, 2 Jan 2026 11:03:20 +0530
Subject: [PATCH 0293/1843] direct client calls via pool

---
 .../src/bidirectional_protocol.rs             | 11 +--
 .../proc-macro-api/src/legacy_protocol.rs     | 11 +--
 .../crates/proc-macro-api/src/lib.rs          | 21 +-----
 .../crates/proc-macro-api/src/pool.rs         | 72 ++++++++++++++++---
 .../crates/proc-macro-api/src/process.rs      | 65 ++++++-----------
 5 files changed, 98 insertions(+), 82 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index 137f2dafc0de..cd1f6f6f1f33 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -138,6 +138,7 @@ pub(crate) fn find_proc_macros(
 
 pub(crate) fn expand(
     proc_macro: &ProcMacro,
+    process: &ProcMacroServerProcess,
     subtree: tt::SubtreeView<'_>,
     attr: Option>,
     env: Vec<(String, String)>,
@@ -147,7 +148,7 @@ pub(crate) fn expand(
     current_dir: String,
     callback: SubCallback<'_>,
 ) -> Result, crate::ServerError> {
-    let version = proc_macro.process.version();
+    let version = process.version();
     let mut span_data_table = SpanDataIndexMap::default();
     let def_site = span_data_table.insert_full(def_site).0;
     let call_site = span_data_table.insert_full(call_site).0;
@@ -164,7 +165,7 @@ pub(crate) fn expand(
                 call_site,
                 mixed_site,
             },
-            span_data_table: if proc_macro.process.rust_analyzer_spans() {
+            span_data_table: if process.rust_analyzer_spans() {
                 serialize_span_data_index_map(&span_data_table)
             } else {
                 Vec::new()
@@ -175,13 +176,13 @@ pub(crate) fn expand(
         current_dir: Some(current_dir),
     })));
 
-    let response_payload = run_request(proc_macro.process.as_ref(), task, callback)?;
+    let response_payload = run_request(process, task, callback)?;
 
     match response_payload {
         BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it
             .map(|tree| {
                 let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
-                if proc_macro.needs_fixup_change() {
+                if proc_macro.needs_fixup_change(process) {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
@@ -194,7 +195,7 @@ pub(crate) fn expand(
                     version,
                     &deserialize_span_data_index_map(&resp.span_data_table),
                 );
-                if proc_macro.needs_fixup_change() {
+                if proc_macro.needs_fixup_change(process) {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
index 7b546cf7aef6..412d20730324 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
@@ -77,6 +77,7 @@ pub(crate) fn find_proc_macros(
 
 pub(crate) fn expand(
     proc_macro: &ProcMacro,
+    process: &ProcMacroServerProcess,
     subtree: tt::SubtreeView<'_>,
     attr: Option>,
     env: Vec<(String, String)>,
@@ -85,7 +86,7 @@ pub(crate) fn expand(
     mixed_site: Span,
     current_dir: String,
 ) -> Result, crate::ServerError> {
-    let version = proc_macro.process.version();
+    let version = process.version();
     let mut span_data_table = SpanDataIndexMap::default();
     let def_site = span_data_table.insert_full(def_site).0;
     let call_site = span_data_table.insert_full(call_site).0;
@@ -102,7 +103,7 @@ pub(crate) fn expand(
                 call_site,
                 mixed_site,
             },
-            span_data_table: if proc_macro.process.rust_analyzer_spans() {
+            span_data_table: if process.rust_analyzer_spans() {
                 serialize_span_data_index_map(&span_data_table)
             } else {
                 Vec::new()
@@ -113,13 +114,13 @@ pub(crate) fn expand(
         current_dir: Some(current_dir),
     };
 
-    let response = send_task(proc_macro.process.as_ref(), Request::ExpandMacro(Box::new(task)))?;
+    let response = send_task(process, Request::ExpandMacro(Box::new(task)))?;
 
     match response {
         Response::ExpandMacro(it) => Ok(it
             .map(|tree| {
                 let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
-                if proc_macro.needs_fixup_change() {
+                if proc_macro.needs_fixup_change(process) {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
@@ -132,7 +133,7 @@ pub(crate) fn expand(
                     version,
                     &deserialize_span_data_index_map(&resp.span_data_table),
                 );
-                if proc_macro.needs_fixup_change() {
+                if proc_macro.needs_fixup_change(process) {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index ffae28f92c05..fe17e14024cc 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -112,7 +112,7 @@ impl MacroDylib {
 /// we share a single expander process for all macros within a workspace.
 #[derive(Debug, Clone)]
 pub struct ProcMacro {
-    process: Arc,
+    process: ProcMacroServerPool,
     dylib_path: Arc,
     name: Box,
     kind: ProcMacroKind,
@@ -126,7 +126,6 @@ impl PartialEq for ProcMacro {
             && self.kind == other.kind
             && self.dylib_path == other.dylib_path
             && self.dylib_last_modified == other.dylib_last_modified
-            && Arc::ptr_eq(&self.process, &other.process)
     }
 }
 
@@ -210,8 +209,8 @@ impl ProcMacro {
         self.kind
     }
 
-    fn needs_fixup_change(&self) -> bool {
-        let version = self.process.version();
+    fn needs_fixup_change(&self, process: &ProcMacroServerProcess) -> bool {
+        let version = process.version();
         (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version)
     }
 
@@ -241,20 +240,6 @@ impl ProcMacro {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
-        let (mut subtree, mut attr) = (subtree, attr);
-        let (mut subtree_changed, mut attr_changed);
-        if self.needs_fixup_change() {
-            subtree_changed = tt::TopSubtree::from_subtree(subtree);
-            self.change_fixup_to_match_old_server(&mut subtree_changed);
-            subtree = subtree_changed.view();
-
-            if let Some(attr) = &mut attr {
-                attr_changed = tt::TopSubtree::from_subtree(*attr);
-                self.change_fixup_to_match_old_server(&mut attr_changed);
-                *attr = attr_changed.view();
-            }
-        }
-
         self.process.expand(
             self,
             subtree,
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index 685bc05be62a..4639374f3e24 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -1,29 +1,38 @@
 use std::sync::Arc;
 
+use tt::Span;
+
 use crate::{
     MacroDylib, ProcMacro, ServerError, bidirectional_protocol::SubCallback,
     process::ProcMacroServerProcess,
 };
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub(crate) struct ProcMacroServerPool {
-    workers: Vec>,
+    workers: Arc<[ProcMacroServerProcess]>,
 }
 
 impl ProcMacroServerPool {
-    pub(crate) fn new(workers: Vec>) -> Self {
-        Self { workers }
+    pub(crate) fn new(workers: Vec) -> Self {
+        Self { workers: workers.into() }
     }
 }
 
 impl ProcMacroServerPool {
     pub(crate) fn exited(&self) -> Option<&ServerError> {
-        for worker in &self.workers {
-            if let Some(e) = worker.exited() {
-                return Some(e);
+        for worker in &*self.workers {
+            worker.exited()?;
+        }
+        self.workers[0].exited()
+    }
+
+    fn pick_process(&self) -> &ProcMacroServerProcess {
+        for workers in &*self.workers {
+            if workers.can_use() {
+                return workers;
             }
         }
-        None
+        &self.workers[0]
     }
 
     pub(crate) fn load_dylib(
@@ -34,16 +43,16 @@ impl ProcMacroServerPool {
         let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered();
         let mut all_macros = Vec::new();
 
-        for worker in &self.workers {
+        for worker in &*self.workers {
             let dylib_path = Arc::new(dylib.path.clone());
             let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
                 .ok()
                 .and_then(|metadata| metadata.modified().ok());
-            let macros = worker.load_dylib(&dylib.path, None)?;
+            let macros = worker.find_proc_macros(&dylib.path, None)?.unwrap();
 
             for (name, kind) in macros {
                 all_macros.push(ProcMacro {
-                    process: worker.clone(),
+                    process: self.clone(),
                     name: name.into(),
                     kind,
                     dylib_path: Arc::new(dylib.path.clone()),
@@ -54,6 +63,47 @@ impl ProcMacroServerPool {
 
         Ok(all_macros)
     }
+
+    pub(crate) fn expand(
+        &self,
+        proc_macro: &ProcMacro,
+        subtree: tt::SubtreeView<'_>,
+        attr: Option>,
+        env: Vec<(String, String)>,
+        def_site: Span,
+        call_site: Span,
+        mixed_site: Span,
+        current_dir: String,
+        callback: Option>,
+    ) -> Result, ServerError> {
+        let process = self.pick_process();
+
+        let (mut subtree, mut attr) = (subtree, attr);
+        let (mut subtree_changed, mut attr_changed);
+        if proc_macro.needs_fixup_change(process) {
+            subtree_changed = tt::TopSubtree::from_subtree(subtree);
+            proc_macro.change_fixup_to_match_old_server(&mut subtree_changed);
+            subtree = subtree_changed.view();
+
+            if let Some(attr) = &mut attr {
+                attr_changed = tt::TopSubtree::from_subtree(*attr);
+                proc_macro.change_fixup_to_match_old_server(&mut attr_changed);
+                *attr = attr_changed.view();
+            }
+        }
+
+        process.expand(
+            proc_macro,
+            subtree,
+            attr,
+            env,
+            def_site,
+            call_site,
+            mixed_site,
+            current_dir,
+            callback,
+        )
+    }
 }
 
 pub(crate) fn default_pool_size() -> usize {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index 30877c5cf491..a41bb58e74be 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -5,7 +5,10 @@ use std::{
     io::{self, BufRead, BufReader, Read, Write},
     panic::AssertUnwindSafe,
     process::{Child, ChildStdin, ChildStdout, Command, Stdio},
-    sync::{Arc, Mutex, OnceLock},
+    sync::{
+        Arc, Mutex, OnceLock,
+        atomic::{AtomicBool, Ordering},
+    },
 };
 
 use paths::AbsPath;
@@ -29,7 +32,7 @@ pub(crate) struct ProcMacroServerProcess {
     protocol: Protocol,
     /// Populated when the server exits.
     exited: OnceLock>,
-    single_use: bool,
+    can_use: AtomicBool,
 }
 
 impl std::fmt::Debug for ProcMacroServerProcess {
@@ -149,11 +152,7 @@ impl ProcMacroServerProcess {
                 let (process, stdin, stdout) = spawn(format)?;
 
                 io::Result::Ok(ProcMacroServerProcess {
-                    state: Mutex::new(ProcessSrvState {
-                        process,
-                        stdin,
-                        stdout,
-                    }),
+                    state: Mutex::new(ProcessSrvState { process, stdin, stdout }),
                     version: 0,
                     protocol: match format {
                         Some(ProtocolFormat::BidirectionalPostcardPrototype) => {
@@ -164,7 +163,7 @@ impl ProcMacroServerProcess {
                         }
                     },
                     exited: OnceLock::new(),
-                    single_use,
+                    can_use: AtomicBool::new(true),
                 })
             };
             let mut srv = create_srv()?;
@@ -204,34 +203,20 @@ impl ProcMacroServerProcess {
         Err(err.unwrap())
     }
 
-    pub(crate) fn load_dylib(
+    /// Finds proc-macros in a given dynamic library.
+    pub(crate) fn find_proc_macros(
         &self,
         dylib_path: &AbsPath,
         callback: Option>,
-    ) -> Result, ServerError> {
-        let _state = self.state.lock().unwrap();
-
-        // if state.loaded_dylibs.contains(dylib_path) {
-        //     // Already loaded in this worker
-        //     return Ok(Vec::new());
-        // }
-
-        let result = match self.protocol {
+    ) -> Result, String>, ServerError> {
+        match self.protocol {
             Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
-                legacy_protocol::find_proc_macros(self, dylib_path)?
+                legacy_protocol::find_proc_macros(self, dylib_path)
             }
             Protocol::BidirectionalPostcardPrototype { .. } => {
-                let cb = callback.expect("callback required");
-                bidirectional_protocol::find_proc_macros(self, dylib_path, cb)?
+                let cb = callback.expect("callback required for bidirectional protocol");
+                bidirectional_protocol::find_proc_macros(self, dylib_path, cb)
             }
-        };
-
-        match result {
-            Ok(macros) => {
-                // state.loaded_dylibs.insert(dylib_path.to_owned());
-                Ok(macros)
-            }
-            Err(message) => Err(ServerError { message, io: None }),
         }
     }
 
@@ -290,9 +275,11 @@ impl ProcMacroServerProcess {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
+        self.can_use.store(false, Ordering::Release);
         let result = match self.protocol {
             Protocol::LegacyJson { .. } => legacy_protocol::expand(
                 proc_macro,
+                    self,
                 subtree,
                 attr,
                 env,
@@ -303,6 +290,7 @@ impl ProcMacroServerProcess {
             ),
             Protocol::BidirectionalPostcardPrototype { .. } => bidirectional_protocol::expand(
                 proc_macro,
+                self,
                 subtree,
                 attr,
                 env,
@@ -314,23 +302,10 @@ impl ProcMacroServerProcess {
             ),
         };
 
-        if !self.is_reusable() {
-            self.terminate();
-        }
-
+        self.can_use.store(true, Ordering::Release);
         result
     }
 
-    fn is_reusable(&self) -> bool {
-        self.single_use
-    }
-
-    fn terminate(&self) {
-        if let Ok(mut state) = self.state.lock() {
-            let _ = state.process.child.kill();
-        }
-    }
-
     pub(crate) fn send_task(
         &self,
         send: impl FnOnce(
@@ -391,6 +366,10 @@ impl ProcMacroServerProcess {
             bidirectional_protocol::run_conversation::(writer, reader, buf, initial, callback)
         })
     }
+
+    pub(crate) fn can_use(&self) -> bool {
+        self.can_use.load(Ordering::Acquire)
+    }
 }
 
 /// Manages the execution of the proc-macro server process.

From 82e758acc1a76b2a8d8083121b031fc2b3e3a653 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Fri, 2 Jan 2026 11:46:00 +0530
Subject: [PATCH 0294/1843] add better process picker and improve loading dylib

---
 .../crates/proc-macro-api/src/pool.rs         | 48 ++++++++++---------
 .../crates/proc-macro-api/src/process.rs      | 14 +++---
 2 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index 4639374f3e24..4215b0f2c04f 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -27,12 +27,10 @@ impl ProcMacroServerPool {
     }
 
     fn pick_process(&self) -> &ProcMacroServerProcess {
-        for workers in &*self.workers {
-            if workers.can_use() {
-                return workers;
-            }
-        }
-        &self.workers[0]
+        self.workers
+            .iter()
+            .min_by_key(|w| w.number_of_active_req())
+            .expect("worker pool must not be empty")
     }
 
     pub(crate) fn load_dylib(
@@ -41,27 +39,31 @@ impl ProcMacroServerPool {
         _callback: Option>,
     ) -> Result, ServerError> {
         let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered();
-        let mut all_macros = Vec::new();
 
-        for worker in &*self.workers {
-            let dylib_path = Arc::new(dylib.path.clone());
-            let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
-                .ok()
-                .and_then(|metadata| metadata.modified().ok());
-            let macros = worker.find_proc_macros(&dylib.path, None)?.unwrap();
+        let dylib_path = Arc::new(dylib.path.clone());
+        let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
+            .ok()
+            .and_then(|metadata| metadata.modified().ok());
 
-            for (name, kind) in macros {
-                all_macros.push(ProcMacro {
-                    process: self.clone(),
-                    name: name.into(),
-                    kind,
-                    dylib_path: Arc::new(dylib.path.clone()),
-                    dylib_last_modified,
-                });
-            }
+        let first = &self.workers[0];
+        let macros = first.find_proc_macros(&dylib.path, None)?.unwrap();
+
+        for worker in &self.workers[1..] {
+            let _ = worker.find_proc_macros(&dylib.path, None)?;
         }
 
-        Ok(all_macros)
+        let result = macros
+            .into_iter()
+            .map(|(name, kind)| ProcMacro {
+                process: self.clone(),
+                name: name.into(),
+                kind,
+                dylib_path: dylib_path.clone(),
+                dylib_last_modified,
+            })
+            .collect();
+
+        Ok(result)
     }
 
     pub(crate) fn expand(
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index a41bb58e74be..775d59174f1c 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -7,7 +7,7 @@ use std::{
     process::{Child, ChildStdin, ChildStdout, Command, Stdio},
     sync::{
         Arc, Mutex, OnceLock,
-        atomic::{AtomicBool, Ordering},
+        atomic::{AtomicU32, Ordering},
     },
 };
 
@@ -32,7 +32,7 @@ pub(crate) struct ProcMacroServerProcess {
     protocol: Protocol,
     /// Populated when the server exits.
     exited: OnceLock>,
-    can_use: AtomicBool,
+    active: AtomicU32,
 }
 
 impl std::fmt::Debug for ProcMacroServerProcess {
@@ -163,7 +163,7 @@ impl ProcMacroServerProcess {
                         }
                     },
                     exited: OnceLock::new(),
-                    can_use: AtomicBool::new(true),
+                    active: AtomicU32::new(0),
                 })
             };
             let mut srv = create_srv()?;
@@ -275,7 +275,7 @@ impl ProcMacroServerProcess {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
-        self.can_use.store(false, Ordering::Release);
+        self.active.fetch_add(1, Ordering::AcqRel);
         let result = match self.protocol {
             Protocol::LegacyJson { .. } => legacy_protocol::expand(
                 proc_macro,
@@ -302,7 +302,7 @@ impl ProcMacroServerProcess {
             ),
         };
 
-        self.can_use.store(true, Ordering::Release);
+        self.active.fetch_sub(1, Ordering::AcqRel);
         result
     }
 
@@ -367,8 +367,8 @@ impl ProcMacroServerProcess {
         })
     }
 
-    pub(crate) fn can_use(&self) -> bool {
-        self.can_use.load(Ordering::Acquire)
+    pub(crate) fn number_of_active_req(&self) -> u32 {
+        self.active.load(Ordering::Acquire)
     }
 }
 

From 922bc7e4d49da5656c901e0a2d4a9805f413dcf8 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Fri, 2 Jan 2026 11:56:19 +0530
Subject: [PATCH 0295/1843] rename process to pool in ProcMacro struct

---
 src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs  | 4 ++--
 src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index fe17e14024cc..09999ea5081a 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -112,7 +112,7 @@ impl MacroDylib {
 /// we share a single expander process for all macros within a workspace.
 #[derive(Debug, Clone)]
 pub struct ProcMacro {
-    process: ProcMacroServerPool,
+    pool: ProcMacroServerPool,
     dylib_path: Arc,
     name: Box,
     kind: ProcMacroKind,
@@ -240,7 +240,7 @@ impl ProcMacro {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
-        self.process.expand(
+        self.pool.expand(
             self,
             subtree,
             attr,
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index 4215b0f2c04f..eef8d0194dc5 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -55,7 +55,7 @@ impl ProcMacroServerPool {
         let result = macros
             .into_iter()
             .map(|(name, kind)| ProcMacro {
-                process: self.clone(),
+                pool: self.clone(),
                 name: name.into(),
                 kind,
                 dylib_path: dylib_path.clone(),

From c4c336ad7c291686542826bfd5b0bc8f2e298777 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Fri, 2 Jan 2026 12:11:11 +0530
Subject: [PATCH 0296/1843] keep it clean and tidy

---
 src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index eef8d0194dc5..fe4649441b89 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -1,3 +1,4 @@
+//! This module represents Process Pool
 use std::sync::Arc;
 
 use tt::Span;

From c8a3551bd162c6374d4c49bfc06070488edb44ed Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sat, 3 Jan 2026 11:32:08 +0530
Subject: [PATCH 0297/1843] change callback from FnMut to Fn as we only
 transform messages and not really change change state

---
 src/tools/rust-analyzer/crates/load-cargo/src/lib.rs        | 6 +++---
 .../crates/proc-macro-api/src/bidirectional_protocol.rs     | 2 +-
 .../rust-analyzer/crates/proc-macro-api/src/process.rs      | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
index 8342492a33a4..ccc9aa4291ec 100644
--- a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
@@ -435,7 +435,7 @@ pub fn load_proc_macro(
 ) -> ProcMacroLoadResult {
     let res: Result, _> = (|| {
         let dylib = MacroDylib::new(path.to_path_buf());
-        let vec = server.load_dylib(dylib, Some(&mut reject_subrequests)).map_err(|e| {
+        let vec = server.load_dylib(dylib, Some(&reject_subrequests)).map_err(|e| {
             ProcMacroLoadingError::ProcMacroSrvError(format!("{e}").into_boxed_str())
         })?;
         if vec.is_empty() {
@@ -541,7 +541,7 @@ impl ProcMacroExpander for Expander {
         mixed_site: Span,
         current_dir: String,
     ) -> Result {
-        let mut cb = |req| match req {
+        let cb = |req| match req {
             SubRequest::LocalFilePath { file_id } => {
                 let file_id = FileId::from_raw(file_id);
                 let source_root_id = db.file_source_root(file_id).source_root_id(db);
@@ -613,7 +613,7 @@ impl ProcMacroExpander for Expander {
             call_site,
             mixed_site,
             current_dir,
-            Some(&mut cb),
+            Some(&cb),
         ) {
             Ok(Ok(subtree)) => Ok(subtree),
             Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err)),
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index cd1f6f6f1f33..25266c46fe89 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -28,7 +28,7 @@ use crate::{
 
 pub mod msg;
 
-pub type SubCallback<'a> = &'a mut dyn FnMut(SubRequest) -> Result;
+pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result;
 
 pub fn run_conversation(
     writer: &mut dyn Write,
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index 775d59174f1c..c1b95fa7f10e 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -168,7 +168,7 @@ impl ProcMacroServerProcess {
             };
             let mut srv = create_srv()?;
             tracing::info!("sending proc-macro server version check");
-            match srv.version_check(Some(&mut reject_subrequests)) {
+            match srv.version_check(Some(&reject_subrequests)) {
                 Ok(v) if v > version::CURRENT_API_VERSION => {
                     let process_version = binary_server_version();
                     err = Some(io::Error::other(format!(
@@ -182,7 +182,7 @@ impl ProcMacroServerProcess {
                     srv.version = v;
                     if srv.version >= version::RUST_ANALYZER_SPAN_SUPPORT
                         && let Ok(new_mode) =
-                            srv.enable_rust_analyzer_spans(Some(&mut reject_subrequests))
+                            srv.enable_rust_analyzer_spans(Some(&reject_subrequests))
                     {
                         match &mut srv.protocol {
                             Protocol::LegacyJson { mode }

From 66bca6a25214332ff559acb7678d3ce423279e77 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sat, 3 Jan 2026 11:36:09 +0530
Subject: [PATCH 0298/1843] propagate error from load dylibs

---
 .../crates/proc-macro-api/src/pool.rs         | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index fe4649441b89..20389f666883 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -37,23 +37,27 @@ impl ProcMacroServerPool {
     pub(crate) fn load_dylib(
         &self,
         dylib: &MacroDylib,
-        _callback: Option>,
+        callback: Option>,
     ) -> Result, ServerError> {
-        let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered();
+        let _span = tracing::info_span!("ProcMacroServer::load_dylib").entered();
 
         let dylib_path = Arc::new(dylib.path.clone());
-        let dylib_last_modified = std::fs::metadata(dylib_path.as_path())
-            .ok()
-            .and_then(|metadata| metadata.modified().ok());
+        let dylib_last_modified =
+            std::fs::metadata(dylib_path.as_path()).ok().and_then(|m| m.modified().ok());
 
-        let first = &self.workers[0];
-        let macros = first.find_proc_macros(&dylib.path, None)?.unwrap();
+        let (first, rest) = self.workers.split_first().expect("worker pool must not be empty");
 
-        for worker in &self.workers[1..] {
-            let _ = worker.find_proc_macros(&dylib.path, None)?;
+        let macros = first
+            .find_proc_macros(&dylib.path, callback)?
+            .map_err(|e| ServerError { message: e, io: None })?;
+
+        for worker in rest {
+            worker
+                .find_proc_macros(&dylib.path, callback)?
+                .map_err(|e| ServerError { message: e, io: None })?;
         }
 
-        let result = macros
+        Ok(macros
             .into_iter()
             .map(|(name, kind)| ProcMacro {
                 pool: self.clone(),
@@ -62,9 +66,7 @@ impl ProcMacroServerPool {
                 dylib_path: dylib_path.clone(),
                 dylib_last_modified,
             })
-            .collect();
-
-        Ok(result)
+            .collect())
     }
 
     pub(crate) fn expand(

From 09c91b79a84faebb2e06094f8c8d1592a371a3e4 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sat, 3 Jan 2026 13:43:23 +0530
Subject: [PATCH 0299/1843] pick workers which have not exited

---
 .../rust-analyzer/crates/proc-macro-api/src/pool.rs    | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index 20389f666883..c75e9742a5d1 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -27,11 +27,15 @@ impl ProcMacroServerPool {
         self.workers[0].exited()
     }
 
-    fn pick_process(&self) -> &ProcMacroServerProcess {
+    fn pick_process(&self) -> Result<&ProcMacroServerProcess, ServerError> {
         self.workers
             .iter()
+            .filter(|w| w.exited().is_none())
             .min_by_key(|w| w.number_of_active_req())
-            .expect("worker pool must not be empty")
+            .ok_or_else(|| ServerError {
+                message: "all proc-macro server workers have exited".into(),
+                io: None,
+            })
     }
 
     pub(crate) fn load_dylib(
@@ -81,7 +85,7 @@ impl ProcMacroServerPool {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
-        let process = self.pick_process();
+        let process = self.pick_process()?;
 
         let (mut subtree, mut attr) = (subtree, attr);
         let (mut subtree_changed, mut attr_changed);

From 0936597b3ea2944869d1a0b2058746e295eeb16d Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 4 Jan 2026 05:02:15 +0530
Subject: [PATCH 0300/1843] add version to pool

---
 .../src/bidirectional_protocol.rs             |  4 ++--
 .../proc-macro-api/src/legacy_protocol.rs     |  4 ++--
 .../crates/proc-macro-api/src/lib.rs          | 18 ++++++++++++--
 .../crates/proc-macro-api/src/pool.rs         | 24 +++++++------------
 4 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index 25266c46fe89..b5f43e1d3726 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -182,7 +182,7 @@ pub(crate) fn expand(
         BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it
             .map(|tree| {
                 let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
-                if proc_macro.needs_fixup_change(process) {
+                if proc_macro.needs_fixup_change() {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
@@ -195,7 +195,7 @@ pub(crate) fn expand(
                     version,
                     &deserialize_span_data_index_map(&resp.span_data_table),
                 );
-                if proc_macro.needs_fixup_change(process) {
+                if proc_macro.needs_fixup_change() {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
index 412d20730324..eedf66d46086 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
@@ -120,7 +120,7 @@ pub(crate) fn expand(
         Response::ExpandMacro(it) => Ok(it
             .map(|tree| {
                 let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
-                if proc_macro.needs_fixup_change(process) {
+                if proc_macro.needs_fixup_change() {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
@@ -133,7 +133,7 @@ pub(crate) fn expand(
                     version,
                     &deserialize_span_data_index_map(&resp.span_data_table),
                 );
-                if proc_macro.needs_fixup_change(process) {
+                if proc_macro.needs_fixup_change() {
                     proc_macro.change_fixup_to_match_old_server(&mut expanded);
                 }
                 expanded
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index 09999ea5081a..4874e63244c4 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -209,8 +209,8 @@ impl ProcMacro {
         self.kind
     }
 
-    fn needs_fixup_change(&self, process: &ProcMacroServerProcess) -> bool {
-        let version = process.version();
+    fn needs_fixup_change(&self) -> bool {
+        let version = self.pool.version();
         (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version)
     }
 
@@ -240,6 +240,20 @@ impl ProcMacro {
         current_dir: String,
         callback: Option>,
     ) -> Result, ServerError> {
+        let (mut subtree, mut attr) = (subtree, attr);
+        let (mut subtree_changed, mut attr_changed);
+        if self.needs_fixup_change() {
+            subtree_changed = tt::TopSubtree::from_subtree(subtree);
+            self.change_fixup_to_match_old_server(&mut subtree_changed);
+            subtree = subtree_changed.view();
+
+            if let Some(attr) = &mut attr {
+                attr_changed = tt::TopSubtree::from_subtree(*attr);
+                self.change_fixup_to_match_old_server(&mut attr_changed);
+                *attr = attr_changed.view();
+            }
+        }
+
         self.pool.expand(
             self,
             subtree,
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index c75e9742a5d1..fd8b726f820e 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -1,4 +1,4 @@
-//! This module represents Process Pool
+//! A pool of proc-macro server processes
 use std::sync::Arc;
 
 use tt::Span;
@@ -11,11 +11,13 @@ use crate::{
 #[derive(Debug, Clone)]
 pub(crate) struct ProcMacroServerPool {
     workers: Arc<[ProcMacroServerProcess]>,
+    version: u32,
 }
 
 impl ProcMacroServerPool {
     pub(crate) fn new(workers: Vec) -> Self {
-        Self { workers: workers.into() }
+        let version = workers[0].version();
+        Self { workers: workers.into(), version }
     }
 }
 
@@ -87,20 +89,6 @@ impl ProcMacroServerPool {
     ) -> Result, ServerError> {
         let process = self.pick_process()?;
 
-        let (mut subtree, mut attr) = (subtree, attr);
-        let (mut subtree_changed, mut attr_changed);
-        if proc_macro.needs_fixup_change(process) {
-            subtree_changed = tt::TopSubtree::from_subtree(subtree);
-            proc_macro.change_fixup_to_match_old_server(&mut subtree_changed);
-            subtree = subtree_changed.view();
-
-            if let Some(attr) = &mut attr {
-                attr_changed = tt::TopSubtree::from_subtree(*attr);
-                proc_macro.change_fixup_to_match_old_server(&mut attr_changed);
-                *attr = attr_changed.view();
-            }
-        }
-
         process.expand(
             proc_macro,
             subtree,
@@ -113,6 +101,10 @@ impl ProcMacroServerPool {
             callback,
         )
     }
+
+    pub(crate) fn version(&self) -> u32 {
+        self.version
+    }
 }
 
 pub(crate) fn default_pool_size() -> usize {

From 263015a4a4d4cbbbc1e7cd0e118c10e0a8e740df Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 4 Jan 2026 05:50:16 +0530
Subject: [PATCH 0301/1843] remove expand from pool

---
 .../crates/proc-macro-api/src/lib.rs          |  2 +-
 .../crates/proc-macro-api/src/pool.rs         | 31 +------------------
 2 files changed, 2 insertions(+), 31 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index 4874e63244c4..ad462ff31a1b 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -254,7 +254,7 @@ impl ProcMacro {
             }
         }
 
-        self.pool.expand(
+        self.pool.pick_process()?.expand(
             self,
             subtree,
             attr,
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index fd8b726f820e..13a4b5ee8f9e 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -1,8 +1,6 @@
 //! A pool of proc-macro server processes
 use std::sync::Arc;
 
-use tt::Span;
-
 use crate::{
     MacroDylib, ProcMacro, ServerError, bidirectional_protocol::SubCallback,
     process::ProcMacroServerProcess,
@@ -29,7 +27,7 @@ impl ProcMacroServerPool {
         self.workers[0].exited()
     }
 
-    fn pick_process(&self) -> Result<&ProcMacroServerProcess, ServerError> {
+    pub(crate) fn pick_process(&self) -> Result<&ProcMacroServerProcess, ServerError> {
         self.workers
             .iter()
             .filter(|w| w.exited().is_none())
@@ -75,33 +73,6 @@ impl ProcMacroServerPool {
             .collect())
     }
 
-    pub(crate) fn expand(
-        &self,
-        proc_macro: &ProcMacro,
-        subtree: tt::SubtreeView<'_>,
-        attr: Option>,
-        env: Vec<(String, String)>,
-        def_site: Span,
-        call_site: Span,
-        mixed_site: Span,
-        current_dir: String,
-        callback: Option>,
-    ) -> Result, ServerError> {
-        let process = self.pick_process()?;
-
-        process.expand(
-            proc_macro,
-            subtree,
-            attr,
-            env,
-            def_site,
-            call_site,
-            mixed_site,
-            current_dir,
-            callback,
-        )
-    }
-
     pub(crate) fn version(&self) -> u32 {
         self.version
     }

From e3e7c2905442499a6c3eb778be14730dc16d82af Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 09:41:39 +0530
Subject: [PATCH 0302/1843] remove default pool size from pool

---
 src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index 13a4b5ee8f9e..0cb505aa40a9 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -77,7 +77,3 @@ impl ProcMacroServerPool {
         self.version
     }
 }
-
-pub(crate) fn default_pool_size() -> usize {
-    std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1).min(4)
-}

From 96ecd1773c56e11def32667a3d70f4b29563f137 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 09:44:15 +0530
Subject: [PATCH 0303/1843] add num process in config

---
 .../crates/rust-analyzer/src/config.rs          | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index 28ac94e4deb6..cb6552c32ffb 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -380,6 +380,8 @@ config_data! {
         /// The number of worker threads in the main loop. The default `null` means to pick
         /// automatically.
         numThreads: Option = None,
+        /// The number of proc-macro-srv processes 
+        proc_macro_processes: NumProcesses = NumProcesses::Concrete(1),
 
         /// Expand attribute macros. Requires `#rust-analyzer.procMacro.enable#` to be set.
         procMacro_attributes_enable: bool = true,
@@ -2641,6 +2643,13 @@ impl Config {
         }
     }
 
+    pub fn proc_macro_num_processes(&self) -> usize {
+        match self.proc_macro_processes() {
+            NumProcesses::Concrete(0) | NumProcesses::Physical => num_cpus::get_physical(),
+            &NumProcesses::Concrete(n) => n,
+        }
+    }
+
     pub fn main_loop_num_threads(&self) -> usize {
         match self.numThreads() {
             Some(NumThreads::Concrete(0)) | None | Some(NumThreads::Physical) => {
@@ -3077,6 +3086,14 @@ pub enum NumThreads {
     Concrete(usize),
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum NumProcesses {
+    Physical,
+    #[serde(untagged)]
+    Concrete(usize),
+}
+
 macro_rules! _default_val {
     ($default:expr, $ty:ty) => {{
         let default_: $ty = $default;

From 9d5e60005addf5eb8635884f051771a8119f0bb2 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 09:44:44 +0530
Subject: [PATCH 0304/1843] add proc_macro_processes in load config

---
 .../crates/load-cargo/src/lib.rs              | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
index ccc9aa4291ec..c2935d94a8a7 100644
--- a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
@@ -45,6 +45,7 @@ pub struct LoadCargoConfig {
     pub load_out_dirs_from_check: bool,
     pub with_proc_macro_server: ProcMacroServerChoice,
     pub prefill_caches: bool,
+    pub proc_macro_processes: usize,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -113,15 +114,25 @@ pub fn load_workspace_into_db(
     let proc_macro_server = match &load_config.with_proc_macro_server {
         ProcMacroServerChoice::Sysroot => ws.find_sysroot_proc_macro_srv().map(|it| {
             it.and_then(|it| {
-                ProcMacroClient::spawn(&it, extra_env, ws.toolchain.as_ref()).map_err(Into::into)
+                ProcMacroClient::spawn(
+                    &it,
+                    extra_env,
+                    ws.toolchain.as_ref(),
+                    load_config.proc_macro_processes,
+                )
+                .map_err(Into::into)
             })
             .map_err(|e| ProcMacroLoadingError::ProcMacroSrvError(e.to_string().into_boxed_str()))
         }),
-        ProcMacroServerChoice::Explicit(path) => {
-            Some(ProcMacroClient::spawn(path, extra_env, ws.toolchain.as_ref()).map_err(|e| {
-                ProcMacroLoadingError::ProcMacroSrvError(e.to_string().into_boxed_str())
-            }))
-        }
+        ProcMacroServerChoice::Explicit(path) => Some(
+            ProcMacroClient::spawn(
+                path,
+                extra_env,
+                ws.toolchain.as_ref(),
+                load_config.proc_macro_processes,
+            )
+            .map_err(|e| ProcMacroLoadingError::ProcMacroSrvError(e.to_string().into_boxed_str())),
+        ),
         ProcMacroServerChoice::None => Some(Err(ProcMacroLoadingError::Disabled)),
     };
     match &proc_macro_server {
@@ -657,6 +668,7 @@ mod tests {
             load_out_dirs_from_check: false,
             with_proc_macro_server: ProcMacroServerChoice::None,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let (db, _vfs, _proc_macro) =
             load_workspace_at(path, &cargo_config, &load_cargo_config, &|_| {}).unwrap();

From 721361f2898b4e3c299e3687b8b4581b2e6c2b48 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 09:45:06 +0530
Subject: [PATCH 0305/1843] update all cli workflows

---
 .../crates/proc-macro-api/src/lib.rs          |  5 ++---
 .../rust-analyzer/src/cli/analysis_stats.rs   |  1 +
 .../rust-analyzer/src/cli/diagnostics.rs      |  1 +
 .../crates/rust-analyzer/src/cli/lsif.rs      |  1 +
 .../rust-analyzer/src/cli/prime_caches.rs     |  1 +
 .../crates/rust-analyzer/src/cli/run_tests.rs |  1 +
 .../rust-analyzer/src/cli/rustc_tests.rs      |  1 +
 .../crates/rust-analyzer/src/cli/scip.rs      |  1 +
 .../crates/rust-analyzer/src/cli/ssr.rs       |  2 ++
 .../src/cli/unresolved_references.rs          |  1 +
 .../src/integrated_benchmarks.rs              |  3 +++
 .../crates/rust-analyzer/src/reload.rs        | 20 +++++++++++--------
 12 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index ad462ff31a1b..2c0008ae1d82 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -29,9 +29,7 @@ use std::{fmt, io, sync::Arc, time::SystemTime};
 
 pub use crate::transport::codec::Codec;
 use crate::{
-    bidirectional_protocol::SubCallback,
-    pool::{ProcMacroServerPool, default_pool_size},
-    process::ProcMacroServerProcess,
+    bidirectional_protocol::SubCallback, pool::ProcMacroServerPool, process::ProcMacroServerProcess,
 };
 
 /// The versions of the server protocol
@@ -155,6 +153,7 @@ impl ProcMacroClient {
             Item = (impl AsRef, &'a Option>),
         > + Clone,
         version: Option<&Version>,
+        num_process: usize,
     ) -> io::Result {
         let process = ProcMacroServerProcess::spawn(process_path, env, version)?;
         Ok(ProcMacroClient { process: Arc::new(process), path: process_path.to_owned() })
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs
index a02d1a78564f..1995d3889891 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs
@@ -91,6 +91,7 @@ impl flags::AnalysisStats {
                 }
             },
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
 
         let build_scripts_time = if self.disable_build_scripts {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs
index 776069f155f0..575c77f8428c 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs
@@ -41,6 +41,7 @@ impl flags::Diagnostics {
             load_out_dirs_from_check: !self.disable_build_scripts,
             with_proc_macro_server,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let (db, _vfs, _proc_macro) =
             load_workspace_at(&self.path, &cargo_config, &load_cargo_config, &|_| {})?;
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
index f3b0699d5515..e5e238db6361 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
@@ -293,6 +293,7 @@ impl flags::Lsif {
             load_out_dirs_from_check: true,
             with_proc_macro_server: ProcMacroServerChoice::Sysroot,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let path = AbsPathBuf::assert_utf8(env::current_dir()?.join(self.path));
         let root = ProjectManifest::discover_single(&path)?;
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/prime_caches.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/prime_caches.rs
index 467d8a53884a..d5da6791797b 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/prime_caches.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/prime_caches.rs
@@ -38,6 +38,7 @@ impl flags::PrimeCaches {
             // we want to ensure that this command, not `load_workspace_at`,
             // is responsible for that work.
             prefill_caches: false,
+            proc_macro_processes: config.proc_macro_num_processes(),
         };
 
         let root = AbsPathBuf::assert_utf8(std::env::current_dir()?.join(root));
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/run_tests.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/run_tests.rs
index 82ace8c8b315..d4a56d773e7d 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/run_tests.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/run_tests.rs
@@ -23,6 +23,7 @@ impl flags::RunTests {
             load_out_dirs_from_check: true,
             with_proc_macro_server: ProcMacroServerChoice::Sysroot,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let (ref db, _vfs, _proc_macro) =
             load_workspace_at(&self.path, &cargo_config, &load_cargo_config, &|_| {})?;
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs
index 249566d2ac16..e8c6c5f4d4f7 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs
@@ -103,6 +103,7 @@ impl Tester {
             load_out_dirs_from_check: false,
             with_proc_macro_server: ProcMacroServerChoice::Sysroot,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let (db, _vfs, _proc_macro) =
             load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
index 271d2507bcfe..ed0476697c9c 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
@@ -52,6 +52,7 @@ impl flags::Scip {
             load_out_dirs_from_check: true,
             with_proc_macro_server: ProcMacroServerChoice::Sysroot,
             prefill_caches: true,
+            proc_macro_processes: config.proc_macro_num_processes(),
         };
         let cargo_config = config.cargo(None);
         let (db, vfs, _) = load_workspace_at(
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/ssr.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/ssr.rs
index 39186831459c..5c69bda723fb 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/ssr.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/ssr.rs
@@ -20,6 +20,7 @@ impl flags::Ssr {
             load_out_dirs_from_check: true,
             with_proc_macro_server: ProcMacroServerChoice::Sysroot,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let (ref db, vfs, _proc_macro) = load_workspace_at(
             &std::env::current_dir()?,
@@ -56,6 +57,7 @@ impl flags::Search {
             load_out_dirs_from_check: true,
             with_proc_macro_server: ProcMacroServerChoice::Sysroot,
             prefill_caches: false,
+            proc_macro_processes: 1,
         };
         let (ref db, _vfs, _proc_macro) = load_workspace_at(
             &std::env::current_dir()?,
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/unresolved_references.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/unresolved_references.rs
index 294add682d01..49c6fcb91ebf 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/unresolved_references.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/unresolved_references.rs
@@ -44,6 +44,7 @@ impl flags::UnresolvedReferences {
             load_out_dirs_from_check: !self.disable_build_scripts,
             with_proc_macro_server,
             prefill_caches: false,
+            proc_macro_processes: config.proc_macro_num_processes(),
         };
         let (db, vfs, _proc_macro) =
             load_workspace_at(&self.path, &cargo_config, &load_cargo_config, &|_| {})?;
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs
index c61825b99fec..d16ca2fb48ac 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs
@@ -53,6 +53,7 @@ fn integrated_highlighting_benchmark() {
         load_out_dirs_from_check: true,
         with_proc_macro_server: ProcMacroServerChoice::Sysroot,
         prefill_caches: false,
+        proc_macro_processes: 1,
     };
 
     let (db, vfs, _proc_macro) = {
@@ -121,6 +122,7 @@ fn integrated_completion_benchmark() {
         load_out_dirs_from_check: true,
         with_proc_macro_server: ProcMacroServerChoice::Sysroot,
         prefill_caches: true,
+        proc_macro_processes: 1,
     };
 
     let (db, vfs, _proc_macro) = {
@@ -322,6 +324,7 @@ fn integrated_diagnostics_benchmark() {
         load_out_dirs_from_check: true,
         with_proc_macro_server: ProcMacroServerChoice::Sysroot,
         prefill_caches: true,
+        proc_macro_processes: 1,
     };
 
     let (db, vfs, _proc_macro) = {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs
index ccafbd7b30b9..83f4a19b39fa 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs
@@ -701,15 +701,19 @@ impl GlobalState {
                     _ => Default::default(),
                 };
                 info!("Using proc-macro server at {path}");
+                let num_process = self.config.proc_macro_num_processes();
 
-                Some(ProcMacroClient::spawn(&path, &env, ws.toolchain.as_ref()).map_err(|err| {
-                    tracing::error!(
-                        "Failed to run proc-macro server from path {path}, error: {err:?}",
-                    );
-                    anyhow::format_err!(
-                        "Failed to run proc-macro server from path {path}, error: {err:?}",
-                    )
-                }))
+                Some(
+                    ProcMacroClient::spawn(&path, &env, ws.toolchain.as_ref(), num_process)
+                        .map_err(|err| {
+                            tracing::error!(
+                                "Failed to run proc-macro server from path {path}, error: {err:?}",
+                            );
+                            anyhow::format_err!(
+                                "Failed to run proc-macro server from path {path}, error: {err:?}",
+                            )
+                        }),
+                )
             }))
         }
 

From 0587cbdd6fc2321564394828d6172fd268fc6617 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 09:56:02 +0530
Subject: [PATCH 0306/1843] optimize pick_process to short circuit and return
 as early as possible if a valid process is found

---
 .../crates/proc-macro-api/src/pool.rs         | 28 +++++++++++++------
 .../crates/rust-analyzer/src/config.rs        |  2 +-
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
index 0cb505aa40a9..a637bc0e480a 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/pool.rs
@@ -28,14 +28,26 @@ impl ProcMacroServerPool {
     }
 
     pub(crate) fn pick_process(&self) -> Result<&ProcMacroServerProcess, ServerError> {
-        self.workers
-            .iter()
-            .filter(|w| w.exited().is_none())
-            .min_by_key(|w| w.number_of_active_req())
-            .ok_or_else(|| ServerError {
-                message: "all proc-macro server workers have exited".into(),
-                io: None,
-            })
+        let mut best: Option<&ProcMacroServerProcess> = None;
+        let mut best_load = u32::MAX;
+
+        for w in self.workers.iter().filter(|w| w.exited().is_none()) {
+            let load = w.number_of_active_req();
+
+            if load == 0 {
+                return Ok(w);
+            }
+
+            if load < best_load {
+                best = Some(w);
+                best_load = load;
+            }
+        }
+
+        best.ok_or_else(|| ServerError {
+            message: "all proc-macro server workers have exited".into(),
+            io: None,
+        })
     }
 
     pub(crate) fn load_dylib(
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index cb6552c32ffb..409f2468a7b1 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -380,7 +380,7 @@ config_data! {
         /// The number of worker threads in the main loop. The default `null` means to pick
         /// automatically.
         numThreads: Option = None,
-        /// The number of proc-macro-srv processes 
+        /// The number of proc-macro-srv processes
         proc_macro_processes: NumProcesses = NumProcesses::Concrete(1),
 
         /// Expand attribute macros. Requires `#rust-analyzer.procMacro.enable#` to be set.

From b49417eea9fcb252ffe62d35068375534d553af0 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 10:22:49 +0530
Subject: [PATCH 0307/1843] fix test and update some autogen files

---
 .../crates/rust-analyzer/src/config.rs        | 24 ++++++++++++++++--
 .../docs/book/src/configuration_generated.md  | 10 ++++++++
 .../rust-analyzer/editors/code/package.json   | 25 +++++++++++++++++++
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index 409f2468a7b1..015e6df96f2f 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -380,8 +380,6 @@ config_data! {
         /// The number of worker threads in the main loop. The default `null` means to pick
         /// automatically.
         numThreads: Option = None,
-        /// The number of proc-macro-srv processes
-        proc_macro_processes: NumProcesses = NumProcesses::Concrete(1),
 
         /// Expand attribute macros. Requires `#rust-analyzer.procMacro.enable#` to be set.
         procMacro_attributes_enable: bool = true,
@@ -392,6 +390,12 @@ config_data! {
         /// Internal config, path to proc-macro server executable.
         procMacro_server: Option = None,
 
+        /// Number of proc-macro server processes to spawn.
+        ///
+        /// Controls how many independent `proc-macro-srv` processes rust-analyzer
+        /// runs in parallel to handle macro expansion.
+        proc_macro_processes: NumProcesses = NumProcesses::Concrete(1),
+
         /// The path where to save memory profiling output.
         ///
         /// **Note:** Memory profiling is not enabled by default in rust-analyzer builds, you need to build
@@ -3920,6 +3924,22 @@ fn field_props(field: &str, ty: &str, doc: &[&str], default: &str) -> serde_json
                 },
             ],
         },
+        "NumProcesses" => set! {
+            "anyOf": [
+                {
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 255
+                },
+                {
+                    "type": "string",
+                    "enum": ["physical"],
+                    "enumDescriptions": [
+                        "Use the number of physical cores",
+                    ],
+                },
+            ],
+        },
         "Option" => set! {
             "anyOf": [
                 {
diff --git a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
index c4124aaae075..d3f41fb152e1 100644
--- a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
+++ b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
@@ -1325,6 +1325,16 @@ Default: `null`
 Internal config, path to proc-macro server executable.
 
 
+## rust-analyzer.proc.macro.processes {#proc.macro.processes}
+
+Default: `1`
+
+Number of proc-macro server processes to spawn.
+
+Controls how many independent `proc-macro-srv` processes rust-analyzer
+runs in parallel to handle macro expansion.
+
+
 ## rust-analyzer.profiling.memoryProfile {#profiling.memoryProfile}
 
 Default: `null`
diff --git a/src/tools/rust-analyzer/editors/code/package.json b/src/tools/rust-analyzer/editors/code/package.json
index 0d91378706a4..a1266c4a67ed 100644
--- a/src/tools/rust-analyzer/editors/code/package.json
+++ b/src/tools/rust-analyzer/editors/code/package.json
@@ -2783,6 +2783,31 @@
                     }
                 }
             },
+            {
+                "title": "Proc",
+                "properties": {
+                    "rust-analyzer.proc.macro.processes": {
+                        "markdownDescription": "Number of proc-macro server processes to spawn.\n\nControls how many independent `proc-macro-srv` processes rust-analyzer\nruns in parallel to handle macro expansion.",
+                        "default": 1,
+                        "anyOf": [
+                            {
+                                "type": "number",
+                                "minimum": 0,
+                                "maximum": 255
+                            },
+                            {
+                                "type": "string",
+                                "enum": [
+                                    "physical"
+                                ],
+                                "enumDescriptions": [
+                                    "Use the number of physical cores"
+                                ]
+                            }
+                        ]
+                    }
+                }
+            },
             {
                 "title": "Profiling",
                 "properties": {

From a81da31f4096bf754eed0c40b384ef2f5b4d854c Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Mon, 5 Jan 2026 10:32:00 +0530
Subject: [PATCH 0308/1843] rename from proc_macro_processes to
 procMacro_processes

---
 .../crates/rust-analyzer/src/config.rs        | 10 +++----
 .../docs/book/src/configuration_generated.md  | 16 +++++------
 .../rust-analyzer/editors/code/package.json   | 28 +++++++++----------
 3 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index 015e6df96f2f..98495f6150da 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -387,14 +387,14 @@ config_data! {
         /// Enable support for procedural macros, implies `#rust-analyzer.cargo.buildScripts.enable#`.
         procMacro_enable: bool = true,
 
-        /// Internal config, path to proc-macro server executable.
-        procMacro_server: Option = None,
-
         /// Number of proc-macro server processes to spawn.
         ///
         /// Controls how many independent `proc-macro-srv` processes rust-analyzer
         /// runs in parallel to handle macro expansion.
-        proc_macro_processes: NumProcesses = NumProcesses::Concrete(1),
+        procMacro_processes: NumProcesses = NumProcesses::Concrete(1),
+
+        /// Internal config, path to proc-macro server executable.
+        procMacro_server: Option = None,
 
         /// The path where to save memory profiling output.
         ///
@@ -2648,7 +2648,7 @@ impl Config {
     }
 
     pub fn proc_macro_num_processes(&self) -> usize {
-        match self.proc_macro_processes() {
+        match self.procMacro_processes() {
             NumProcesses::Concrete(0) | NumProcesses::Physical => num_cpus::get_physical(),
             &NumProcesses::Concrete(n) => n,
         }
diff --git a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
index d3f41fb152e1..5b1a2e111196 100644
--- a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
+++ b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
@@ -1318,14 +1318,7 @@ These proc-macros will be ignored when trying to expand them.
 This config takes a map of crate names with the exported proc-macro names to ignore as values.
 
 
-## rust-analyzer.procMacro.server {#procMacro.server}
-
-Default: `null`
-
-Internal config, path to proc-macro server executable.
-
-
-## rust-analyzer.proc.macro.processes {#proc.macro.processes}
+## rust-analyzer.procMacro.processes {#procMacro.processes}
 
 Default: `1`
 
@@ -1335,6 +1328,13 @@ Controls how many independent `proc-macro-srv` processes rust-analyzer
 runs in parallel to handle macro expansion.
 
 
+## rust-analyzer.procMacro.server {#procMacro.server}
+
+Default: `null`
+
+Internal config, path to proc-macro server executable.
+
+
 ## rust-analyzer.profiling.memoryProfile {#profiling.memoryProfile}
 
 Default: `null`
diff --git a/src/tools/rust-analyzer/editors/code/package.json b/src/tools/rust-analyzer/editors/code/package.json
index a1266c4a67ed..406e41767f6d 100644
--- a/src/tools/rust-analyzer/editors/code/package.json
+++ b/src/tools/rust-analyzer/editors/code/package.json
@@ -2773,20 +2773,7 @@
             {
                 "title": "Proc Macro",
                 "properties": {
-                    "rust-analyzer.procMacro.server": {
-                        "markdownDescription": "Internal config, path to proc-macro server executable.",
-                        "default": null,
-                        "type": [
-                            "null",
-                            "string"
-                        ]
-                    }
-                }
-            },
-            {
-                "title": "Proc",
-                "properties": {
-                    "rust-analyzer.proc.macro.processes": {
+                    "rust-analyzer.procMacro.processes": {
                         "markdownDescription": "Number of proc-macro server processes to spawn.\n\nControls how many independent `proc-macro-srv` processes rust-analyzer\nruns in parallel to handle macro expansion.",
                         "default": 1,
                         "anyOf": [
@@ -2808,6 +2795,19 @@
                     }
                 }
             },
+            {
+                "title": "Proc Macro",
+                "properties": {
+                    "rust-analyzer.procMacro.server": {
+                        "markdownDescription": "Internal config, path to proc-macro server executable.",
+                        "default": null,
+                        "type": [
+                            "null",
+                            "string"
+                        ]
+                    }
+                }
+            },
             {
                 "title": "Profiling",
                 "properties": {

From 8da5de0ca02c316fee3fb97c11052e3d70a32bef Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 11 Jan 2026 15:46:03 +0530
Subject: [PATCH 0309/1843] rebased changes

---
 .../crates/proc-macro-api/src/lib.rs          | 26 +++++++++++++++----
 .../crates/proc-macro-api/src/process.rs      | 13 +++-------
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index 2c0008ae1d82..3acd0b292a31 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -155,8 +155,15 @@ impl ProcMacroClient {
         version: Option<&Version>,
         num_process: usize,
     ) -> io::Result {
-        let process = ProcMacroServerProcess::spawn(process_path, env, version)?;
-        Ok(ProcMacroClient { process: Arc::new(process), path: process_path.to_owned() })
+        let pool_size = num_process;
+        let mut workers = Vec::with_capacity(pool_size);
+        for _ in 0..pool_size {
+            let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?;
+            workers.push(worker);
+        }
+
+        let pool = ProcMacroServerPool::new(workers);
+        Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() })
     }
 
     /// Invokes `spawn` and returns a client connected to the resulting read and write handles.
@@ -170,11 +177,20 @@ impl ProcMacroClient {
             Box,
             Box,
             Box,
-        )>,
+        )> + Clone,
         version: Option<&Version>,
+        num_process: usize,
     ) -> io::Result {
-        let process = ProcMacroServerProcess::run(spawn, version, || "".to_owned())?;
-        Ok(ProcMacroClient { worker: Arc::new(process), path: process_path.to_owned() })
+        let pool_size = num_process;
+        let mut workers = Vec::with_capacity(pool_size);
+        for _ in 0..pool_size {
+            let worker =
+                ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?;
+            workers.push(worker);
+        }
+
+        let pool = ProcMacroServerPool::new(workers);
+        Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() })
     }
 
     /// Returns the absolute path to the proc-macro server.
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index c1b95fa7f10e..2f5bef69abd5 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -210,9 +210,8 @@ impl ProcMacroServerProcess {
         callback: Option>,
     ) -> Result, String>, ServerError> {
         match self.protocol {
-            Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => {
-                legacy_protocol::find_proc_macros(self, dylib_path)
-            }
+            Protocol::LegacyJson { .. } => legacy_protocol::find_proc_macros(self, dylib_path),
+
             Protocol::BidirectionalPostcardPrototype { .. } => {
                 let cb = callback.expect("callback required for bidirectional protocol");
                 bidirectional_protocol::find_proc_macros(self, dylib_path, cb)
@@ -279,7 +278,7 @@ impl ProcMacroServerProcess {
         let result = match self.protocol {
             Protocol::LegacyJson { .. } => legacy_protocol::expand(
                 proc_macro,
-                    self,
+                self,
                 subtree,
                 attr,
                 env,
@@ -344,11 +343,7 @@ impl ProcMacroServerProcess {
                 match state.process.exit_err() {
                     None => e,
                     Some(server_error) => {
-                        proc_macro_worker
-                            .get_exited()
-                            .get_or_init(|| AssertUnwindSafe(server_error))
-                            .0
-                            .clone()
+                        self.exited.get_or_init(|| AssertUnwindSafe(server_error)).0.clone()
                     }
                 }
             } else {

From b7e4315b3287ba27cbaf396a0f3d7baf7c237ecd Mon Sep 17 00:00:00 2001
From: Samuel Tardieu 
Date: Thu, 15 Jan 2026 20:35:35 +0100
Subject: [PATCH 0310/1843] Do not output an error if standard output is full
 on --help/--version

This matches rustc's behavior.
---
 src/driver.rs | 13 ++++++++-----
 src/main.rs   | 17 +++++++++++------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/driver.rs b/src/driver.rs
index 8693973ef78c..6094c8445398 100644
--- a/src/driver.rs
+++ b/src/driver.rs
@@ -32,11 +32,10 @@ use rustc_span::symbol::Symbol;
 
 use std::env;
 use std::fs::read_to_string;
+use std::io::Write as _;
 use std::path::Path;
 use std::process::exit;
 
-use anstream::println;
-
 /// If a command-line option matches `find_arg`, then apply the predicate `pred` on its value. If
 /// true, then return it. The parameter is assumed to be either `--arg=value` or `--arg value`.
 fn arg_value<'a>(args: &'a [String], find_arg: &str, pred: impl Fn(&str) -> bool) -> Option<&'a str> {
@@ -186,7 +185,9 @@ impl rustc_driver::Callbacks for ClippyCallbacks {
 }
 
 fn display_help() {
-    println!("{}", help_message());
+    if writeln!(&mut anstream::stdout().lock(), "{}", help_message()).is_err() {
+        exit(rustc_driver::EXIT_FAILURE);
+    }
 }
 
 const BUG_REPORT_URL: &str = "https://github.com/rust-lang/rust-clippy/issues/new?template=ice.yml";
@@ -253,8 +254,10 @@ pub fn main() {
         if orig_args.iter().any(|a| a == "--version" || a == "-V") {
             let version_info = rustc_tools_util::get_version_info!();
 
-            println!("{version_info}");
-            exit(0);
+            match writeln!(&mut anstream::stdout().lock(), "{version_info}") {
+                Ok(()) => exit(rustc_driver::EXIT_SUCCESS),
+                Err(_) => exit(rustc_driver::EXIT_FAILURE),
+            }
         }
 
         // Setting RUSTC_WRAPPER causes Cargo to pass 'rustc' as the first argument.
diff --git a/src/main.rs b/src/main.rs
index 688161c7bfcb..98b888444831 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,19 +4,24 @@
 // warn on lints, that are included in `rust-lang/rust`s bootstrap
 #![warn(rust_2018_idioms, unused_lifetimes)]
 
-use std::env;
-use std::path::PathBuf;
-use std::process::{self, Command};
+extern crate rustc_driver;
 
-use anstream::println;
+use std::env;
+use std::io::Write as _;
+use std::path::PathBuf;
+use std::process::{self, Command, exit};
 
 fn show_help() {
-    println!("{}", help_message());
+    if writeln!(&mut anstream::stdout().lock(), "{}", help_message()).is_err() {
+        exit(rustc_driver::EXIT_FAILURE);
+    }
 }
 
 fn show_version() {
     let version_info = rustc_tools_util::get_version_info!();
-    println!("{version_info}");
+    if writeln!(&mut anstream::stdout().lock(), "{version_info}").is_err() {
+        exit(rustc_driver::EXIT_FAILURE);
+    }
 }
 
 pub fn main() {

From 2177aa9ac736b0c16c8ad73230508fcb273b980e Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Fri, 16 Jan 2026 18:09:37 +0100
Subject: [PATCH 0311/1843] Manually include intrinsic conversion that is not
 present in LLVM files

---
 src/intrinsic/old_archs.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/intrinsic/old_archs.rs b/src/intrinsic/old_archs.rs
index 3a59707b2ebe..8d3e3487b5cb 100644
--- a/src/intrinsic/old_archs.rs
+++ b/src/intrinsic/old_archs.rs
@@ -242,6 +242,8 @@ pub(crate) fn old_archs(arch: &str, name: &str) -> ArchCheckResult {
             "avx.cvt.ps2.pd.256" => "__builtin_ia32_cvtps2pd256",
             "avx.cvtdq2.pd.256" => "__builtin_ia32_cvtdq2pd256",
             "avx.cvtdq2.ps.256" => "__builtin_ia32_cvtdq2ps256",
+            "avx.round.pd.256" => "__builtin_ia32_roundpd256",
+            "avx.round.ps.256" => "__builtin_ia32_roundps256",
             "avx.sqrt.pd.256" => "__builtin_ia32_sqrtpd256",
             "avx.sqrt.ps.256" => "__builtin_ia32_sqrtps256",
             "avx.storeu.dq.256" => "__builtin_ia32_storedqu256",
@@ -1352,6 +1354,10 @@ pub(crate) fn old_archs(arch: &str, name: &str) -> ArchCheckResult {
             "sse41.pmovzxwd" => "__builtin_ia32_pmovzxwd128",
             "sse41.pmovzxwq" => "__builtin_ia32_pmovzxwq128",
             "sse41.pmuldq" => "__builtin_ia32_pmuldq128",
+            "sse41.round.pd" => "__builtin_ia32_roundpd",
+            "sse41.round.ps" => "__builtin_ia32_roundps",
+            "sse41.round.sd" => "__builtin_ia32_roundsd",
+            "sse41.round.ss" => "__builtin_ia32_roundss",
             "sse4a.movnt.sd" => "__builtin_ia32_movntsd",
             "sse4a.movnt.ss" => "__builtin_ia32_movntss",
             "ssse3.pabs.b.128" => "__builtin_ia32_pabsb128",

From 5c85d522d0083bee0d5fd7f26868fa0e62372485 Mon Sep 17 00:00:00 2001
From: Manuel Drehwald 
Date: Tue, 13 Jan 2026 09:52:36 -0800
Subject: [PATCH 0312/1843] Generate global openmp metadata to trigger llvm
 openmp-opt pass

---
 compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs | 4 ++++
 tests/codegen-llvm/gpu_offload/gpu_host.rs             | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
index b8eb4f038216..084d40317ba8 100644
--- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
+++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
@@ -55,6 +55,10 @@ impl<'ll> OffloadGlobals<'ll> {
         let init_ty = cx.type_func(&[], cx.type_void());
         let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty);
 
+        // We want LLVM's openmp-opt pass to pick up and optimize this module, since it covers both
+        // openmp and offload optimizations.
+        llvm::add_module_flag_u32(cx.llmod(), llvm::ModuleFlagMergeBehavior::Max, "openmp", 51);
+
         OffloadGlobals {
             launcher_fn,
             launcher_ty,
diff --git a/tests/codegen-llvm/gpu_offload/gpu_host.rs b/tests/codegen-llvm/gpu_offload/gpu_host.rs
index dcbd65b14427..27ff6f325aa0 100644
--- a/tests/codegen-llvm/gpu_offload/gpu_host.rs
+++ b/tests/codegen-llvm/gpu_offload/gpu_host.rs
@@ -104,3 +104,5 @@ pub fn _kernel_1(x: &mut [f32; 256]) {
 // CHECK-NEXT:   call void @__tgt_unregister_lib(ptr nonnull %EmptyDesc)
 // CHECK-NEXT:   ret void
 // CHECK-NEXT: }
+
+// CHECK: !{i32 7, !"openmp", i32 51}

From c9bbf951c22fff8a9c1bc4cfc4c04b43bd0fe431 Mon Sep 17 00:00:00 2001
From: linshuy2 
Date: Fri, 16 Jan 2026 20:22:33 +0000
Subject: [PATCH 0313/1843] Allow `unnecessary_sort_by` to lint closures with
 input patterns

---
 .../src/methods/unnecessary_sort_by.rs        | 288 +++++++++++++-----
 clippy_utils/src/ty/mod.rs                    |  15 +
 tests/ui/unnecessary_sort_by.fixed            |  32 ++
 tests/ui/unnecessary_sort_by.rs               |  32 ++
 tests/ui/unnecessary_sort_by.stderr           |  44 ++-
 5 files changed, 341 insertions(+), 70 deletions(-)

diff --git a/clippy_lints/src/methods/unnecessary_sort_by.rs b/clippy_lints/src/methods/unnecessary_sort_by.rs
index c7339a3bdd05..ac13fc0b6850 100644
--- a/clippy_lints/src/methods/unnecessary_sort_by.rs
+++ b/clippy_lints/src/methods/unnecessary_sort_by.rs
@@ -1,42 +1,51 @@
 use clippy_utils::diagnostics::span_lint_and_then;
 use clippy_utils::res::{MaybeDef, MaybeTypeckRes};
+use clippy_utils::source::snippet_with_applicability;
 use clippy_utils::std_or_core;
 use clippy_utils::sugg::Sugg;
-use clippy_utils::ty::{implements_trait, is_copy};
+use clippy_utils::ty::{implements_trait, is_copy, peel_n_ty_refs};
+use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::Applicability;
 use rustc_hir::{Closure, Expr, ExprKind, Mutability, Param, Pat, PatKind, Path, PathSegment, QPath};
 use rustc_lint::LateContext;
 use rustc_middle::ty::{self, GenericArgKind, Ty};
-use rustc_span::sym;
-use rustc_span::symbol::{Ident, Symbol};
+use rustc_span::symbol::Ident;
+use rustc_span::{Span, sym};
 use std::iter;
+use std::ops::Not;
 
 use super::UNNECESSARY_SORT_BY;
 
-enum LintTrigger<'tcx> {
+enum LintTrigger {
     Sort,
-    SortByKey(SortByKeyDetection<'tcx>),
+    SortByKey(SortByKeyDetection),
 }
 
-struct SortByKeyDetection<'tcx> {
-    closure_arg: Symbol,
-    closure_body: &'tcx Expr<'tcx>,
+struct SortByKeyDetection {
+    closure_arg: Span,
+    closure_body: String,
     reverse: bool,
+    applicability: Applicability,
 }
 
 /// Detect if the two expressions are mirrored (identical, except one
 /// contains a and the other replaces it with b)
-fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: Ident, b_expr: &Expr<'_>, b_ident: Ident) -> bool {
+fn mirrored_exprs(
+    a_expr: &Expr<'_>,
+    b_expr: &Expr<'_>,
+    binding_map: &BindingMap,
+    binding_source: BindingSource,
+) -> bool {
     match (a_expr.kind, b_expr.kind) {
         // Two arrays with mirrored contents
-        (ExprKind::Array(left_exprs), ExprKind::Array(right_exprs)) => {
-            iter::zip(left_exprs, right_exprs).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident))
-        },
+        (ExprKind::Array(left_exprs), ExprKind::Array(right_exprs)) => iter::zip(left_exprs, right_exprs)
+            .all(|(left, right)| mirrored_exprs(left, right, binding_map, binding_source)),
         // The two exprs are function calls.
         // Check to see that the function itself and its arguments are mirrored
         (ExprKind::Call(left_expr, left_args), ExprKind::Call(right_expr, right_args)) => {
-            mirrored_exprs(left_expr, a_ident, right_expr, b_ident)
-                && iter::zip(left_args, right_args).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident))
+            mirrored_exprs(left_expr, right_expr, binding_map, binding_source)
+                && iter::zip(left_args, right_args)
+                    .all(|(left, right)| mirrored_exprs(left, right, binding_map, binding_source))
         },
         // The two exprs are method calls.
         // Check to see that the function is the same and the arguments and receivers are mirrored
@@ -45,31 +54,31 @@ fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: Ident, b_expr: &Expr<'_>, b_ident:
             ExprKind::MethodCall(right_segment, right_receiver, right_args, _),
         ) => {
             left_segment.ident == right_segment.ident
-                && iter::zip(left_args, right_args).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident))
-                && mirrored_exprs(left_receiver, a_ident, right_receiver, b_ident)
+                && iter::zip(left_args, right_args)
+                    .all(|(left, right)| mirrored_exprs(left, right, binding_map, binding_source))
+                && mirrored_exprs(left_receiver, right_receiver, binding_map, binding_source)
         },
         // Two tuples with mirrored contents
-        (ExprKind::Tup(left_exprs), ExprKind::Tup(right_exprs)) => {
-            iter::zip(left_exprs, right_exprs).all(|(left, right)| mirrored_exprs(left, a_ident, right, b_ident))
-        },
+        (ExprKind::Tup(left_exprs), ExprKind::Tup(right_exprs)) => iter::zip(left_exprs, right_exprs)
+            .all(|(left, right)| mirrored_exprs(left, right, binding_map, binding_source)),
         // Two binary ops, which are the same operation and which have mirrored arguments
         (ExprKind::Binary(left_op, left_left, left_right), ExprKind::Binary(right_op, right_left, right_right)) => {
             left_op.node == right_op.node
-                && mirrored_exprs(left_left, a_ident, right_left, b_ident)
-                && mirrored_exprs(left_right, a_ident, right_right, b_ident)
+                && mirrored_exprs(left_left, right_left, binding_map, binding_source)
+                && mirrored_exprs(left_right, right_right, binding_map, binding_source)
         },
         // Two unary ops, which are the same operation and which have the same argument
         (ExprKind::Unary(left_op, left_expr), ExprKind::Unary(right_op, right_expr)) => {
-            left_op == right_op && mirrored_exprs(left_expr, a_ident, right_expr, b_ident)
+            left_op == right_op && mirrored_exprs(left_expr, right_expr, binding_map, binding_source)
         },
         // The two exprs are literals of some kind
         (ExprKind::Lit(left_lit), ExprKind::Lit(right_lit)) => left_lit.node == right_lit.node,
-        (ExprKind::Cast(left, _), ExprKind::Cast(right, _)) => mirrored_exprs(left, a_ident, right, b_ident),
+        (ExprKind::Cast(left, _), ExprKind::Cast(right, _)) => mirrored_exprs(left, right, binding_map, binding_source),
         (ExprKind::DropTemps(left_block), ExprKind::DropTemps(right_block)) => {
-            mirrored_exprs(left_block, a_ident, right_block, b_ident)
+            mirrored_exprs(left_block, right_block, binding_map, binding_source)
         },
         (ExprKind::Field(left_expr, left_ident), ExprKind::Field(right_expr, right_ident)) => {
-            left_ident.name == right_ident.name && mirrored_exprs(left_expr, a_ident, right_expr, b_ident)
+            left_ident.name == right_ident.name && mirrored_exprs(left_expr, right_expr, binding_map, binding_source)
         },
         // Two paths: either one is a and the other is b, or they're identical to each other
         (
@@ -89,60 +98,174 @@ fn mirrored_exprs(a_expr: &Expr<'_>, a_ident: Ident, b_expr: &Expr<'_>, b_ident:
             )),
         ) => {
             (iter::zip(left_segments, right_segments).all(|(left, right)| left.ident == right.ident)
-                && left_segments
-                    .iter()
-                    .all(|seg| seg.ident != a_ident && seg.ident != b_ident))
+                && left_segments.iter().all(|seg| {
+                    !binding_map.contains_key(&BindingKey {
+                        ident: seg.ident,
+                        source: BindingSource::Left,
+                    }) && !binding_map.contains_key(&BindingKey {
+                        ident: seg.ident,
+                        source: BindingSource::Right,
+                    })
+                }))
                 || (left_segments.len() == 1
-                    && left_segments[0].ident == a_ident
                     && right_segments.len() == 1
-                    && right_segments[0].ident == b_ident)
+                    && binding_map
+                        .get(&BindingKey {
+                            ident: left_segments[0].ident,
+                            source: binding_source,
+                        })
+                        .is_some_and(|value| value.mirrored.ident == right_segments[0].ident))
         },
         // Matching expressions, but one or both is borrowed
         (
             ExprKind::AddrOf(left_kind, Mutability::Not, left_expr),
             ExprKind::AddrOf(right_kind, Mutability::Not, right_expr),
-        ) => left_kind == right_kind && mirrored_exprs(left_expr, a_ident, right_expr, b_ident),
-        (_, ExprKind::AddrOf(_, Mutability::Not, right_expr)) => mirrored_exprs(a_expr, a_ident, right_expr, b_ident),
-        (ExprKind::AddrOf(_, Mutability::Not, left_expr), _) => mirrored_exprs(left_expr, a_ident, b_expr, b_ident),
+        ) => left_kind == right_kind && mirrored_exprs(left_expr, right_expr, binding_map, binding_source),
+        (_, ExprKind::AddrOf(_, Mutability::Not, right_expr)) => {
+            mirrored_exprs(a_expr, right_expr, binding_map, binding_source)
+        },
+        (ExprKind::AddrOf(_, Mutability::Not, left_expr), _) => {
+            mirrored_exprs(left_expr, b_expr, binding_map, binding_source)
+        },
         _ => false,
     }
 }
 
-fn detect_lint<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, arg: &Expr<'_>) -> Option> {
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+enum BindingSource {
+    Left,
+    Right,
+}
+
+impl Not for BindingSource {
+    type Output = BindingSource;
+
+    fn not(self) -> Self::Output {
+        match self {
+            BindingSource::Left => BindingSource::Right,
+            BindingSource::Right => BindingSource::Left,
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+struct BindingKey {
+    /// The identifier of the binding.
+    ident: Ident,
+    /// The source of the binding.
+    source: BindingSource,
+}
+
+struct BindingValue {
+    /// The mirrored binding.
+    mirrored: BindingKey,
+    /// The number of refs the binding is wrapped in.
+    n_refs: usize,
+}
+
+/// A map from binding info to the number of refs the binding is wrapped in.
+type BindingMap = FxHashMap;
+/// Extract the binding pairs, if the two patterns are mirrored. The pats are assumed to be used in
+/// closure inputs and thus irrefutable.
+fn mapping_of_mirrored_pats(a_pat: &Pat<'_>, b_pat: &Pat<'_>) -> Option {
+    fn mapping_of_mirrored_pats_inner(
+        a_pat: &Pat<'_>,
+        b_pat: &Pat<'_>,
+        mapping: &mut BindingMap,
+        n_refs: usize,
+    ) -> bool {
+        match (&a_pat.kind, &b_pat.kind) {
+            (PatKind::Tuple(a_pats, a_dots), PatKind::Tuple(b_pats, b_dots)) => {
+                a_dots == b_dots
+                    && a_pats.len() == b_pats.len()
+                    && iter::zip(a_pats.iter(), b_pats.iter())
+                        .all(|(a, b)| mapping_of_mirrored_pats_inner(a, b, mapping, n_refs))
+            },
+            (PatKind::Binding(_, _, a_ident, _), PatKind::Binding(_, _, b_ident, _)) => {
+                let a_key = BindingKey {
+                    ident: *a_ident,
+                    source: BindingSource::Left,
+                };
+                let b_key = BindingKey {
+                    ident: *b_ident,
+                    source: BindingSource::Right,
+                };
+                let a_value = BindingValue {
+                    mirrored: b_key,
+                    n_refs,
+                };
+                let b_value = BindingValue {
+                    mirrored: a_key,
+                    n_refs,
+                };
+                mapping.insert(a_key, a_value);
+                mapping.insert(b_key, b_value);
+                true
+            },
+            (PatKind::Wild, PatKind::Wild) => true,
+            (PatKind::TupleStruct(_, a_pats, a_dots), PatKind::TupleStruct(_, b_pats, b_dots)) => {
+                a_dots == b_dots
+                    && a_pats.len() == b_pats.len()
+                    && iter::zip(a_pats.iter(), b_pats.iter())
+                        .all(|(a, b)| mapping_of_mirrored_pats_inner(a, b, mapping, n_refs))
+            },
+            (PatKind::Struct(_, a_fields, a_rest), PatKind::Struct(_, b_fields, b_rest)) => {
+                a_rest == b_rest
+                    && a_fields.len() == b_fields.len()
+                    && iter::zip(a_fields.iter(), b_fields.iter()).all(|(a_field, b_field)| {
+                        a_field.ident == b_field.ident
+                            && mapping_of_mirrored_pats_inner(a_field.pat, b_field.pat, mapping, n_refs)
+                    })
+            },
+            (PatKind::Ref(a_inner, _, _), PatKind::Ref(b_inner, _, _)) => {
+                mapping_of_mirrored_pats_inner(a_inner, b_inner, mapping, n_refs + 1)
+            },
+            (PatKind::Slice(a_elems, None, a_rest), PatKind::Slice(b_elems, None, b_rest)) => {
+                a_elems.len() == b_elems.len()
+                    && iter::zip(a_elems.iter(), b_elems.iter())
+                        .all(|(a, b)| mapping_of_mirrored_pats_inner(a, b, mapping, n_refs))
+                    && a_rest.len() == b_rest.len()
+                    && iter::zip(a_rest.iter(), b_rest.iter())
+                        .all(|(a, b)| mapping_of_mirrored_pats_inner(a, b, mapping, n_refs))
+            },
+            _ => false,
+        }
+    }
+
+    let mut mapping = FxHashMap::default();
+    if mapping_of_mirrored_pats_inner(a_pat, b_pat, &mut mapping, 0) {
+        return Some(mapping);
+    }
+
+    None
+}
+
+fn detect_lint(cx: &LateContext<'_>, expr: &Expr<'_>, arg: &Expr<'_>) -> Option {
     if let Some(method_id) = cx.typeck_results().type_dependent_def_id(expr.hir_id)
         && let Some(impl_id) = cx.tcx.impl_of_assoc(method_id)
         && cx.tcx.type_of(impl_id).instantiate_identity().is_slice()
         && let ExprKind::Closure(&Closure { body, .. }) = arg.kind
         && let closure_body = cx.tcx.hir_body(body)
-        && let &[
-            Param {
-                pat:
-                    &Pat {
-                        kind: PatKind::Binding(_, _, left_ident, _),
-                        ..
-                    },
-                ..
-            },
-            Param {
-                pat:
-                    &Pat {
-                        kind: PatKind::Binding(_, _, right_ident, _),
-                        ..
-                    },
-                ..
-            },
-        ] = closure_body.params
+        && let &[Param { pat: l_pat, .. }, Param { pat: r_pat, .. }] = closure_body.params
+        && let Some(binding_map) = mapping_of_mirrored_pats(l_pat, r_pat)
         && let ExprKind::MethodCall(method_path, left_expr, [right_expr], _) = closure_body.value.kind
         && method_path.ident.name == sym::cmp
         && let Some(ord_trait) = cx.tcx.get_diagnostic_item(sym::Ord)
         && cx.ty_based_def(closure_body.value).opt_parent(cx).opt_def_id() == Some(ord_trait)
     {
-        let (closure_body, closure_arg, reverse) = if mirrored_exprs(left_expr, left_ident, right_expr, right_ident) {
-            (left_expr, left_ident.name, false)
-        } else if mirrored_exprs(left_expr, right_ident, right_expr, left_ident) {
-            (left_expr, right_ident.name, true)
+        let (closure_body, closure_arg, reverse) =
+            if mirrored_exprs(left_expr, right_expr, &binding_map, BindingSource::Left) {
+                (left_expr, l_pat.span, false)
+            } else if mirrored_exprs(left_expr, right_expr, &binding_map, BindingSource::Right) {
+                (left_expr, r_pat.span, true)
+            } else {
+                return None;
+            };
+
+        let mut applicability = if reverse {
+            Applicability::MaybeIncorrect
         } else {
-            return None;
+            Applicability::MachineApplicable
         };
 
         if let ExprKind::Path(QPath::Resolved(
@@ -152,10 +275,39 @@ fn detect_lint<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, arg: &Expr<'_>) ->
                 ..
             },
         )) = left_expr.kind
-            && *left_name == left_ident
-            && implements_trait(cx, cx.typeck_results().expr_ty(left_expr), ord_trait, &[])
         {
-            return Some(LintTrigger::Sort);
+            if let PatKind::Binding(_, _, left_ident, _) = l_pat.kind
+                && *left_name == left_ident
+                && implements_trait(cx, cx.typeck_results().expr_ty(left_expr), ord_trait, &[])
+            {
+                return Some(LintTrigger::Sort);
+            }
+
+            let mut left_expr_ty = cx.typeck_results().expr_ty(left_expr);
+            let left_ident_n_refs = binding_map
+                .get(&BindingKey {
+                    ident: *left_name,
+                    source: BindingSource::Left,
+                })
+                .map_or(0, |value| value.n_refs);
+            // Peel off the outer-most ref which is introduced by the closure, if it is not already peeled
+            // by the pattern
+            if left_ident_n_refs == 0 {
+                (left_expr_ty, _) = peel_n_ty_refs(left_expr_ty, 1);
+            }
+            if !reverse && is_copy(cx, left_expr_ty) {
+                let mut closure_body =
+                    snippet_with_applicability(cx, closure_body.span, "_", &mut applicability).to_string();
+                if left_ident_n_refs == 0 {
+                    closure_body = format!("*{closure_body}");
+                }
+                return Some(LintTrigger::SortByKey(SortByKeyDetection {
+                    closure_arg,
+                    closure_body,
+                    reverse,
+                    applicability,
+                }));
+            }
         }
 
         let left_expr_ty = cx.typeck_results().expr_ty(left_expr);
@@ -163,10 +315,12 @@ fn detect_lint<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, arg: &Expr<'_>) ->
             // Don't lint if the closure is accessing non-Copy fields
             && (!expr_is_field_access(left_expr) || is_copy(cx, left_expr_ty))
         {
+            let closure_body = Sugg::hir_with_applicability(cx, closure_body, "_", &mut applicability).to_string();
             return Some(LintTrigger::SortByKey(SortByKeyDetection {
                 closure_arg,
                 closure_body,
                 reverse,
+                applicability,
             }));
         }
     }
@@ -212,22 +366,18 @@ pub(super) fn check<'tcx>(
                 expr.span,
                 format!("consider using `{method}`"),
                 |diag| {
-                    let mut app = if trigger.reverse {
-                        Applicability::MaybeIncorrect
-                    } else {
-                        Applicability::MachineApplicable
-                    };
+                    let mut app = trigger.applicability;
                     let recv = Sugg::hir_with_applicability(cx, recv, "(_)", &mut app);
-                    let closure_body = Sugg::hir_with_applicability(cx, trigger.closure_body, "_", &mut app);
                     let closure_body = if trigger.reverse {
-                        format!("{std_or_core}::cmp::Reverse({closure_body})")
+                        format!("{std_or_core}::cmp::Reverse({})", trigger.closure_body)
                     } else {
-                        closure_body.to_string()
+                        trigger.closure_body
                     };
+                    let closure_arg = snippet_with_applicability(cx, trigger.closure_arg, "_", &mut app);
                     diag.span_suggestion(
                         expr.span,
                         "try",
-                        format!("{recv}.{method}(|{}| {})", trigger.closure_arg, closure_body),
+                        format!("{recv}.{method}(|{closure_arg}| {closure_body})"),
                         app,
                     );
                 },
diff --git a/clippy_utils/src/ty/mod.rs b/clippy_utils/src/ty/mod.rs
index 979170d72006..a0fa5e714289 100644
--- a/clippy_utils/src/ty/mod.rs
+++ b/clippy_utils/src/ty/mod.rs
@@ -438,6 +438,21 @@ pub fn peel_and_count_ty_refs(mut ty: Ty<'_>) -> (Ty<'_>, usize, Option, n: usize) -> (Ty<'_>, Option) {
+    let mut mutbl = None;
+    for _ in 0..n {
+        if let ty::Ref(_, dest_ty, m) = ty.kind() {
+            ty = *dest_ty;
+            mutbl.replace(mutbl.map_or(*m, |mutbl: Mutability| mutbl.min(*m)));
+        } else {
+            break;
+        }
+    }
+    (ty, mutbl)
+}
+
 /// Checks whether `a` and `b` are same types having same `Const` generic args, but ignores
 /// lifetimes.
 ///
diff --git a/tests/ui/unnecessary_sort_by.fixed b/tests/ui/unnecessary_sort_by.fixed
index 372cf683f223..6870470e74c5 100644
--- a/tests/ui/unnecessary_sort_by.fixed
+++ b/tests/ui/unnecessary_sort_by.fixed
@@ -141,3 +141,35 @@ fn issue16405() {
     items.sort_by_key(|item1| item1.value.clone());
     //~^ unnecessary_sort_by
 }
+
+fn issue16348() {
+    let mut v: Vec<(i32, &str)> = vec![(1, "foo"), (2, "bar")];
+    v.sort_by_key(|(_, s1)| *s1);
+    //~^ unnecessary_sort_by
+
+    struct Foo {
+        bar: i32,
+    }
+    let mut v: Vec = vec![Foo { bar: 1 }, Foo { bar: 2 }];
+    v.sort_by_key(|Foo { bar: b1 }| *b1);
+    //~^ unnecessary_sort_by
+
+    struct Baz(i32);
+    let mut v: Vec = vec![Baz(1), Baz(2)];
+    v.sort_by_key(|Baz(b1)| *b1);
+    //~^ unnecessary_sort_by
+
+    v.sort_by_key(|&Baz(b1)| b1);
+    //~^ unnecessary_sort_by
+
+    let mut v: Vec<&i32> = vec![&1, &2];
+    v.sort_by_key(|&&b1| b1);
+    //~^ unnecessary_sort_by
+
+    let mut v: Vec<[i32; 2]> = vec![[1, 2], [3, 4]];
+    v.sort_by_key(|[a1, b1]| *a1);
+    //~^ unnecessary_sort_by
+
+    v.sort_by_key(|[a1, b1]| a1 - b1);
+    //~^ unnecessary_sort_by
+}
diff --git a/tests/ui/unnecessary_sort_by.rs b/tests/ui/unnecessary_sort_by.rs
index 7dd274623b35..d95306176817 100644
--- a/tests/ui/unnecessary_sort_by.rs
+++ b/tests/ui/unnecessary_sort_by.rs
@@ -141,3 +141,35 @@ fn issue16405() {
     items.sort_by(|item1, item2| item1.value.clone().cmp(&item2.value.clone()));
     //~^ unnecessary_sort_by
 }
+
+fn issue16348() {
+    let mut v: Vec<(i32, &str)> = vec![(1, "foo"), (2, "bar")];
+    v.sort_by(|(_, s1), (_, s2)| s1.cmp(s2));
+    //~^ unnecessary_sort_by
+
+    struct Foo {
+        bar: i32,
+    }
+    let mut v: Vec = vec![Foo { bar: 1 }, Foo { bar: 2 }];
+    v.sort_by(|Foo { bar: b1 }, Foo { bar: b2 }| b1.cmp(b2));
+    //~^ unnecessary_sort_by
+
+    struct Baz(i32);
+    let mut v: Vec = vec![Baz(1), Baz(2)];
+    v.sort_by(|Baz(b1), Baz(b2)| b1.cmp(b2));
+    //~^ unnecessary_sort_by
+
+    v.sort_by(|&Baz(b1), &Baz(b2)| b1.cmp(&b2));
+    //~^ unnecessary_sort_by
+
+    let mut v: Vec<&i32> = vec![&1, &2];
+    v.sort_by(|&&b1, &&b2| b1.cmp(&b2));
+    //~^ unnecessary_sort_by
+
+    let mut v: Vec<[i32; 2]> = vec![[1, 2], [3, 4]];
+    v.sort_by(|[a1, b1], [a2, b2]| a1.cmp(a2));
+    //~^ unnecessary_sort_by
+
+    v.sort_by(|[a1, b1], [a2, b2]| (a1 - b1).cmp(&(a2 - b2)));
+    //~^ unnecessary_sort_by
+}
diff --git a/tests/ui/unnecessary_sort_by.stderr b/tests/ui/unnecessary_sort_by.stderr
index b555245b0d01..b23d27c3729f 100644
--- a/tests/ui/unnecessary_sort_by.stderr
+++ b/tests/ui/unnecessary_sort_by.stderr
@@ -91,5 +91,47 @@ error: consider using `sort_by_key`
 LL |     items.sort_by(|item1, item2| item1.value.clone().cmp(&item2.value.clone()));
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `items.sort_by_key(|item1| item1.value.clone())`
 
-error: aborting due to 15 previous errors
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:147:5
+   |
+LL |     v.sort_by(|(_, s1), (_, s2)| s1.cmp(s2));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|(_, s1)| *s1)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:154:5
+   |
+LL |     v.sort_by(|Foo { bar: b1 }, Foo { bar: b2 }| b1.cmp(b2));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|Foo { bar: b1 }| *b1)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:159:5
+   |
+LL |     v.sort_by(|Baz(b1), Baz(b2)| b1.cmp(b2));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|Baz(b1)| *b1)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:162:5
+   |
+LL |     v.sort_by(|&Baz(b1), &Baz(b2)| b1.cmp(&b2));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|&Baz(b1)| b1)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:166:5
+   |
+LL |     v.sort_by(|&&b1, &&b2| b1.cmp(&b2));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|&&b1| b1)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:170:5
+   |
+LL |     v.sort_by(|[a1, b1], [a2, b2]| a1.cmp(a2));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|[a1, b1]| *a1)`
+
+error: consider using `sort_by_key`
+  --> tests/ui/unnecessary_sort_by.rs:173:5
+   |
+LL |     v.sort_by(|[a1, b1], [a2, b2]| (a1 - b1).cmp(&(a2 - b2)));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|[a1, b1]| a1 - b1)`
+
+error: aborting due to 22 previous errors
 

From 5937b8ba10651eef9e328c6d6f054ba09726ead2 Mon Sep 17 00:00:00 2001
From: linshuy2 
Date: Fri, 16 Jan 2026 20:33:32 +0000
Subject: [PATCH 0314/1843] Apply `unnecessary_sort_by` to Clippy itself

---
 clippy_utils/src/ty/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clippy_utils/src/ty/mod.rs b/clippy_utils/src/ty/mod.rs
index a0fa5e714289..747dc1f13ab5 100644
--- a/clippy_utils/src/ty/mod.rs
+++ b/clippy_utils/src/ty/mod.rs
@@ -830,7 +830,7 @@ impl AdtVariantInfo {
                     .enumerate()
                     .map(|(i, f)| (i, approx_ty_size(cx, f.ty(cx.tcx, subst))))
                     .collect::>();
-                fields_size.sort_by(|(_, a_size), (_, b_size)| a_size.cmp(b_size));
+                fields_size.sort_by_key(|(_, a_size)| *a_size);
 
                 Self {
                     ind: i,

From 196c098d24f54b661f76c2f9a3e1d74b8a5ce9b7 Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Fri, 16 Jan 2026 22:42:53 +0100
Subject: [PATCH 0315/1843] `unnecessary_sort_by`: reduce suggestion diffs

Now, only `call_span` is replaced, so the receiver is not a part of the
diff. This also removes the need to create a snippet for the receiver.
---
 clippy_lints/src/methods/mod.rs               |   4 +-
 .../src/methods/unnecessary_sort_by.rs        |  18 +-
 tests/ui/unnecessary_sort_by.stderr           | 175 +++++++++++++++---
 tests/ui/unnecessary_sort_by_no_std.stderr    |  15 +-
 4 files changed, 178 insertions(+), 34 deletions(-)

diff --git a/clippy_lints/src/methods/mod.rs b/clippy_lints/src/methods/mod.rs
index 659a704a1ecb..376e93aa7e7d 100644
--- a/clippy_lints/src/methods/mod.rs
+++ b/clippy_lints/src/methods/mod.rs
@@ -5532,10 +5532,10 @@ impl Methods {
                     stable_sort_primitive::check(cx, expr, recv);
                 },
                 (sym::sort_by, [arg]) => {
-                    unnecessary_sort_by::check(cx, expr, recv, arg, false);
+                    unnecessary_sort_by::check(cx, expr, call_span, arg, false);
                 },
                 (sym::sort_unstable_by, [arg]) => {
-                    unnecessary_sort_by::check(cx, expr, recv, arg, true);
+                    unnecessary_sort_by::check(cx, expr, call_span, arg, true);
                 },
                 (sym::split, [arg]) => {
                     str_split::check(cx, expr, recv, arg);
diff --git a/clippy_lints/src/methods/unnecessary_sort_by.rs b/clippy_lints/src/methods/unnecessary_sort_by.rs
index ac13fc0b6850..3f81a6ecd2f8 100644
--- a/clippy_lints/src/methods/unnecessary_sort_by.rs
+++ b/clippy_lints/src/methods/unnecessary_sort_by.rs
@@ -343,7 +343,7 @@ fn expr_is_field_access(expr: &Expr<'_>) -> bool {
 pub(super) fn check<'tcx>(
     cx: &LateContext<'tcx>,
     expr: &'tcx Expr<'_>,
-    recv: &'tcx Expr<'_>,
+    call_span: Span,
     arg: &'tcx Expr<'_>,
     is_unstable: bool,
 ) {
@@ -367,17 +367,16 @@ pub(super) fn check<'tcx>(
                 format!("consider using `{method}`"),
                 |diag| {
                     let mut app = trigger.applicability;
-                    let recv = Sugg::hir_with_applicability(cx, recv, "(_)", &mut app);
                     let closure_body = if trigger.reverse {
                         format!("{std_or_core}::cmp::Reverse({})", trigger.closure_body)
                     } else {
                         trigger.closure_body
                     };
                     let closure_arg = snippet_with_applicability(cx, trigger.closure_arg, "_", &mut app);
-                    diag.span_suggestion(
-                        expr.span,
+                    diag.span_suggestion_verbose(
+                        call_span,
                         "try",
-                        format!("{recv}.{method}(|{closure_arg}| {closure_body})"),
+                        format!("{method}(|{closure_arg}| {closure_body})"),
                         app,
                     );
                 },
@@ -391,9 +390,12 @@ pub(super) fn check<'tcx>(
                 expr.span,
                 format!("consider using `{method}`"),
                 |diag| {
-                    let mut app = Applicability::MachineApplicable;
-                    let recv = Sugg::hir_with_applicability(cx, recv, "(_)", &mut app);
-                    diag.span_suggestion(expr.span, "try", format!("{recv}.{method}()"), app);
+                    diag.span_suggestion_verbose(
+                        call_span,
+                        "try",
+                        format!("{method}()"),
+                        Applicability::MachineApplicable,
+                    );
                 },
             );
         },
diff --git a/tests/ui/unnecessary_sort_by.stderr b/tests/ui/unnecessary_sort_by.stderr
index b23d27c3729f..cc545d604ff3 100644
--- a/tests/ui/unnecessary_sort_by.stderr
+++ b/tests/ui/unnecessary_sort_by.stderr
@@ -2,136 +2,267 @@ error: consider using `sort`
   --> tests/ui/unnecessary_sort_by.rs:12:5
    |
 LL |     vec.sort_by(|a, b| a.cmp(b));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort()`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: `-D clippy::unnecessary-sort-by` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::unnecessary_sort_by)]`
+help: try
+   |
+LL -     vec.sort_by(|a, b| a.cmp(b));
+LL +     vec.sort();
+   |
 
 error: consider using `sort_unstable`
   --> tests/ui/unnecessary_sort_by.rs:14:5
    |
 LL |     vec.sort_unstable_by(|a, b| a.cmp(b));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_unstable()`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_unstable_by(|a, b| a.cmp(b));
+LL +     vec.sort_unstable();
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:16:5
    |
 LL |     vec.sort_by(|a, b| (a + 5).abs().cmp(&(b + 5).abs()));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_by_key(|a| (a + 5).abs())`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_by(|a, b| (a + 5).abs().cmp(&(b + 5).abs()));
+LL +     vec.sort_by_key(|a| (a + 5).abs());
+   |
 
 error: consider using `sort_unstable_by_key`
   --> tests/ui/unnecessary_sort_by.rs:18:5
    |
 LL |     vec.sort_unstable_by(|a, b| id(-a).cmp(&id(-b)));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_unstable_by_key(|a| id(-a))`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_unstable_by(|a, b| id(-a).cmp(&id(-b)));
+LL +     vec.sort_unstable_by_key(|a| id(-a));
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:22:5
    |
 LL |     vec.sort_by(|a, b| (b + 5).abs().cmp(&(a + 5).abs()));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_by_key(|b| std::cmp::Reverse((b + 5).abs()))`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_by(|a, b| (b + 5).abs().cmp(&(a + 5).abs()));
+LL +     vec.sort_by_key(|b| std::cmp::Reverse((b + 5).abs()));
+   |
 
 error: consider using `sort_unstable_by_key`
   --> tests/ui/unnecessary_sort_by.rs:24:5
    |
 LL |     vec.sort_unstable_by(|a, b| id(-b).cmp(&id(-a)));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_unstable_by_key(|b| std::cmp::Reverse(id(-b)))`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_unstable_by(|a, b| id(-b).cmp(&id(-a)));
+LL +     vec.sort_unstable_by_key(|b| std::cmp::Reverse(id(-b)));
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:35:5
    |
 LL |     vec.sort_by(|a, b| (***a).abs().cmp(&(***b).abs()));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_by_key(|a| (***a).abs())`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_by(|a, b| (***a).abs().cmp(&(***b).abs()));
+LL +     vec.sort_by_key(|a| (***a).abs());
+   |
 
 error: consider using `sort_unstable_by_key`
   --> tests/ui/unnecessary_sort_by.rs:37:5
    |
 LL |     vec.sort_unstable_by(|a, b| (***a).abs().cmp(&(***b).abs()));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_unstable_by_key(|a| (***a).abs())`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_unstable_by(|a, b| (***a).abs().cmp(&(***b).abs()));
+LL +     vec.sort_unstable_by_key(|a| (***a).abs());
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:97:9
    |
 LL |         args.sort_by(|a, b| a.name().cmp(&b.name()));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `args.sort_by_key(|a| a.name())`
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -         args.sort_by(|a, b| a.name().cmp(&b.name()));
+LL +         args.sort_by_key(|a| a.name());
+   |
 
 error: consider using `sort_unstable_by_key`
   --> tests/ui/unnecessary_sort_by.rs:99:9
    |
 LL |         args.sort_unstable_by(|a, b| a.name().cmp(&b.name()));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `args.sort_unstable_by_key(|a| a.name())`
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -         args.sort_unstable_by(|a, b| a.name().cmp(&b.name()));
+LL +         args.sort_unstable_by_key(|a| a.name());
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:102:9
    |
 LL |         args.sort_by(|a, b| b.name().cmp(&a.name()));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `args.sort_by_key(|b| std::cmp::Reverse(b.name()))`
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -         args.sort_by(|a, b| b.name().cmp(&a.name()));
+LL +         args.sort_by_key(|b| std::cmp::Reverse(b.name()));
+   |
 
 error: consider using `sort_unstable_by_key`
   --> tests/ui/unnecessary_sort_by.rs:104:9
    |
 LL |         args.sort_unstable_by(|a, b| b.name().cmp(&a.name()));
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `args.sort_unstable_by_key(|b| std::cmp::Reverse(b.name()))`
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -         args.sort_unstable_by(|a, b| b.name().cmp(&a.name()));
+LL +         args.sort_unstable_by_key(|b| std::cmp::Reverse(b.name()));
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:118:5
    |
 LL |     v.sort_by(|a, b| a.0.cmp(&b.0));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|a| a.0)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|a, b| a.0.cmp(&b.0));
+LL +     v.sort_by_key(|a| a.0);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:136:5
    |
 LL |     items.sort_by(|item1, item2| item1.key.cmp(&item2.key));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `items.sort_by_key(|item1| item1.key)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     items.sort_by(|item1, item2| item1.key.cmp(&item2.key));
+LL +     items.sort_by_key(|item1| item1.key);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:141:5
    |
 LL |     items.sort_by(|item1, item2| item1.value.clone().cmp(&item2.value.clone()));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `items.sort_by_key(|item1| item1.value.clone())`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     items.sort_by(|item1, item2| item1.value.clone().cmp(&item2.value.clone()));
+LL +     items.sort_by_key(|item1| item1.value.clone());
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:147:5
    |
 LL |     v.sort_by(|(_, s1), (_, s2)| s1.cmp(s2));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|(_, s1)| *s1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|(_, s1), (_, s2)| s1.cmp(s2));
+LL +     v.sort_by_key(|(_, s1)| *s1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:154:5
    |
 LL |     v.sort_by(|Foo { bar: b1 }, Foo { bar: b2 }| b1.cmp(b2));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|Foo { bar: b1 }| *b1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|Foo { bar: b1 }, Foo { bar: b2 }| b1.cmp(b2));
+LL +     v.sort_by_key(|Foo { bar: b1 }| *b1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:159:5
    |
 LL |     v.sort_by(|Baz(b1), Baz(b2)| b1.cmp(b2));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|Baz(b1)| *b1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|Baz(b1), Baz(b2)| b1.cmp(b2));
+LL +     v.sort_by_key(|Baz(b1)| *b1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:162:5
    |
 LL |     v.sort_by(|&Baz(b1), &Baz(b2)| b1.cmp(&b2));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|&Baz(b1)| b1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|&Baz(b1), &Baz(b2)| b1.cmp(&b2));
+LL +     v.sort_by_key(|&Baz(b1)| b1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:166:5
    |
 LL |     v.sort_by(|&&b1, &&b2| b1.cmp(&b2));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|&&b1| b1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|&&b1, &&b2| b1.cmp(&b2));
+LL +     v.sort_by_key(|&&b1| b1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:170:5
    |
 LL |     v.sort_by(|[a1, b1], [a2, b2]| a1.cmp(a2));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|[a1, b1]| *a1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|[a1, b1], [a2, b2]| a1.cmp(a2));
+LL +     v.sort_by_key(|[a1, b1]| *a1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by.rs:173:5
    |
 LL |     v.sort_by(|[a1, b1], [a2, b2]| (a1 - b1).cmp(&(a2 - b2)));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `v.sort_by_key(|[a1, b1]| a1 - b1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     v.sort_by(|[a1, b1], [a2, b2]| (a1 - b1).cmp(&(a2 - b2)));
+LL +     v.sort_by_key(|[a1, b1]| a1 - b1);
+   |
 
 error: aborting due to 22 previous errors
 
diff --git a/tests/ui/unnecessary_sort_by_no_std.stderr b/tests/ui/unnecessary_sort_by_no_std.stderr
index de3ef4123514..b4dd6a6dbdc5 100644
--- a/tests/ui/unnecessary_sort_by_no_std.stderr
+++ b/tests/ui/unnecessary_sort_by_no_std.stderr
@@ -2,16 +2,27 @@ error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by_no_std.rs:10:5
    |
 LL |     vec.sort_by(|a, b| (a + 1).cmp(&(b + 1)));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_by_key(|a| a + 1)`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: `-D clippy::unnecessary-sort-by` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::unnecessary_sort_by)]`
+help: try
+   |
+LL -     vec.sort_by(|a, b| (a + 1).cmp(&(b + 1)));
+LL +     vec.sort_by_key(|a| a + 1);
+   |
 
 error: consider using `sort_by_key`
   --> tests/ui/unnecessary_sort_by_no_std.rs:19:5
    |
 LL |     vec.sort_by(|a, b| (b + 1).cmp(&(a + 1)));
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `vec.sort_by_key(|b| core::cmp::Reverse(b + 1))`
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: try
+   |
+LL -     vec.sort_by(|a, b| (b + 1).cmp(&(a + 1)));
+LL +     vec.sort_by_key(|b| core::cmp::Reverse(b + 1));
+   |
 
 error: aborting due to 2 previous errors
 

From ccdda582a84a2985a90374cb8720b0084b4c9408 Mon Sep 17 00:00:00 2001
From: Ada Alakbarova 
Date: Fri, 16 Jan 2026 22:53:37 +0100
Subject: [PATCH 0316/1843] clean-up

---
 clippy_lints/src/methods/str_split.rs |  6 ++++--
 tests/ui/str_split.fixed              |  2 --
 tests/ui/str_split.rs                 |  2 --
 tests/ui/str_split.stderr             | 20 ++++++++++----------
 4 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/clippy_lints/src/methods/str_split.rs b/clippy_lints/src/methods/str_split.rs
index 479064a0671e..9644b2c774b5 100644
--- a/clippy_lints/src/methods/str_split.rs
+++ b/clippy_lints/src/methods/str_split.rs
@@ -19,8 +19,10 @@ pub(super) fn check<'a>(cx: &LateContext<'a>, expr: &'_ Expr<'_>, split_recv: &'
         && cx.typeck_results().expr_ty_adjusted(trim_recv).peel_refs().is_str()
         && !is_const_evaluatable(cx, trim_recv)
         && let ExprKind::Lit(split_lit) = split_arg.kind
-        && (matches!(split_lit.node, LitKind::Char('\n'))
-            || matches!(split_lit.node, LitKind::Str(sym::LF | sym::CRLF, _)))
+        && matches!(
+            split_lit.node,
+            LitKind::Char('\n') | LitKind::Str(sym::LF | sym::CRLF, _)
+        )
     {
         let mut app = Applicability::MaybeIncorrect;
         span_lint_and_sugg(
diff --git a/tests/ui/str_split.fixed b/tests/ui/str_split.fixed
index 6aca5051c570..c93d4d4f818d 100644
--- a/tests/ui/str_split.fixed
+++ b/tests/ui/str_split.fixed
@@ -1,7 +1,5 @@
 #![warn(clippy::str_split_at_newline)]
-#![allow(clippy::needless_lifetimes)]
 
-use core::str::Split;
 use std::ops::Deref;
 
 struct NotStr<'a> {
diff --git a/tests/ui/str_split.rs b/tests/ui/str_split.rs
index 11e9862da14b..5792ce04ec33 100644
--- a/tests/ui/str_split.rs
+++ b/tests/ui/str_split.rs
@@ -1,7 +1,5 @@
 #![warn(clippy::str_split_at_newline)]
-#![allow(clippy::needless_lifetimes)]
 
-use core::str::Split;
 use std::ops::Deref;
 
 struct NotStr<'a> {
diff --git a/tests/ui/str_split.stderr b/tests/ui/str_split.stderr
index c4eca81004c5..40202c8ac47a 100644
--- a/tests/ui/str_split.stderr
+++ b/tests/ui/str_split.stderr
@@ -1,5 +1,5 @@
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:60:13
+  --> tests/ui/str_split.rs:58:13
    |
 LL |     let _ = s1.trim().split('\n');
    |             ^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s1.lines()`
@@ -8,55 +8,55 @@ LL |     let _ = s1.trim().split('\n');
    = help: to override `-D warnings` add `#[allow(clippy::str_split_at_newline)]`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:63:13
+  --> tests/ui/str_split.rs:61:13
    |
 LL |     let _ = s1.trim().split("\n");
    |             ^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s1.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:65:13
+  --> tests/ui/str_split.rs:63:13
    |
 LL |     let _ = s1.trim().split("\r\n");
    |             ^^^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s1.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:69:13
+  --> tests/ui/str_split.rs:67:13
    |
 LL |     let _ = s2.trim().split('\n');
    |             ^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s2.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:72:13
+  --> tests/ui/str_split.rs:70:13
    |
 LL |     let _ = s2.trim().split("\n");
    |             ^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s2.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:74:13
+  --> tests/ui/str_split.rs:72:13
    |
 LL |     let _ = s2.trim().split("\r\n");
    |             ^^^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s2.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:79:13
+  --> tests/ui/str_split.rs:77:13
    |
 LL |     let _ = s3.trim().split('\n');
    |             ^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s3.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:82:13
+  --> tests/ui/str_split.rs:80:13
    |
 LL |     let _ = s3.trim().split("\n");
    |             ^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s3.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:84:13
+  --> tests/ui/str_split.rs:82:13
    |
 LL |     let _ = s3.trim().split("\r\n");
    |             ^^^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `s3.lines()`
 
 error: using `str.trim().split()` with hard-coded newlines
-  --> tests/ui/str_split.rs:88:13
+  --> tests/ui/str_split.rs:86:13
    |
 LL |     let _ = make_str!(s1).trim().split('\n');
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use `str.lines()` instead: `make_str!(s1).lines()`

From bcde8c10707c521480d1beb1bf4dd0fc72cc3902 Mon Sep 17 00:00:00 2001
From: Samuel Tardieu 
Date: Sat, 17 Jan 2026 00:00:06 +0100
Subject: [PATCH 0317/1843] Do not consider binary operators as commutative by
 default

Only `==` (and thus `!=`) are supposed to be commutative according to
Rust's documentation. Do not make assumptions about other operators
whose meaning may depend on the types on which they apply. However,
special-case operators known to be commutative for primitive types
such as addition or multiplication.
---
 clippy_utils/src/hir_utils.rs     |  31 ++--
 tests/ui/if_same_then_else.rs     |  84 +++++++++--
 tests/ui/if_same_then_else.stderr | 230 ++++++++++++++++++++++++++++--
 3 files changed, 309 insertions(+), 36 deletions(-)

diff --git a/clippy_utils/src/hir_utils.rs b/clippy_utils/src/hir_utils.rs
index bdc1550d69b7..79994d8a3c23 100644
--- a/clippy_utils/src/hir_utils.rs
+++ b/clippy_utils/src/hir_utils.rs
@@ -505,7 +505,7 @@ impl HirEqInterExpr<'_, '_, '_> {
             (ExprKind::Block(l, _), ExprKind::Block(r, _)) => self.eq_block(l, r),
             (ExprKind::Binary(l_op, ll, lr), ExprKind::Binary(r_op, rl, rr)) => {
                 l_op.node == r_op.node && self.eq_expr(ll, rl) && self.eq_expr(lr, rr)
-                    || swap_binop(l_op.node, ll, lr).is_some_and(|(l_op, ll, lr)| {
+                    || swap_binop(self.inner.cx, l_op.node, ll, lr).is_some_and(|(l_op, ll, lr)| {
                         l_op == r_op.node && self.eq_expr(ll, rl) && self.eq_expr(lr, rr)
                     })
             },
@@ -939,26 +939,35 @@ fn reduce_exprkind<'hir>(cx: &LateContext<'_>, kind: &'hir ExprKind<'hir>) -> &'
 }
 
 fn swap_binop<'a>(
+    cx: &LateContext<'_>,
     binop: BinOpKind,
     lhs: &'a Expr<'a>,
     rhs: &'a Expr<'a>,
 ) -> Option<(BinOpKind, &'a Expr<'a>, &'a Expr<'a>)> {
     match binop {
-        BinOpKind::Add | BinOpKind::Eq | BinOpKind::Ne | BinOpKind::BitAnd | BinOpKind::BitXor | BinOpKind::BitOr => {
-            Some((binop, rhs, lhs))
-        },
+        // `==` and `!=`, are commutative
+        BinOpKind::Eq | BinOpKind::Ne => Some((binop, rhs, lhs)),
+        // Comparisons can be reversed
         BinOpKind::Lt => Some((BinOpKind::Gt, rhs, lhs)),
         BinOpKind::Le => Some((BinOpKind::Ge, rhs, lhs)),
         BinOpKind::Ge => Some((BinOpKind::Le, rhs, lhs)),
         BinOpKind::Gt => Some((BinOpKind::Lt, rhs, lhs)),
-        BinOpKind::Mul // Not always commutative, e.g. with matrices. See issue #5698
-        | BinOpKind::Shl
-        | BinOpKind::Shr
-        | BinOpKind::Rem
-        | BinOpKind::Sub
-        | BinOpKind::Div
+        // Non-commutative operators
+        BinOpKind::Shl | BinOpKind::Shr | BinOpKind::Rem | BinOpKind::Sub | BinOpKind::Div => None,
+        // We know that those operators are commutative for primitive types,
+        // and we don't assume anything for other types
+        BinOpKind::Mul
+        | BinOpKind::Add
         | BinOpKind::And
-        | BinOpKind::Or => None,
+        | BinOpKind::Or
+        | BinOpKind::BitAnd
+        | BinOpKind::BitXor
+        | BinOpKind::BitOr => cx
+            .typeck_results()
+            .expr_ty_adjusted(lhs)
+            .peel_refs()
+            .is_primitive()
+            .then_some((binop, rhs, lhs)),
     }
 }
 
diff --git a/tests/ui/if_same_then_else.rs b/tests/ui/if_same_then_else.rs
index 6d2e63e7299a..6e0e2c5ea720 100644
--- a/tests/ui/if_same_then_else.rs
+++ b/tests/ui/if_same_then_else.rs
@@ -11,6 +11,8 @@
     unreachable_code
 )]
 
+use std::ops::*;
+
 struct Foo {
     bar: u8,
 }
@@ -133,18 +135,6 @@ fn func() {
 
 fn f(val: &[u8]) {}
 
-mod issue_5698 {
-    fn mul_not_always_commutative(x: i32, y: i32) -> i32 {
-        if x == 42 {
-            x * y
-        } else if x == 21 {
-            y * x
-        } else {
-            0
-        }
-    }
-}
-
 mod issue_8836 {
     fn do_not_lint() {
         if true {
@@ -245,3 +235,73 @@ mod issue_11213 {
 }
 
 fn main() {}
+
+fn issue16416(x: bool, a: T, b: T)
+where
+    T: Add + Sub + Mul + Div + Rem + BitAnd + BitOr + BitXor + PartialEq + Eq + PartialOrd + Ord + Shr + Shl + Copy,
+{
+    // Non-guaranteed-commutative operators
+    _ = if x { a * b } else { b * a };
+    _ = if x { a + b } else { b + a };
+    _ = if x { a - b } else { b - a };
+    _ = if x { a / b } else { b / a };
+    _ = if x { a % b } else { b % a };
+    _ = if x { a << b } else { b << a };
+    _ = if x { a >> b } else { b >> a };
+    _ = if x { a & b } else { b & a };
+    _ = if x { a ^ b } else { b ^ a };
+    _ = if x { a | b } else { b | a };
+
+    // Guaranteed commutative operators
+    //~v if_same_then_else
+    _ = if x { a == b } else { b == a };
+    //~v if_same_then_else
+    _ = if x { a != b } else { b != a };
+
+    // Symetric operators
+    //~v if_same_then_else
+    _ = if x { a < b } else { b > a };
+    //~v if_same_then_else
+    _ = if x { a <= b } else { b >= a };
+    //~v if_same_then_else
+    _ = if x { a > b } else { b < a };
+    //~v if_same_then_else
+    _ = if x { a >= b } else { b <= a };
+}
+
+fn issue16416_prim(x: bool, a: u32, b: u32) {
+    // Non-commutative operators
+    _ = if x { a - b } else { b - a };
+    _ = if x { a / b } else { b / a };
+    _ = if x { a % b } else { b % a };
+    _ = if x { a << b } else { b << a };
+    _ = if x { a >> b } else { b >> a };
+
+    // Commutative operators on primitive types
+    //~v if_same_then_else
+    _ = if x { a * b } else { b * a };
+    //~v if_same_then_else
+    _ = if x { a + b } else { b + a };
+    //~v if_same_then_else
+    _ = if x { a & b } else { b & a };
+    //~v if_same_then_else
+    _ = if x { a ^ b } else { b ^ a };
+    //~v if_same_then_else
+    _ = if x { a | b } else { b | a };
+
+    // Always commutative operators
+    //~v if_same_then_else
+    _ = if x { a == b } else { b == a };
+    //~v if_same_then_else
+    _ = if x { a != b } else { b != a };
+
+    // Symetric operators
+    //~v if_same_then_else
+    _ = if x { a < b } else { b > a };
+    //~v if_same_then_else
+    _ = if x { a <= b } else { b >= a };
+    //~v if_same_then_else
+    _ = if x { a > b } else { b < a };
+    //~v if_same_then_else
+    _ = if x { a >= b } else { b <= a };
+}
diff --git a/tests/ui/if_same_then_else.stderr b/tests/ui/if_same_then_else.stderr
index b76da3fb1cb5..57396a566941 100644
--- a/tests/ui/if_same_then_else.stderr
+++ b/tests/ui/if_same_then_else.stderr
@@ -1,5 +1,5 @@
 error: this `if` has identical blocks
-  --> tests/ui/if_same_then_else.rs:23:13
+  --> tests/ui/if_same_then_else.rs:25:13
    |
 LL |       if true {
    |  _____________^
@@ -12,7 +12,7 @@ LL | |     } else {
    | |_____^
    |
 note: same as this
-  --> tests/ui/if_same_then_else.rs:31:12
+  --> tests/ui/if_same_then_else.rs:33:12
    |
 LL |       } else {
    |  ____________^
@@ -27,43 +27,43 @@ LL | |     }
    = help: to override `-D warnings` add `#[allow(clippy::if_same_then_else)]`
 
 error: this `if` has identical blocks
-  --> tests/ui/if_same_then_else.rs:67:21
+  --> tests/ui/if_same_then_else.rs:69:21
    |
 LL |     let _ = if true { 0.0 } else { 0.0 };
    |                     ^^^^^^^
    |
 note: same as this
-  --> tests/ui/if_same_then_else.rs:67:34
+  --> tests/ui/if_same_then_else.rs:69:34
    |
 LL |     let _ = if true { 0.0 } else { 0.0 };
    |                                  ^^^^^^^
 
 error: this `if` has identical blocks
-  --> tests/ui/if_same_then_else.rs:70:21
+  --> tests/ui/if_same_then_else.rs:72:21
    |
 LL |     let _ = if true { -0.0 } else { -0.0 };
    |                     ^^^^^^^^
    |
 note: same as this
-  --> tests/ui/if_same_then_else.rs:70:35
+  --> tests/ui/if_same_then_else.rs:72:35
    |
 LL |     let _ = if true { -0.0 } else { -0.0 };
    |                                   ^^^^^^^^
 
 error: this `if` has identical blocks
-  --> tests/ui/if_same_then_else.rs:82:21
+  --> tests/ui/if_same_then_else.rs:84:21
    |
 LL |     let _ = if true { 42 } else { 42 };
    |                     ^^^^^^
    |
 note: same as this
-  --> tests/ui/if_same_then_else.rs:82:33
+  --> tests/ui/if_same_then_else.rs:84:33
    |
 LL |     let _ = if true { 42 } else { 42 };
    |                                 ^^^^^^
 
 error: this `if` has identical blocks
-  --> tests/ui/if_same_then_else.rs:85:13
+  --> tests/ui/if_same_then_else.rs:87:13
    |
 LL |       if true {
    |  _____________^
@@ -76,7 +76,7 @@ LL | |     } else {
    | |_____^
    |
 note: same as this
-  --> tests/ui/if_same_then_else.rs:92:12
+  --> tests/ui/if_same_then_else.rs:94:12
    |
 LL |       } else {
    |  ____________^
@@ -89,7 +89,7 @@ LL | |     }
    | |_____^
 
 error: this `if` has identical blocks
-  --> tests/ui/if_same_then_else.rs:238:14
+  --> tests/ui/if_same_then_else.rs:228:14
    |
 LL |           if x {
    |  ______________^
@@ -98,7 +98,7 @@ LL | |         } else {
    | |_________^
    |
 note: same as this
-  --> tests/ui/if_same_then_else.rs:240:16
+  --> tests/ui/if_same_then_else.rs:230:16
    |
 LL |           } else {
    |  ________________^
@@ -106,5 +106,209 @@ LL | |             0_u8.is_power_of_two()
 LL | |         }
    | |_________^
 
-error: aborting due to 6 previous errors
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:257:14
+   |
+LL |     _ = if x { a == b } else { b == a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:257:30
+   |
+LL |     _ = if x { a == b } else { b == a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:259:14
+   |
+LL |     _ = if x { a != b } else { b != a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:259:30
+   |
+LL |     _ = if x { a != b } else { b != a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:263:14
+   |
+LL |     _ = if x { a < b } else { b > a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:263:29
+   |
+LL |     _ = if x { a < b } else { b > a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:265:14
+   |
+LL |     _ = if x { a <= b } else { b >= a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:265:30
+   |
+LL |     _ = if x { a <= b } else { b >= a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:267:14
+   |
+LL |     _ = if x { a > b } else { b < a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:267:29
+   |
+LL |     _ = if x { a > b } else { b < a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:269:14
+   |
+LL |     _ = if x { a >= b } else { b <= a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:269:30
+   |
+LL |     _ = if x { a >= b } else { b <= a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:282:14
+   |
+LL |     _ = if x { a * b } else { b * a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:282:29
+   |
+LL |     _ = if x { a * b } else { b * a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:284:14
+   |
+LL |     _ = if x { a + b } else { b + a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:284:29
+   |
+LL |     _ = if x { a + b } else { b + a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:286:14
+   |
+LL |     _ = if x { a & b } else { b & a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:286:29
+   |
+LL |     _ = if x { a & b } else { b & a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:288:14
+   |
+LL |     _ = if x { a ^ b } else { b ^ a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:288:29
+   |
+LL |     _ = if x { a ^ b } else { b ^ a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:290:14
+   |
+LL |     _ = if x { a | b } else { b | a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:290:29
+   |
+LL |     _ = if x { a | b } else { b | a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:294:14
+   |
+LL |     _ = if x { a == b } else { b == a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:294:30
+   |
+LL |     _ = if x { a == b } else { b == a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:296:14
+   |
+LL |     _ = if x { a != b } else { b != a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:296:30
+   |
+LL |     _ = if x { a != b } else { b != a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:300:14
+   |
+LL |     _ = if x { a < b } else { b > a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:300:29
+   |
+LL |     _ = if x { a < b } else { b > a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:302:14
+   |
+LL |     _ = if x { a <= b } else { b >= a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:302:30
+   |
+LL |     _ = if x { a <= b } else { b >= a };
+   |                              ^^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:304:14
+   |
+LL |     _ = if x { a > b } else { b < a };
+   |              ^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:304:29
+   |
+LL |     _ = if x { a > b } else { b < a };
+   |                             ^^^^^^^^^
+
+error: this `if` has identical blocks
+  --> tests/ui/if_same_then_else.rs:306:14
+   |
+LL |     _ = if x { a >= b } else { b <= a };
+   |              ^^^^^^^^^^
+   |
+note: same as this
+  --> tests/ui/if_same_then_else.rs:306:30
+   |
+LL |     _ = if x { a >= b } else { b <= a };
+   |                              ^^^^^^^^^^
+
+error: aborting due to 23 previous errors
 

From dd61de95e47de79c4676c528f275cd07654fe377 Mon Sep 17 00:00:00 2001
From: Samuel Tardieu 
Date: Sat, 17 Jan 2026 00:41:20 +0100
Subject: [PATCH 0318/1843] Remove empty stderr file from tests

---
 tests/ui/manual_take_nocore.stderr | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 tests/ui/manual_take_nocore.stderr

diff --git a/tests/ui/manual_take_nocore.stderr b/tests/ui/manual_take_nocore.stderr
deleted file mode 100644
index e69de29bb2d1..000000000000

From 79f12eb1ed02e763dafe13938996f69e6b0b5f7a Mon Sep 17 00:00:00 2001
From: dianne 
Date: Fri, 16 Jan 2026 15:42:25 -0800
Subject: [PATCH 0319/1843] remove some confusing mutation

`oprnd_t` was used both for the type of the operand of a unary operator
and for the type of the operator expression as a whole. Now it's only
used for the operand's type.
---
 compiler/rustc_hir_typeck/src/expr.rs | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 9f3ff0b2d03c..1d7faf1345cf 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -434,14 +434,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             hir::UnOp::Not | hir::UnOp::Neg => expected,
             hir::UnOp::Deref => NoExpectation,
         };
-        let mut oprnd_t = self.check_expr_with_expectation(oprnd, expected_inner);
+        let oprnd_t = self.check_expr_with_expectation(oprnd, expected_inner);
 
-        if !oprnd_t.references_error() {
-            oprnd_t = self.structurally_resolve_type(expr.span, oprnd_t);
+        if let Err(guar) = oprnd_t.error_reported() {
+            Ty::new_error(tcx, guar)
+        } else {
+            let oprnd_t = self.structurally_resolve_type(expr.span, oprnd_t);
             match unop {
                 hir::UnOp::Deref => {
                     if let Some(ty) = self.lookup_derefing(expr, oprnd, oprnd_t) {
-                        oprnd_t = ty;
+                        ty
                     } else {
                         let mut err =
                             self.dcx().create_err(CantDereference { span: expr.span, ty: oprnd_t });
@@ -451,26 +453,25 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         {
                             err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
                         }
-                        oprnd_t = Ty::new_error(tcx, err.emit());
+                        Ty::new_error(tcx, err.emit())
                     }
                 }
                 hir::UnOp::Not => {
                     let result = self.check_user_unop(expr, oprnd_t, unop, expected_inner);
                     // If it's builtin, we can reuse the type, this helps inference.
-                    if !(oprnd_t.is_integral() || *oprnd_t.kind() == ty::Bool) {
-                        oprnd_t = result;
+                    if oprnd_t.is_integral() || *oprnd_t.kind() == ty::Bool {
+                        oprnd_t
+                    } else {
+                        result
                     }
                 }
                 hir::UnOp::Neg => {
                     let result = self.check_user_unop(expr, oprnd_t, unop, expected_inner);
                     // If it's builtin, we can reuse the type, this helps inference.
-                    if !oprnd_t.is_numeric() {
-                        oprnd_t = result;
-                    }
+                    if oprnd_t.is_numeric() { oprnd_t } else { result }
                 }
             }
         }
-        oprnd_t
     }
 
     fn check_expr_addr_of(

From 91bbb692d5a32faae9a8fb3f212f518bd98ff3cd Mon Sep 17 00:00:00 2001
From: dianne 
Date: Fri, 16 Jan 2026 15:51:29 -0800
Subject: [PATCH 0320/1843] re-style `check_expr_unop`

---
 compiler/rustc_hir_typeck/src/expr.rs | 54 +++++++++++----------------
 1 file changed, 22 insertions(+), 32 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 1d7faf1345cf..885af3b909b8 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -437,39 +437,29 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let oprnd_t = self.check_expr_with_expectation(oprnd, expected_inner);
 
         if let Err(guar) = oprnd_t.error_reported() {
-            Ty::new_error(tcx, guar)
-        } else {
-            let oprnd_t = self.structurally_resolve_type(expr.span, oprnd_t);
-            match unop {
-                hir::UnOp::Deref => {
-                    if let Some(ty) = self.lookup_derefing(expr, oprnd, oprnd_t) {
-                        ty
-                    } else {
-                        let mut err =
-                            self.dcx().create_err(CantDereference { span: expr.span, ty: oprnd_t });
-                        let sp = tcx.sess.source_map().start_point(expr.span).with_parent(None);
-                        if let Some(sp) =
-                            tcx.sess.psess.ambiguous_block_expr_parse.borrow().get(&sp)
-                        {
-                            err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
-                        }
-                        Ty::new_error(tcx, err.emit())
-                    }
-                }
-                hir::UnOp::Not => {
-                    let result = self.check_user_unop(expr, oprnd_t, unop, expected_inner);
-                    // If it's builtin, we can reuse the type, this helps inference.
-                    if oprnd_t.is_integral() || *oprnd_t.kind() == ty::Bool {
-                        oprnd_t
-                    } else {
-                        result
-                    }
-                }
-                hir::UnOp::Neg => {
-                    let result = self.check_user_unop(expr, oprnd_t, unop, expected_inner);
-                    // If it's builtin, we can reuse the type, this helps inference.
-                    if oprnd_t.is_numeric() { oprnd_t } else { result }
+            return Ty::new_error(tcx, guar);
+        }
+
+        let oprnd_t = self.structurally_resolve_type(expr.span, oprnd_t);
+        match unop {
+            hir::UnOp::Deref => self.lookup_derefing(expr, oprnd, oprnd_t).unwrap_or_else(|| {
+                let mut err =
+                    self.dcx().create_err(CantDereference { span: expr.span, ty: oprnd_t });
+                let sp = tcx.sess.source_map().start_point(expr.span).with_parent(None);
+                if let Some(sp) = tcx.sess.psess.ambiguous_block_expr_parse.borrow().get(&sp) {
+                    err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
                 }
+                Ty::new_error(tcx, err.emit())
+            }),
+            hir::UnOp::Not => {
+                let result = self.check_user_unop(expr, oprnd_t, unop, expected_inner);
+                // If it's builtin, we can reuse the type, this helps inference.
+                if oprnd_t.is_integral() || *oprnd_t.kind() == ty::Bool { oprnd_t } else { result }
+            }
+            hir::UnOp::Neg => {
+                let result = self.check_user_unop(expr, oprnd_t, unop, expected_inner);
+                // If it's builtin, we can reuse the type, this helps inference.
+                if oprnd_t.is_numeric() { oprnd_t } else { result }
             }
         }
     }

From 696f616b78940380e34f06f6d094c17ab7c647a4 Mon Sep 17 00:00:00 2001
From: xtqqczze <45661989+xtqqczze@users.noreply.github.com>
Date: Fri, 16 Jan 2026 23:59:16 +0000
Subject: [PATCH 0321/1843] Remove known problems from `comparison_chain`

This reverts commit e42ba4829c02e8308ae142b5a2fd5efb6ccf0a7b, reversing
changes made to d75bc868caee77f1b29763f8ba5a00494ed52f6b.
---
 clippy_lints/src/comparison_chain.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/clippy_lints/src/comparison_chain.rs b/clippy_lints/src/comparison_chain.rs
index 238ebd4a444c..a2ddf3dad7a2 100644
--- a/clippy_lints/src/comparison_chain.rs
+++ b/clippy_lints/src/comparison_chain.rs
@@ -17,10 +17,6 @@ declare_clippy_lint! {
     /// `if` is not guaranteed to be exhaustive and conditionals can get
     /// repetitive
     ///
-    /// ### Known problems
-    /// The match statement may be slower due to the compiler
-    /// not inlining the call to cmp. See issue [#5354](https://github.com/rust-lang/rust-clippy/issues/5354)
-    ///
     /// ### Example
     /// ```rust,ignore
     /// # fn a() {}

From 27b02796609e43103ce06a5c38cfc82e029c53c7 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Sat, 17 Jan 2026 01:53:08 +0100
Subject: [PATCH 0322/1843] Change field `bit_width: usize` to `bits: u32` in
 type info

---
 .../src/const_eval/type_info.rs               |  8 ++++----
 compiler/rustc_span/src/symbol.rs             |  2 +-
 library/core/src/mem/type_info.rs             |  4 ++--
 library/coretests/tests/mem/type_info.rs      | 14 ++++++-------
 tests/ui/reflection/dump.bit32.run.stdout     | 20 +++++++++----------
 tests/ui/reflection/dump.bit64.run.stdout     | 20 +++++++++----------
 6 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 5d37db06d76a..195714b1c0e6 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -249,8 +249,8 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         {
             let field_place = self.project_field(&place, field_idx)?;
             match field.name {
-                sym::bit_width => self.write_scalar(
-                    ScalarInt::try_from_target_usize(bit_width, self.tcx.tcx).unwrap(),
+                sym::bits => self.write_scalar(
+                    Scalar::from_u32(bit_width.try_into().expect("bit_width overflowed")),
                     &field_place,
                 )?,
                 sym::signed => self.write_scalar(Scalar::from_bool(signed), &field_place)?,
@@ -270,8 +270,8 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         {
             let field_place = self.project_field(&place, field_idx)?;
             match field.name {
-                sym::bit_width => self.write_scalar(
-                    ScalarInt::try_from_target_usize(bit_width, self.tcx.tcx).unwrap(),
+                sym::bits => self.write_scalar(
+                    Scalar::from_u32(bit_width.try_into().expect("bit_width overflowed")),
                     &field_place,
                 )?,
                 other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 51920db8cd79..2bd9ee6a6123 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -589,12 +589,12 @@ symbols! {
         binaryheap_iter,
         bind_by_move_pattern_guards,
         bindings_after_at,
-        bit_width,
         bitand,
         bitand_assign,
         bitor,
         bitor_assign,
         bitreverse,
+        bits,
         bitxor,
         bitxor_assign,
         black_box,
diff --git a/library/core/src/mem/type_info.rs b/library/core/src/mem/type_info.rs
index 2e3bdb45ce05..5a105573db77 100644
--- a/library/core/src/mem/type_info.rs
+++ b/library/core/src/mem/type_info.rs
@@ -112,7 +112,7 @@ pub struct Char {
 #[unstable(feature = "type_info", issue = "146922")]
 pub struct Int {
     /// The bit width of the signed integer type.
-    pub bit_width: usize,
+    pub bits: u32,
     /// Whether the integer type is signed.
     pub signed: bool,
 }
@@ -123,7 +123,7 @@ pub struct Int {
 #[unstable(feature = "type_info", issue = "146922")]
 pub struct Float {
     /// The bit width of the floating-point type.
-    pub bit_width: usize,
+    pub bits: u32,
 }
 
 /// Compile-time type information about string slice types.
diff --git a/library/coretests/tests/mem/type_info.rs b/library/coretests/tests/mem/type_info.rs
index fc13637a5574..53195fc5be0e 100644
--- a/library/coretests/tests/mem/type_info.rs
+++ b/library/coretests/tests/mem/type_info.rs
@@ -47,8 +47,8 @@ fn test_tuples() {
 
                 match (a.ty.info().kind, b.ty.info().kind) {
                     (TypeKind::Int(a), TypeKind::Int(b)) => {
-                        assert!(a.bit_width == 8 && a.signed);
-                        assert!(b.bit_width == 8 && !b.signed);
+                        assert!(a.bits == 8 && a.signed);
+                        assert!(b.bits == 8 && !b.signed);
                     }
                     _ => unreachable!(),
                 }
@@ -70,27 +70,27 @@ fn test_primitives() {
 
     let Type { kind: Int(ty), size, .. } = (const { Type::of::() }) else { panic!() };
     assert_eq!(size, Some(4));
-    assert_eq!(ty.bit_width, 32);
+    assert_eq!(ty.bits, 32);
     assert!(ty.signed);
 
     let Type { kind: Int(ty), size, .. } = (const { Type::of::() }) else { panic!() };
     assert_eq!(size, Some(size_of::()));
-    assert_eq!(ty.bit_width, size_of::() * 8);
+    assert_eq!(ty.bits as usize, size_of::() * 8);
     assert!(ty.signed);
 
     let Type { kind: Int(ty), size, .. } = (const { Type::of::() }) else { panic!() };
     assert_eq!(size, Some(4));
-    assert_eq!(ty.bit_width, 32);
+    assert_eq!(ty.bits, 32);
     assert!(!ty.signed);
 
     let Type { kind: Int(ty), size, .. } = (const { Type::of::() }) else { panic!() };
     assert_eq!(size, Some(size_of::()));
-    assert_eq!(ty.bit_width, size_of::() * 8);
+    assert_eq!(ty.bits as usize, size_of::() * 8);
     assert!(!ty.signed);
 
     let Type { kind: Float(ty), size, .. } = (const { Type::of::() }) else { panic!() };
     assert_eq!(size, Some(4));
-    assert_eq!(ty.bit_width, 32);
+    assert_eq!(ty.bits, 32);
 
     let Type { kind: Str(_ty), size, .. } = (const { Type::of::() }) else { panic!() };
     assert_eq!(size, None);
diff --git a/tests/ui/reflection/dump.bit32.run.stdout b/tests/ui/reflection/dump.bit32.run.stdout
index 483efdbbd12a..c086cdd11aff 100644
--- a/tests/ui/reflection/dump.bit32.run.stdout
+++ b/tests/ui/reflection/dump.bit32.run.stdout
@@ -35,7 +35,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 8,
+            bits: 8,
             signed: true,
         },
     ),
@@ -46,7 +46,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 32,
+            bits: 32,
             signed: true,
         },
     ),
@@ -57,7 +57,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 64,
+            bits: 64,
             signed: true,
         },
     ),
@@ -68,7 +68,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 128,
+            bits: 128,
             signed: true,
         },
     ),
@@ -79,7 +79,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 32,
+            bits: 32,
             signed: true,
         },
     ),
@@ -90,7 +90,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 8,
+            bits: 8,
             signed: false,
         },
     ),
@@ -101,7 +101,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 32,
+            bits: 32,
             signed: false,
         },
     ),
@@ -112,7 +112,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 64,
+            bits: 64,
             signed: false,
         },
     ),
@@ -123,7 +123,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 128,
+            bits: 128,
             signed: false,
         },
     ),
@@ -134,7 +134,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 32,
+            bits: 32,
             signed: false,
         },
     ),
diff --git a/tests/ui/reflection/dump.bit64.run.stdout b/tests/ui/reflection/dump.bit64.run.stdout
index 681e81b71d56..5f82ab44a91d 100644
--- a/tests/ui/reflection/dump.bit64.run.stdout
+++ b/tests/ui/reflection/dump.bit64.run.stdout
@@ -35,7 +35,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 8,
+            bits: 8,
             signed: true,
         },
     ),
@@ -46,7 +46,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 32,
+            bits: 32,
             signed: true,
         },
     ),
@@ -57,7 +57,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 64,
+            bits: 64,
             signed: true,
         },
     ),
@@ -68,7 +68,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 128,
+            bits: 128,
             signed: true,
         },
     ),
@@ -79,7 +79,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 64,
+            bits: 64,
             signed: true,
         },
     ),
@@ -90,7 +90,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 8,
+            bits: 8,
             signed: false,
         },
     ),
@@ -101,7 +101,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 32,
+            bits: 32,
             signed: false,
         },
     ),
@@ -112,7 +112,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 64,
+            bits: 64,
             signed: false,
         },
     ),
@@ -123,7 +123,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 128,
+            bits: 128,
             signed: false,
         },
     ),
@@ -134,7 +134,7 @@ Type {
 Type {
     kind: Int(
         Int {
-            bit_width: 64,
+            bits: 64,
             signed: false,
         },
     ),

From ebfd22796fa3eb0bc52612ce7b45cd789f8cb089 Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Sat, 17 Jan 2026 12:02:37 +0000
Subject: [PATCH 0323/1843] Port #[needs_allocator] to attribute parser

---
 compiler/rustc_attr_parsing/src/attributes/link_attrs.rs | 9 +++++++++
 compiler/rustc_attr_parsing/src/context.rs               | 3 ++-
 compiler/rustc_hir/src/attrs/data_structures.rs          | 3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs       | 1 +
 compiler/rustc_metadata/src/rmeta/encoder.rs             | 2 +-
 compiler/rustc_passes/src/check_attr.rs                  | 2 +-
 6 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs
index 61f975555884..2a04b55f469e 100644
--- a/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs
@@ -658,3 +658,12 @@ impl SingleAttributeParser for LinkageParser {
         Some(AttributeKind::Linkage(linkage, cx.attr_span))
     }
 }
+
+pub(crate) struct NeedsAllocatorParser;
+
+impl NoArgsAttributeParser for NeedsAllocatorParser {
+    const PATH: &[Symbol] = &[sym::needs_allocator];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Crate)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::NeedsAllocator;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 2a2fccd32202..449894f7834b 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -41,7 +41,7 @@ use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
 use crate::attributes::instruction_set::InstructionSetParser;
 use crate::attributes::link_attrs::{
     ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
-    LinkParser, LinkSectionParser, LinkageParser, StdInternalSymbolParser,
+    LinkParser, LinkSectionParser, LinkageParser, NeedsAllocatorParser, StdInternalSymbolParser,
 };
 use crate::attributes::lint_helpers::{
     AsPtrParser, AutomaticallyDerivedParser, PassByValueParser, PubTransparentParser,
@@ -259,6 +259,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 126fb8eac11b..7b7fae9fdcca 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -840,6 +840,9 @@ pub enum AttributeKind {
     /// Represents `#[naked]`
     Naked(Span),
 
+    /// Represents `#[needs_allocator]`
+    NeedsAllocator,
+
     /// Represents `#[no_core]`
     NoCore(Span),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 760e7c76df35..dff8a5727771 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -75,6 +75,7 @@ impl AttributeKind {
             MustNotSupend { .. } => Yes,
             MustUse { .. } => Yes,
             Naked(..) => No,
+            NeedsAllocator => No,
             NoCore(..) => No,
             NoImplicitPrelude(..) => No,
             NoLink => No,
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index a62826cd7cec..dbcb44e3220e 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -742,7 +742,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 proc_macro_data,
                 debugger_visualizers,
                 compiler_builtins: ast::attr::contains_name(attrs, sym::compiler_builtins),
-                needs_allocator: ast::attr::contains_name(attrs, sym::needs_allocator),
+                needs_allocator: find_attr!(attrs, AttributeKind::NeedsAllocator),
                 needs_panic_runtime: ast::attr::contains_name(attrs, sym::needs_panic_runtime),
                 no_builtins: ast::attr::contains_name(attrs, sym::no_builtins),
                 panic_runtime: ast::attr::contains_name(attrs, sym::panic_runtime),
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index f57bd62d3e99..4b71d4755cb6 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -315,6 +315,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDumpPredicates
                     | AttributeKind::RustcDumpDefParents
                     | AttributeKind::RustcDumpVtable(..)
+                    | AttributeKind::NeedsAllocator
                 ) => { /* do nothing  */ }
                 Attribute::Unparsed(attr_item) => {
                     style = Some(attr_item.style);
@@ -346,7 +347,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::prelude_import
                             | sym::panic_handler
                             | sym::lang
-                            | sym::needs_allocator
                             | sym::default_lib_allocator
                             | sym::rustc_diagnostic_item
                             | sym::rustc_no_mir_inline

From c31698b6958b5c818e4f4c86d3e4d12e128152f9 Mon Sep 17 00:00:00 2001
From: A4-Tacks 
Date: Sat, 17 Jan 2026 19:24:19 +0800
Subject: [PATCH 0324/1843] Improve move_guard redundanted block

Example
---
```rust
fn main() {
    match 92 {
        x $0if x > 10 => {
            let _ = true;
            false
        },
        _ => true
    }
}
```

**Before this PR**

```rust
fn main() {
    match 92 {
        x => if x > 10 {
            {
                let _ = true;
                false
            }
        },
        _ => true
    }
}
```

**After this PR**

```rust
fn main() {
    match 92 {
        x => if x > 10 {
            let _ = true;
            false
        },
        _ => true
    }
}
```
---
 .../ide-assists/src/handlers/move_guard.rs    | 35 ++++++++++++++++---
 .../crates/ide-assists/src/utils.rs           | 11 ++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs
index 31baa63372ff..84f02bdfdba6 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/move_guard.rs
@@ -49,7 +49,7 @@ pub(crate) fn move_guard_to_arm_body(acc: &mut Assists, ctx: &AssistContext<'_>)
 
     let guard_condition = guard.condition()?.reset_indent();
     let arm_expr = match_arm.expr()?;
-    let then_branch = make::block_expr(None, Some(arm_expr.reset_indent().indent(1.into())));
+    let then_branch = crate::utils::wrap_block(&arm_expr);
     let if_expr = make::expr_if(guard_condition, then_branch, None).indent(arm_expr.indent_level());
 
     let target = guard.syntax().text_range();
@@ -344,6 +344,35 @@ fn main() {
         );
     }
 
+    #[test]
+    fn move_guard_to_block_arm_body_works() {
+        check_assist(
+            move_guard_to_arm_body,
+            r#"
+fn main() {
+    match 92 {
+        x $0if x > 10 => {
+            let _ = true;
+            false
+        },
+        _ => true
+    }
+}
+"#,
+            r#"
+fn main() {
+    match 92 {
+        x => if x > 10 {
+            let _ = true;
+            false
+        },
+        _ => true
+    }
+}
+"#,
+        );
+    }
+
     #[test]
     fn move_let_guard_to_arm_body_works() {
         check_assist(
@@ -395,9 +424,7 @@ fn main() {
             && true
             && true {
             {
-                {
-                    false
-                }
+                false
             }
         },
         _ => true
diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs b/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs
index 9a96374c00af..4b8c19305793 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs
@@ -86,6 +86,17 @@ pub fn extract_trivial_expression(block_expr: &ast::BlockExpr) -> Option ast::BlockExpr {
+    if let ast::Expr::BlockExpr(block) = expr
+        && let Some(first) = block.syntax().first_token()
+        && first.kind() == T!['{']
+    {
+        block.reset_indent()
+    } else {
+        make::block_expr(None, Some(expr.reset_indent().indent(1.into())))
+    }
+}
+
 /// This is a method with a heuristics to support test methods annotated with custom test annotations, such as
 /// `#[test_case(...)]`, `#[tokio::test]` and similar.
 /// Also a regular `#[test]` annotation is supported.

From fe63539778fd3ae3b7aae1e03b96f10991f41d16 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sat, 17 Jan 2026 14:18:11 +0100
Subject: [PATCH 0325/1843] Explicitly list crate level attrs

---
 compiler/rustc_passes/src/check_attr.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index f57bd62d3e99..a4aed2b92804 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -389,13 +389,25 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_partition_reused
                             | sym::rustc_partition_codegened
                             | sym::rustc_expected_cgu_reuse
-                            | sym::rustc_nounwind,
+                            | sym::rustc_nounwind
+                            // crate-level attrs, are checked below
+                            | sym::feature
+                            | sym::register_tool
+                            | sym::rustc_no_implicit_bounds
+                            | sym::test_runner
+                            | sym::reexport_test_harness_main
+                            | sym::no_main
+                            | sym::no_builtins
+                            | sym::crate_type
+                            | sym::compiler_builtins
+                            | sym::profiler_runtime
+                            | sym::needs_panic_runtime
+                            | sym::panic_runtime
+                            | sym::rustc_preserve_ub_checks,
                             ..
                         ] => {}
                         [name, rest@..] => {
                             match BUILTIN_ATTRIBUTE_MAP.get(name) {
-                                // checked below
-                                Some(BuiltinAttribute { type_: AttributeType::CrateLevel, .. }) => {}
                                 Some(_) => {
                                     if rest.len() > 0 && AttributeParser::::is_parsed_attribute(slice::from_ref(name)) {
                                         // Check if we tried to use a builtin attribute as an attribute namespace, like `#[must_use::skip]`.

From ad3e082afe997060dc1be247dc227db3e8d250d6 Mon Sep 17 00:00:00 2001
From: Mark Rousskov 
Date: Sat, 17 Jan 2026 10:19:26 -0500
Subject: [PATCH 0326/1843] Bump version to 1.95.0

---
 src/version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/version b/src/version
index 8db4a57b3d02..55f6ae93382d 100644
--- a/src/version
+++ b/src/version
@@ -1 +1 @@
-1.94.0
+1.95.0

From 308b736541d4fdc000cf8abb1577f0d55e05c77c Mon Sep 17 00:00:00 2001
From: Redddy 
Date: Thu, 15 Jan 2026 23:56:14 +0900
Subject: [PATCH 0327/1843] Added section on using GitHub Dev for PR inspection

---
 src/doc/rustc-dev-guide/src/git.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/doc/rustc-dev-guide/src/git.md b/src/doc/rustc-dev-guide/src/git.md
index e7e84e2ea7f5..abe72b29cb1e 100644
--- a/src/doc/rustc-dev-guide/src/git.md
+++ b/src/doc/rustc-dev-guide/src/git.md
@@ -495,6 +495,14 @@ command to check it out locally. See  for more info.
 
 ![`gh` suggestion](./img/github-cli.png)
 
+### Using GitHub dev
+
+As an alternative to the GitHub web UI, GitHub Dev provides a web-based editor for browsing
+repository and PRs. It can be opened by replacing `github.com` with `github.dev` in the URL
+or by pressing `.` on a GitHub page.
+See [the docs for github.dev editor](https://docs.github.com/en/codespaces/the-githubdev-web-based-editor)
+for more details.
+
 ### Moving large sections of code
 
 Git and Github's default diff view for large moves *within* a file is quite poor; it will show each

From b3a5f53e8f50e5c2532b52cb7dc9222692e739f4 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Sat, 17 Jan 2026 17:02:46 +0100
Subject: [PATCH 0328/1843] rustdoc: Fix ICE when deprecated note is not
 resolved on the correct `DefId`

---
 .../passes/collect_intra_doc_links.rs         | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/librustdoc/passes/collect_intra_doc_links.rs b/src/librustdoc/passes/collect_intra_doc_links.rs
index 018e20b76dc8..a68e9dc87ae5 100644
--- a/src/librustdoc/passes/collect_intra_doc_links.rs
+++ b/src/librustdoc/passes/collect_intra_doc_links.rs
@@ -11,10 +11,11 @@ use rustc_ast::util::comments::may_have_doc_links;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
 use rustc_errors::{Applicability, Diag, DiagMessage};
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::Namespace::*;
 use rustc_hir::def::{DefKind, MacroKinds, Namespace, PerNS};
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LOCAL_CRATE};
-use rustc_hir::{Mutability, Safety};
+use rustc_hir::{Attribute, Mutability, Safety};
 use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_middle::{bug, span_bug, ty};
 use rustc_resolve::rustdoc::pulldown_cmark::LinkType;
@@ -1108,10 +1109,8 @@ impl LinkCollector<'_, '_> {
 
         // Also resolve links in the note text of `#[deprecated]`.
         for attr in &item.attrs.other_attrs {
-            let rustc_hir::Attribute::Parsed(rustc_hir::attrs::AttributeKind::Deprecation {
-                span,
-                deprecation,
-            }) = attr
+            let Attribute::Parsed(AttributeKind::Deprecation { span: depr_span, deprecation }) =
+                attr
             else {
                 continue;
             };
@@ -1128,8 +1127,14 @@ impl LinkCollector<'_, '_> {
             // inlined item.
             // 
             let item_id = if let Some(inline_stmt_id) = item.inline_stmt_id
-                && item.span(tcx).is_none_or(|item_span| !item_span.inner().contains(*span))
-            {
+                && tcx.get_all_attrs(inline_stmt_id).iter().any(|attr| {
+                    matches!(
+                        attr,
+                        Attribute::Parsed(AttributeKind::Deprecation {
+                            span: attr_span, ..
+                        }) if attr_span == depr_span,
+                    )
+                }) {
                 inline_stmt_id.to_def_id()
             } else {
                 item.item_id.expect_def_id()

From 1faaa769615cda3c662f96b5f48678e6f97ee245 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Sat, 17 Jan 2026 17:03:04 +0100
Subject: [PATCH 0329/1843] Add regression test for ICE when deprecated note is
 not resolved on the correct `DefId`

---
 .../deprecated-note-from-reexported.rs        | 16 +++++++++
 .../deprecated-note-from-reexported.stderr    | 34 +++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.rs
 create mode 100644 tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.stderr

diff --git a/tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.rs b/tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.rs
new file mode 100644
index 000000000000..3d1e48a4c638
--- /dev/null
+++ b/tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.rs
@@ -0,0 +1,16 @@
+// This test ensures that the intra-doc link from reexported deprecated attribute note
+// are resolved where they are declared.
+
+#![deny(rustdoc::broken_intra_doc_links)]
+
+#[doc(inline)]
+pub use bar::sql_function_proc as sql_function;
+
+pub fn define_sql_function() {}
+
+pub mod bar {
+    #[deprecated(note = "Use [`define_sql_function`] instead")]
+    //~^ ERROR: unresolved link
+    //~| ERROR: unresolved link
+    pub fn sql_function_proc() {}
+}
diff --git a/tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.stderr b/tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.stderr
new file mode 100644
index 000000000000..25f10b24d9fb
--- /dev/null
+++ b/tests/rustdoc-ui/intra-doc/deprecated-note-from-reexported.stderr
@@ -0,0 +1,34 @@
+error: unresolved link to `define_sql_function`
+  --> $DIR/deprecated-note-from-reexported.rs:12:25
+   |
+LL |     #[deprecated(note = "Use [`define_sql_function`] instead")]
+   |                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: the link appears in this line:
+           
+           Use [`define_sql_function`] instead
+                ^^^^^^^^^^^^^^^^^^^^^
+   = note: no item named `define_sql_function` in scope
+   = help: to escape `[` and `]` characters, add '\' before them like `\[` or `\]`
+note: the lint level is defined here
+  --> $DIR/deprecated-note-from-reexported.rs:4:9
+   |
+LL | #![deny(rustdoc::broken_intra_doc_links)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: unresolved link to `define_sql_function`
+  --> $DIR/deprecated-note-from-reexported.rs:12:25
+   |
+LL |     #[deprecated(note = "Use [`define_sql_function`] instead")]
+   |                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: the link appears in this line:
+           
+           Use [`define_sql_function`] instead
+                ^^^^^^^^^^^^^^^^^^^^^
+   = note: no item named `define_sql_function` in scope
+   = help: to escape `[` and `]` characters, add '\' before them like `\[` or `\]`
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error: aborting due to 2 previous errors
+

From a0f9a15b4a916c92d51131418e9fe080c83f9d3c Mon Sep 17 00:00:00 2001
From: Andreas Liljeqvist 
Date: Sat, 17 Jan 2026 11:36:25 +0100
Subject: [PATCH 0330/1843] Fix is_ascii performance regression on AVX-512 CPUs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When `[u8]::is_ascii()` is compiled with `-C target-cpu=native` on
AVX-512 CPUs, LLVM generates inefficient code. Because `is_ascii` is
marked `#[inline]`, it gets inlined and recompiled with the user's
target settings. The previous implementation used a counting loop that
LLVM auto-vectorizes to `pmovmskb` on SSE2, but with AVX-512 enabled,
LLVM uses k-registers and extracts bits individually with ~31
`kshiftrd` instructions.

This fix replaces the counting loop with explicit SSE2 intrinsics
(`_mm_loadu_si128`, `_mm_or_si128`, `_mm_movemask_epi8`) for x86_64.
`_mm_movemask_epi8` compiles to `pmovmskb`, forcing efficient codegen
regardless of CPU features.

Benchmark results on AMD Ryzen 5 7500F (Zen 4 with AVX-512):
- Default build: ~73 GB/s → ~74 GB/s (no regression)
- With -C target-cpu=native: ~3 GB/s → ~67 GB/s (22x improvement)

The loongarch64 implementation retains the original counting loop
since it doesn't have this issue.

Regression from: https://github.com/rust-lang/rust/pull/130733
---
 library/core/src/slice/ascii.rs              | 86 +++++++++++++++++---
 tests/assembly-llvm/slice-is-ascii-avx512.rs | 18 ++++
 tests/codegen-llvm/slice-is-ascii.rs         |  9 +-
 3 files changed, 98 insertions(+), 15 deletions(-)
 create mode 100644 tests/assembly-llvm/slice-is-ascii-avx512.rs

diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs
index 3e8c553f9f15..c9e168d6cbf8 100644
--- a/library/core/src/slice/ascii.rs
+++ b/library/core/src/slice/ascii.rs
@@ -3,10 +3,7 @@
 use core::ascii::EscapeDefault;
 
 use crate::fmt::{self, Write};
-#[cfg(not(any(
-    all(target_arch = "x86_64", target_feature = "sse2"),
-    all(target_arch = "loongarch64", target_feature = "lsx")
-)))]
+#[cfg(not(all(target_arch = "loongarch64", target_feature = "lsx")))]
 use crate::intrinsics::const_eval_select;
 use crate::{ascii, iter, ops};
 
@@ -463,19 +460,84 @@ const fn is_ascii(s: &[u8]) -> bool {
     )
 }
 
-/// ASCII test optimized to use the `pmovmskb` instruction on `x86-64` and the
-/// `vmskltz.b` instruction on `loongarch64`.
+/// SSE2 implementation using `_mm_movemask_epi8` (compiles to `pmovmskb`) to
+/// avoid LLVM's broken AVX-512 auto-vectorization of counting loops.
+///
+/// # Safety
+/// Requires SSE2 support (guaranteed on x86_64).
+#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
+#[target_feature(enable = "sse2")]
+unsafe fn is_ascii_sse2(bytes: &[u8]) -> bool {
+    use crate::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128};
+
+    const CHUNK_SIZE: usize = 32;
+
+    let mut i = 0;
+
+    while i + CHUNK_SIZE <= bytes.len() {
+        // SAFETY: We have verified that `i + CHUNK_SIZE <= bytes.len()`.
+        let ptr = unsafe { bytes.as_ptr().add(i) };
+
+        // Load two 16-byte chunks and combine them.
+        // SAFETY: We verified `i + 32 <= len`, so ptr is valid for 32 bytes.
+        // `_mm_loadu_si128` allows unaligned loads.
+        let chunk1 = unsafe { _mm_loadu_si128(ptr as *const __m128i) };
+        // SAFETY: Same as above - ptr.add(16) is within the valid 32-byte range.
+        let chunk2 = unsafe { _mm_loadu_si128(ptr.add(16) as *const __m128i) };
+
+        // OR them together - if any byte has the high bit set, the result will too
+        let combined = _mm_or_si128(chunk1, chunk2);
+
+        // Create a mask from the MSBs of each byte.
+        // If any byte is >= 128, its MSB is 1, so the mask will be non-zero.
+        let mask = _mm_movemask_epi8(combined);
+
+        if mask != 0 {
+            return false;
+        }
+
+        i += CHUNK_SIZE;
+    }
+
+    // Handle remaining bytes with simple loop
+    while i < bytes.len() {
+        if !bytes[i].is_ascii() {
+            return false;
+        }
+        i += 1;
+    }
+
+    true
+}
+
+/// ASCII test optimized to use the `pmovmskb` instruction on `x86-64`.
+///
+/// Uses explicit SSE2 intrinsics to prevent LLVM from auto-vectorizing with
+/// broken AVX-512 code that extracts mask bits one-by-one.
+#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
+#[inline]
+#[rustc_allow_const_fn_unstable(const_eval_select)]
+const fn is_ascii(bytes: &[u8]) -> bool {
+    const_eval_select!(
+        @capture { bytes: &[u8] } -> bool:
+        if const {
+            is_ascii_simple(bytes)
+        } else {
+            // SAFETY: SSE2 is guaranteed available on x86_64
+            unsafe { is_ascii_sse2(bytes) }
+        }
+    )
+}
+
+/// ASCII test optimized to use the `vmskltz.b` instruction on `loongarch64`.
 ///
 /// Other platforms are not likely to benefit from this code structure, so they
 /// use SWAR techniques to test for ASCII in `usize`-sized chunks.
-#[cfg(any(
-    all(target_arch = "x86_64", target_feature = "sse2"),
-    all(target_arch = "loongarch64", target_feature = "lsx")
-))]
+#[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
 #[inline]
 const fn is_ascii(bytes: &[u8]) -> bool {
     // Process chunks of 32 bytes at a time in the fast path to enable
-    // auto-vectorization and use of `pmovmskb`. Two 128-bit vector registers
+    // auto-vectorization and use of `vmskltz.b`. Two 128-bit vector registers
     // can be OR'd together and then the resulting vector can be tested for
     // non-ASCII bytes.
     const CHUNK_SIZE: usize = 32;
@@ -485,7 +547,7 @@ const fn is_ascii(bytes: &[u8]) -> bool {
     while i + CHUNK_SIZE <= bytes.len() {
         let chunk_end = i + CHUNK_SIZE;
 
-        // Get LLVM to produce a `pmovmskb` instruction on x86-64 which
+        // Get LLVM to produce a `vmskltz.b` instruction on loongarch64 which
         // creates a mask from the most significant bit of each byte.
         // ASCII bytes are less than 128 (0x80), so their most significant
         // bit is unset.
diff --git a/tests/assembly-llvm/slice-is-ascii-avx512.rs b/tests/assembly-llvm/slice-is-ascii-avx512.rs
new file mode 100644
index 000000000000..d3a441fec96c
--- /dev/null
+++ b/tests/assembly-llvm/slice-is-ascii-avx512.rs
@@ -0,0 +1,18 @@
+//@ only-x86_64
+//@ compile-flags: -C opt-level=3 -C target-cpu=znver4
+//@ compile-flags: -C llvm-args=-x86-asm-syntax=intel
+//@ assembly-output: emit-asm
+#![crate_type = "lib"]
+
+// Verify is_ascii uses pmovmskb/vpmovmskb instead of kshiftrd with AVX-512.
+// The fix uses explicit SSE2 intrinsics to avoid LLVM's broken auto-vectorization.
+//
+// See: https://github.com/rust-lang/rust/issues/129293
+
+// CHECK-LABEL: test_is_ascii
+#[no_mangle]
+pub fn test_is_ascii(s: &[u8]) -> bool {
+    // CHECK-NOT: kshiftrd
+    // CHECK-NOT: kshiftrq
+    s.is_ascii()
+}
diff --git a/tests/codegen-llvm/slice-is-ascii.rs b/tests/codegen-llvm/slice-is-ascii.rs
index 67537c871a0a..1f41b69e4396 100644
--- a/tests/codegen-llvm/slice-is-ascii.rs
+++ b/tests/codegen-llvm/slice-is-ascii.rs
@@ -1,10 +1,13 @@
-//@ only-x86_64
-//@ compile-flags: -C opt-level=3 -C target-cpu=x86-64
+//@ only-loongarch64
+//@ compile-flags: -C opt-level=3
 #![crate_type = "lib"]
 
-/// Check that the fast-path of `is_ascii` uses a `pmovmskb` instruction.
+/// Check that the fast-path of `is_ascii` uses a `vmskltz.b` instruction.
 /// Platforms lacking an equivalent instruction use other techniques for
 /// optimizing `is_ascii`.
+///
+/// Note: x86_64 uses explicit SSE2 intrinsics instead of relying on
+/// auto-vectorization. See `slice-is-ascii-avx512.rs`.
 // CHECK-LABEL: @is_ascii_autovectorized
 #[no_mangle]
 pub fn is_ascii_autovectorized(s: &[u8]) -> bool {

From 4d2f6a0843b161df938c341ecf142e636cd54b5b Mon Sep 17 00:00:00 2001
From: Takayuki Maeda 
Date: Sun, 18 Jan 2026 04:34:46 +0900
Subject: [PATCH 0331/1843] fix ICE on inconsistent import resolution with
 macro-attributed extern crate

---
 compiler/rustc_resolve/src/imports.rs              |  2 +-
 .../resolve/ice-inconsistent-resolution-151213.rs  | 14 ++++++++++++++
 .../ice-inconsistent-resolution-151213.stderr      | 14 ++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/resolve/ice-inconsistent-resolution-151213.rs
 create mode 100644 tests/ui/resolve/ice-inconsistent-resolution-151213.stderr

diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index 451779ae32c6..016fc407daab 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -989,7 +989,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             PathResult::Module(module) => {
                 // Consistency checks, analogous to `finalize_macro_resolutions`.
                 if let Some(initial_module) = import.imported_module.get() {
-                    if module != initial_module && no_ambiguity {
+                    if module != initial_module && no_ambiguity && !self.issue_145575_hack_applied {
                         span_bug!(import.span, "inconsistent resolution for an import");
                     }
                 } else if self.privacy_errors.is_empty() {
diff --git a/tests/ui/resolve/ice-inconsistent-resolution-151213.rs b/tests/ui/resolve/ice-inconsistent-resolution-151213.rs
new file mode 100644
index 000000000000..ea0f1c2858ef
--- /dev/null
+++ b/tests/ui/resolve/ice-inconsistent-resolution-151213.rs
@@ -0,0 +1,14 @@
+//@ edition: 2024
+
+#[attr]
+//~^ ERROR cannot find attribute `attr` in this scope
+extern crate core as std;
+//~^ ERROR macro-expanded `extern crate` items cannot shadow names passed with `--extern`
+
+mod inner {
+    use std::str;
+
+    use crate::*;
+}
+
+fn main() {}
diff --git a/tests/ui/resolve/ice-inconsistent-resolution-151213.stderr b/tests/ui/resolve/ice-inconsistent-resolution-151213.stderr
new file mode 100644
index 000000000000..deb1e6c3e1cf
--- /dev/null
+++ b/tests/ui/resolve/ice-inconsistent-resolution-151213.stderr
@@ -0,0 +1,14 @@
+error: macro-expanded `extern crate` items cannot shadow names passed with `--extern`
+  --> $DIR/ice-inconsistent-resolution-151213.rs:5:1
+   |
+LL | extern crate core as std;
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: cannot find attribute `attr` in this scope
+  --> $DIR/ice-inconsistent-resolution-151213.rs:3:3
+   |
+LL | #[attr]
+   |   ^^^^
+
+error: aborting due to 2 previous errors
+

From b49539e0490188136ae5805e144967ad6d510a2b Mon Sep 17 00:00:00 2001
From: Scott McMurray 
Date: Sat, 17 Jan 2026 15:08:53 -0800
Subject: [PATCH 0332/1843] Include a link to `count_ones` in the docs for
 `uN::count_zeros`

---
 library/core/src/num/uint_macros.rs | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs
index eea5ce348355..57f0cd48fbe8 100644
--- a/library/core/src/num/uint_macros.rs
+++ b/library/core/src/num/uint_macros.rs
@@ -93,6 +93,28 @@ macro_rules! uint_impl {
         #[doc = concat!("let max = ", stringify!($SelfT),"::MAX;")]
         /// assert_eq!(max.count_zeros(), 0);
         /// ```
+        ///
+        /// This is heavily dependent on the width of the type, and thus
+        /// might give surprising results depending on type inference:
+        /// ```
+        /// # fn foo(_: u8) {}
+        /// # fn bar(_: u16) {}
+        /// let lucky = 7;
+        /// foo(lucky);
+        /// assert_eq!(lucky.count_zeros(), 5);
+        /// assert_eq!(lucky.count_ones(), 3);
+        ///
+        /// let lucky = 7;
+        /// bar(lucky);
+        /// assert_eq!(lucky.count_zeros(), 13);
+        /// assert_eq!(lucky.count_ones(), 3);
+        /// ```
+        /// You might want to use [`Self::count_ones`] instead, or emphasize
+        /// the type you're using in the call rather than method syntax:
+        /// ```
+        /// let small = 1;
+        #[doc = concat!("assert_eq!(", stringify!($SelfT), "::count_zeros(small), ", stringify!($BITS_MINUS_ONE) ,");")]
+        /// ```
         #[stable(feature = "rust1", since = "1.0.0")]
         #[rustc_const_stable(feature = "const_math", since = "1.32.0")]
         #[must_use = "this returns the result of the operation, \

From bdc08aeb91626171a7dc5bfbc1e0d109770ee457 Mon Sep 17 00:00:00 2001
From: Scott McMurray 
Date: Thu, 6 Jun 2024 12:49:31 -0700
Subject: [PATCH 0333/1843] Reword the caveats on `array::map`

Thanks to 107634 and some improvements in LLVM (particularly `dead_on_unwind`), the method actually optimizes reasonably well now.

So focus the discussion on the fundamental ordering differences where the optimizer might never be able to fix it because of the different behaviour, and encouraging `Iterator::map` where an array wasn't actually ever needed.
---
 library/core/src/array/mod.rs | 51 ++++++++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/library/core/src/array/mod.rs b/library/core/src/array/mod.rs
index 6cca2e6358b6..4a2cf1527744 100644
--- a/library/core/src/array/mod.rs
+++ b/library/core/src/array/mod.rs
@@ -516,20 +516,47 @@ impl [T; N] {
     ///
     /// # Note on performance and stack usage
     ///
-    /// Unfortunately, usages of this method are currently not always optimized
-    /// as well as they could be. This mainly concerns large arrays, as mapping
-    /// over small arrays seem to be optimized just fine. Also note that in
-    /// debug mode (i.e. without any optimizations), this method can use a lot
-    /// of stack space (a few times the size of the array or more).
+    /// Note that this method is *eager*.  It evaluates `f` all `N` times before
+    /// returning the new array.
     ///
-    /// Therefore, in performance-critical code, try to avoid using this method
-    /// on large arrays or check the emitted code. Also try to avoid chained
-    /// maps (e.g. `arr.map(...).map(...)`).
+    /// That means that `arr.map(f).map(g)` is, in general, *not* equivalent to
+    /// `array.map(|x| g(f(x)))`, as the former calls `f` 4 times then `g` 4 times,
+    /// whereas the latter interleaves the calls (`fgfgfgfg`).
     ///
-    /// In many cases, you can instead use [`Iterator::map`] by calling `.iter()`
-    /// or `.into_iter()` on your array. `[T; N]::map` is only necessary if you
-    /// really need a new array of the same size as the result. Rust's lazy
-    /// iterators tend to get optimized very well.
+    /// A consequence of this is that it can have fairly-high stack usage, especially
+    /// in debug mode or for long arrays.  The backend may be able to optimize it
+    /// away, but especially for complicated mappings it might not be able to.
+    ///
+    /// If you're doing a one-step `map` and really want an array as the result,
+    /// then absolutely use this method.  Its implementation uses a bunch of tricks
+    /// to help the optimizer handle it well.  Particularly for simple arrays,
+    /// like `[u8; 3]` or `[f32; 4]`, there's nothing to be concerned about.
+    ///
+    /// However, if you don't actually need an *array* of the results specifically,
+    /// just to process them, then you likely want [`Iterator::map`] instead.
+    ///
+    /// For example, rather than doing an array-to-array map of all the elements
+    /// in the array up-front and only iterating after that completes,
+    ///
+    /// ```
+    /// # let my_array = [1, 2, 3];
+    /// # let f = |x: i32| x + 1;
+    /// for x in my_array.map(f) {
+    ///     // ...
+    /// }
+    /// ```
+    ///
+    /// It's often better to use an iterator along the lines of
+    ///
+    /// ```
+    /// # let my_array = [1, 2, 3];
+    /// # let f = |x: i32| x + 1;
+    /// for x in my_array.into_iter().map(f) {
+    ///     // ...
+    /// }
+    /// ```
+    ///
+    /// as that's more likely to avoid large temporaries.
     ///
     ///
     /// # Examples

From 216ca145fd5967d1ff8731b8cc76eca7b53cff03 Mon Sep 17 00:00:00 2001
From: Deadbeef 
Date: Sat, 17 Jan 2026 21:43:50 -0500
Subject: [PATCH 0334/1843] remove trailing periods in built-in attribute gate
 messages

---
 compiler/rustc_feature/src/builtin_attrs.rs | 22 ++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 22753adb4c99..1fd6c3c10fab 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -678,7 +678,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     rustc_attr!(
         rustc_pass_indirectly_in_non_rustic_abis, Normal, template!(Word), ErrorFollowing,
         EncodeCrossCrate::No,
-        "types marked with `#[rustc_pass_indirectly_in_non_rustic_abis]` are always passed indirectly by non-Rustic abis."
+        "types marked with `#[rustc_pass_indirectly_in_non_rustic_abis]` are always passed indirectly by non-Rustic ABIs"
     ),
 
     // Limits:
@@ -1275,38 +1275,38 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     rustc_attr!(
         rustc_as_ptr, Normal, template!(Word), ErrorFollowing,
         EncodeCrossCrate::Yes,
-        "`#[rustc_as_ptr]` is used to mark functions returning pointers to their inner allocations."
+        "`#[rustc_as_ptr]` is used to mark functions returning pointers to their inner allocations"
     ),
     rustc_attr!(
         rustc_should_not_be_called_on_const_items, Normal, template!(Word), ErrorFollowing,
         EncodeCrossCrate::Yes,
-        "`#[rustc_should_not_be_called_on_const_items]` is used to mark methods that don't make sense to be called on interior mutable consts."
+        "`#[rustc_should_not_be_called_on_const_items]` is used to mark methods that don't make sense to be called on interior mutable consts"
     ),
     rustc_attr!(
         rustc_pass_by_value, Normal, template!(Word), ErrorFollowing,
         EncodeCrossCrate::Yes,
-        "`#[rustc_pass_by_value]` is used to mark types that must be passed by value instead of reference."
+        "`#[rustc_pass_by_value]` is used to mark types that must be passed by value instead of reference"
     ),
     rustc_attr!(
         rustc_never_returns_null_ptr, Normal, template!(Word), ErrorFollowing,
         EncodeCrossCrate::Yes,
-        "`#[rustc_never_returns_null_ptr]` is used to mark functions returning non-null pointers."
+        "`#[rustc_never_returns_null_ptr]` is used to mark functions returning non-null pointers"
     ),
     rustc_attr!(
         rustc_no_implicit_autorefs, AttributeType::Normal, template!(Word), ErrorFollowing, EncodeCrossCrate::Yes,
-        "`#[rustc_no_implicit_autorefs]` is used to mark functions for which an autoref to the dereference of a raw pointer should not be used as an argument."
+        "`#[rustc_no_implicit_autorefs]` is used to mark functions for which an autoref to the dereference of a raw pointer should not be used as an argument"
     ),
     rustc_attr!(
         rustc_coherence_is_core, AttributeType::CrateLevel, template!(Word), ErrorFollowing, EncodeCrossCrate::No,
-        "`#![rustc_coherence_is_core]` allows inherent methods on builtin types, only intended to be used in `core`."
+        "`#![rustc_coherence_is_core]` allows inherent methods on builtin types, only intended to be used in `core`"
     ),
     rustc_attr!(
         rustc_coinductive, AttributeType::Normal, template!(Word), WarnFollowing, EncodeCrossCrate::No,
-        "`#[rustc_coinductive]` changes a trait to be coinductive, allowing cycles in the trait solver."
+        "`#[rustc_coinductive]` changes a trait to be coinductive, allowing cycles in the trait solver"
     ),
     rustc_attr!(
         rustc_allow_incoherent_impl, AttributeType::Normal, template!(Word), ErrorFollowing, EncodeCrossCrate::No,
-        "`#[rustc_allow_incoherent_impl]` has to be added to all impl items of an incoherent inherent impl."
+        "`#[rustc_allow_incoherent_impl]` has to be added to all impl items of an incoherent inherent impl"
     ),
     rustc_attr!(
         rustc_preserve_ub_checks, AttributeType::CrateLevel, template!(Word), ErrorFollowing, EncodeCrossCrate::No,
@@ -1333,7 +1333,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_has_incoherent_inherent_impls, AttributeType::Normal, template!(Word),
         ErrorFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_has_incoherent_inherent_impls]` allows the addition of incoherent inherent impls for \
-         the given type by annotating all impl items with `#[rustc_allow_incoherent_impl]`."
+         the given type by annotating all impl items with `#[rustc_allow_incoherent_impl]`"
     ),
 
     BuiltinAttribute {
@@ -1396,7 +1396,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         EncodeCrossCrate::No,
         "the `#[rustc_skip_during_method_dispatch]` attribute is used to exclude a trait \
         from method dispatch when the receiver is of the following type, for compatibility in \
-        editions < 2021 (array) or editions < 2024 (boxed_slice)."
+        editions < 2021 (array) or editions < 2024 (boxed_slice)"
     ),
     rustc_attr!(
         rustc_must_implement_one_of, Normal, template!(List: &["function1, function2, ..."]),

From 7510f747a8599c430a054ba58d53dc8dae1465ee Mon Sep 17 00:00:00 2001
From: mu001999 
Date: Sun, 18 Jan 2026 11:40:42 +0800
Subject: [PATCH 0335/1843] Normalize type_const items even with feature
 `generic_const_exprs`

---
 .../rustc_trait_selection/src/traits/normalize.rs     |  7 ++++++-
 tests/crashes/138089.rs                               |  2 ++
 .../const-generics/mgca/cyclic-type-const-151251.rs   | 11 +++++++++++
 .../mgca/cyclic-type-const-151251.stderr              | 11 +++++++++++
 4 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/const-generics/mgca/cyclic-type-const-151251.rs
 create mode 100644 tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr

diff --git a/compiler/rustc_trait_selection/src/traits/normalize.rs b/compiler/rustc_trait_selection/src/traits/normalize.rs
index 71e9914f93fa..24854990fe71 100644
--- a/compiler/rustc_trait_selection/src/traits/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/normalize.rs
@@ -433,7 +433,12 @@ impl<'a, 'b, 'tcx> TypeFolder> for AssocTypeNormalizer<'a, 'b, 'tcx
     #[instrument(skip(self), level = "debug")]
     fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
         let tcx = self.selcx.tcx();
-        if tcx.features().generic_const_exprs() || !needs_normalization(self.selcx.infcx, &ct) {
+
+        if tcx.features().generic_const_exprs()
+            // Normalize type_const items even with feature `generic_const_exprs`.
+            && !matches!(ct.kind(), ty::ConstKind::Unevaluated(uv) if tcx.is_type_const(uv.def))
+            || !needs_normalization(self.selcx.infcx, &ct)
+        {
             return ct;
         }
 
diff --git a/tests/crashes/138089.rs b/tests/crashes/138089.rs
index 054d1b216959..f4864971ae5d 100644
--- a/tests/crashes/138089.rs
+++ b/tests/crashes/138089.rs
@@ -1,4 +1,6 @@
 //@ known-bug: #138089
+//@ needs-rustc-debug-assertions
+
 #![feature(generic_const_exprs)]
 #![feature(min_generic_const_args)]
 #![feature(inherent_associated_types)]
diff --git a/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs b/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs
new file mode 100644
index 000000000000..823a6d58bf47
--- /dev/null
+++ b/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs
@@ -0,0 +1,11 @@
+//@ needs-rustc-debug-assertions
+
+#![feature(min_generic_const_args)]
+#![feature(generic_const_exprs)]
+#![expect(incomplete_features)]
+
+#[type_const]
+const A: u8 = A;
+//~^ ERROR overflow normalizing the unevaluated constant `A`
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr b/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr
new file mode 100644
index 000000000000..1ce2af817277
--- /dev/null
+++ b/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr
@@ -0,0 +1,11 @@
+error[E0275]: overflow normalizing the unevaluated constant `A`
+  --> $DIR/cyclic-type-const-151251.rs:8:1
+   |
+LL | const A: u8 = A;
+   | ^^^^^^^^^^^
+   |
+   = note: in case this is a recursive type alias, consider using a struct, enum, or union instead
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0275`.

From cf5a7da23f59af669aab128d919d04b96d9ec08b Mon Sep 17 00:00:00 2001
From: tuturuu 
Date: Sun, 18 Jan 2026 05:16:29 +0100
Subject: [PATCH 0336/1843] move some tests

---
 .../associated-type-as-value.rs}                                  | 0
 .../associated-type-as-value.stderr}                              | 0
 tests/ui/{issues/issue-17441.rs => cast/cast-to-unsized-type.rs}  | 0
 .../issue-17441.stderr => cast/cast-to-unsized-type.stderr}       | 0
 .../issue-2995.rs => cast/non-primitive-isize-ref-cast.rs}        | 0
 .../non-primitive-isize-ref-cast.stderr}                          | 0
 .../ui/{issues/issue-19922.rs => enum/enum-nonexisting-field.rs}  | 0
 .../issue-19922.stderr => enum/enum-nonexisting-field.stderr}     | 0
 .../ui/{issues/issue-17351.rs => lint/unused/unused-trait-fn.rs}  | 0
 .../issue-17351.stderr => lint/unused/unused-trait-fn.stderr}     | 0
 .../issue-22599.rs => lint/unused/unused-var-in-match-arm.rs}     | 0
 .../unused/unused-var-in-match-arm.stderr}                        | 0
 .../issue-19692.rs => methods/method-not-found-on-struct.rs}      | 0
 .../method-not-found-on-struct.stderr}                            | 0
 .../{issues/issue-34209.rs => pattern/enum-variant-not-found.rs}  | 0
 .../issue-34209.stderr => pattern/enum-variant-not-found.stderr}  | 0
 .../issue-16745.rs => pattern/match-constant-and-byte-literal.rs} | 0
 .../auxiliary/imported-enum-is-private.rs}                        | 0
 .../issue-11680.rs => privacy/imported-enum-is-private.rs}        | 0
 .../imported-enum-is-private.stderr}                              | 0
 .../issue-26472.rs => privacy/private-struct-field-in-module.rs}  | 0
 .../private-struct-field-in-module.stderr}                        | 0
 .../panic-with-unspecified-type.rs}                               | 0
 .../panic-with-unspecified-type.stderr}                           | 0
 .../send-with-unspecified-type.rs}                                | 0
 .../send-with-unspecified-type.stderr}                            | 0
 .../swap-with-unspecified-type.rs}                                | 0
 .../swap-with-unspecified-type.stderr}                            | 0
 .../issue-35241.rs => type/struct-constructor-as-value.rs}        | 0
 .../struct-constructor-as-value.stderr}                           | 0
 30 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/ui/{issues/issue-38919.rs => associated-types/associated-type-as-value.rs} (100%)
 rename tests/ui/{issues/issue-38919.stderr => associated-types/associated-type-as-value.stderr} (100%)
 rename tests/ui/{issues/issue-17441.rs => cast/cast-to-unsized-type.rs} (100%)
 rename tests/ui/{issues/issue-17441.stderr => cast/cast-to-unsized-type.stderr} (100%)
 rename tests/ui/{issues/issue-2995.rs => cast/non-primitive-isize-ref-cast.rs} (100%)
 rename tests/ui/{issues/issue-2995.stderr => cast/non-primitive-isize-ref-cast.stderr} (100%)
 rename tests/ui/{issues/issue-19922.rs => enum/enum-nonexisting-field.rs} (100%)
 rename tests/ui/{issues/issue-19922.stderr => enum/enum-nonexisting-field.stderr} (100%)
 rename tests/ui/{issues/issue-17351.rs => lint/unused/unused-trait-fn.rs} (100%)
 rename tests/ui/{issues/issue-17351.stderr => lint/unused/unused-trait-fn.stderr} (100%)
 rename tests/ui/{issues/issue-22599.rs => lint/unused/unused-var-in-match-arm.rs} (100%)
 rename tests/ui/{issues/issue-22599.stderr => lint/unused/unused-var-in-match-arm.stderr} (100%)
 rename tests/ui/{issues/issue-19692.rs => methods/method-not-found-on-struct.rs} (100%)
 rename tests/ui/{issues/issue-19692.stderr => methods/method-not-found-on-struct.stderr} (100%)
 rename tests/ui/{issues/issue-34209.rs => pattern/enum-variant-not-found.rs} (100%)
 rename tests/ui/{issues/issue-34209.stderr => pattern/enum-variant-not-found.stderr} (100%)
 rename tests/ui/{issues/issue-16745.rs => pattern/match-constant-and-byte-literal.rs} (100%)
 rename tests/ui/{issues/auxiliary/issue-11680.rs => privacy/auxiliary/imported-enum-is-private.rs} (100%)
 rename tests/ui/{issues/issue-11680.rs => privacy/imported-enum-is-private.rs} (100%)
 rename tests/ui/{issues/issue-11680.stderr => privacy/imported-enum-is-private.stderr} (100%)
 rename tests/ui/{issues/issue-26472.rs => privacy/private-struct-field-in-module.rs} (100%)
 rename tests/ui/{issues/issue-26472.stderr => privacy/private-struct-field-in-module.stderr} (100%)
 rename tests/ui/{issues/issue-16966.rs => type-inference/panic-with-unspecified-type.rs} (100%)
 rename tests/ui/{issues/issue-16966.stderr => type-inference/panic-with-unspecified-type.stderr} (100%)
 rename tests/ui/{issues/issue-25368.rs => type-inference/send-with-unspecified-type.rs} (100%)
 rename tests/ui/{issues/issue-25368.stderr => type-inference/send-with-unspecified-type.stderr} (100%)
 rename tests/ui/{issues/issue-24013.rs => type-inference/swap-with-unspecified-type.rs} (100%)
 rename tests/ui/{issues/issue-24013.stderr => type-inference/swap-with-unspecified-type.stderr} (100%)
 rename tests/ui/{issues/issue-35241.rs => type/struct-constructor-as-value.rs} (100%)
 rename tests/ui/{issues/issue-35241.stderr => type/struct-constructor-as-value.stderr} (100%)

diff --git a/tests/ui/issues/issue-38919.rs b/tests/ui/associated-types/associated-type-as-value.rs
similarity index 100%
rename from tests/ui/issues/issue-38919.rs
rename to tests/ui/associated-types/associated-type-as-value.rs
diff --git a/tests/ui/issues/issue-38919.stderr b/tests/ui/associated-types/associated-type-as-value.stderr
similarity index 100%
rename from tests/ui/issues/issue-38919.stderr
rename to tests/ui/associated-types/associated-type-as-value.stderr
diff --git a/tests/ui/issues/issue-17441.rs b/tests/ui/cast/cast-to-unsized-type.rs
similarity index 100%
rename from tests/ui/issues/issue-17441.rs
rename to tests/ui/cast/cast-to-unsized-type.rs
diff --git a/tests/ui/issues/issue-17441.stderr b/tests/ui/cast/cast-to-unsized-type.stderr
similarity index 100%
rename from tests/ui/issues/issue-17441.stderr
rename to tests/ui/cast/cast-to-unsized-type.stderr
diff --git a/tests/ui/issues/issue-2995.rs b/tests/ui/cast/non-primitive-isize-ref-cast.rs
similarity index 100%
rename from tests/ui/issues/issue-2995.rs
rename to tests/ui/cast/non-primitive-isize-ref-cast.rs
diff --git a/tests/ui/issues/issue-2995.stderr b/tests/ui/cast/non-primitive-isize-ref-cast.stderr
similarity index 100%
rename from tests/ui/issues/issue-2995.stderr
rename to tests/ui/cast/non-primitive-isize-ref-cast.stderr
diff --git a/tests/ui/issues/issue-19922.rs b/tests/ui/enum/enum-nonexisting-field.rs
similarity index 100%
rename from tests/ui/issues/issue-19922.rs
rename to tests/ui/enum/enum-nonexisting-field.rs
diff --git a/tests/ui/issues/issue-19922.stderr b/tests/ui/enum/enum-nonexisting-field.stderr
similarity index 100%
rename from tests/ui/issues/issue-19922.stderr
rename to tests/ui/enum/enum-nonexisting-field.stderr
diff --git a/tests/ui/issues/issue-17351.rs b/tests/ui/lint/unused/unused-trait-fn.rs
similarity index 100%
rename from tests/ui/issues/issue-17351.rs
rename to tests/ui/lint/unused/unused-trait-fn.rs
diff --git a/tests/ui/issues/issue-17351.stderr b/tests/ui/lint/unused/unused-trait-fn.stderr
similarity index 100%
rename from tests/ui/issues/issue-17351.stderr
rename to tests/ui/lint/unused/unused-trait-fn.stderr
diff --git a/tests/ui/issues/issue-22599.rs b/tests/ui/lint/unused/unused-var-in-match-arm.rs
similarity index 100%
rename from tests/ui/issues/issue-22599.rs
rename to tests/ui/lint/unused/unused-var-in-match-arm.rs
diff --git a/tests/ui/issues/issue-22599.stderr b/tests/ui/lint/unused/unused-var-in-match-arm.stderr
similarity index 100%
rename from tests/ui/issues/issue-22599.stderr
rename to tests/ui/lint/unused/unused-var-in-match-arm.stderr
diff --git a/tests/ui/issues/issue-19692.rs b/tests/ui/methods/method-not-found-on-struct.rs
similarity index 100%
rename from tests/ui/issues/issue-19692.rs
rename to tests/ui/methods/method-not-found-on-struct.rs
diff --git a/tests/ui/issues/issue-19692.stderr b/tests/ui/methods/method-not-found-on-struct.stderr
similarity index 100%
rename from tests/ui/issues/issue-19692.stderr
rename to tests/ui/methods/method-not-found-on-struct.stderr
diff --git a/tests/ui/issues/issue-34209.rs b/tests/ui/pattern/enum-variant-not-found.rs
similarity index 100%
rename from tests/ui/issues/issue-34209.rs
rename to tests/ui/pattern/enum-variant-not-found.rs
diff --git a/tests/ui/issues/issue-34209.stderr b/tests/ui/pattern/enum-variant-not-found.stderr
similarity index 100%
rename from tests/ui/issues/issue-34209.stderr
rename to tests/ui/pattern/enum-variant-not-found.stderr
diff --git a/tests/ui/issues/issue-16745.rs b/tests/ui/pattern/match-constant-and-byte-literal.rs
similarity index 100%
rename from tests/ui/issues/issue-16745.rs
rename to tests/ui/pattern/match-constant-and-byte-literal.rs
diff --git a/tests/ui/issues/auxiliary/issue-11680.rs b/tests/ui/privacy/auxiliary/imported-enum-is-private.rs
similarity index 100%
rename from tests/ui/issues/auxiliary/issue-11680.rs
rename to tests/ui/privacy/auxiliary/imported-enum-is-private.rs
diff --git a/tests/ui/issues/issue-11680.rs b/tests/ui/privacy/imported-enum-is-private.rs
similarity index 100%
rename from tests/ui/issues/issue-11680.rs
rename to tests/ui/privacy/imported-enum-is-private.rs
diff --git a/tests/ui/issues/issue-11680.stderr b/tests/ui/privacy/imported-enum-is-private.stderr
similarity index 100%
rename from tests/ui/issues/issue-11680.stderr
rename to tests/ui/privacy/imported-enum-is-private.stderr
diff --git a/tests/ui/issues/issue-26472.rs b/tests/ui/privacy/private-struct-field-in-module.rs
similarity index 100%
rename from tests/ui/issues/issue-26472.rs
rename to tests/ui/privacy/private-struct-field-in-module.rs
diff --git a/tests/ui/issues/issue-26472.stderr b/tests/ui/privacy/private-struct-field-in-module.stderr
similarity index 100%
rename from tests/ui/issues/issue-26472.stderr
rename to tests/ui/privacy/private-struct-field-in-module.stderr
diff --git a/tests/ui/issues/issue-16966.rs b/tests/ui/type-inference/panic-with-unspecified-type.rs
similarity index 100%
rename from tests/ui/issues/issue-16966.rs
rename to tests/ui/type-inference/panic-with-unspecified-type.rs
diff --git a/tests/ui/issues/issue-16966.stderr b/tests/ui/type-inference/panic-with-unspecified-type.stderr
similarity index 100%
rename from tests/ui/issues/issue-16966.stderr
rename to tests/ui/type-inference/panic-with-unspecified-type.stderr
diff --git a/tests/ui/issues/issue-25368.rs b/tests/ui/type-inference/send-with-unspecified-type.rs
similarity index 100%
rename from tests/ui/issues/issue-25368.rs
rename to tests/ui/type-inference/send-with-unspecified-type.rs
diff --git a/tests/ui/issues/issue-25368.stderr b/tests/ui/type-inference/send-with-unspecified-type.stderr
similarity index 100%
rename from tests/ui/issues/issue-25368.stderr
rename to tests/ui/type-inference/send-with-unspecified-type.stderr
diff --git a/tests/ui/issues/issue-24013.rs b/tests/ui/type-inference/swap-with-unspecified-type.rs
similarity index 100%
rename from tests/ui/issues/issue-24013.rs
rename to tests/ui/type-inference/swap-with-unspecified-type.rs
diff --git a/tests/ui/issues/issue-24013.stderr b/tests/ui/type-inference/swap-with-unspecified-type.stderr
similarity index 100%
rename from tests/ui/issues/issue-24013.stderr
rename to tests/ui/type-inference/swap-with-unspecified-type.stderr
diff --git a/tests/ui/issues/issue-35241.rs b/tests/ui/type/struct-constructor-as-value.rs
similarity index 100%
rename from tests/ui/issues/issue-35241.rs
rename to tests/ui/type/struct-constructor-as-value.rs
diff --git a/tests/ui/issues/issue-35241.stderr b/tests/ui/type/struct-constructor-as-value.stderr
similarity index 100%
rename from tests/ui/issues/issue-35241.stderr
rename to tests/ui/type/struct-constructor-as-value.stderr

From 2c7969a0cd4217900f96511b16117a0ef31836c4 Mon Sep 17 00:00:00 2001
From: tuturuu 
Date: Sun, 18 Jan 2026 06:10:16 +0100
Subject: [PATCH 0337/1843] add metadata and bless moved tests

---
 .../ui/associated-types/associated-type-as-value.rs  |  2 ++
 .../associated-types/associated-type-as-value.stderr |  2 +-
 tests/ui/cast/cast-to-unsized-type.rs                |  2 ++
 tests/ui/cast/cast-to-unsized-type.stderr            | 12 ++++++------
 tests/ui/cast/non-primitive-isize-ref-cast.rs        |  2 ++
 tests/ui/cast/non-primitive-isize-ref-cast.stderr    |  2 +-
 tests/ui/enum/enum-nonexisting-field.rs              |  2 ++
 tests/ui/enum/enum-nonexisting-field.stderr          |  2 +-
 tests/ui/lint/unused/unused-trait-fn.rs              |  1 +
 tests/ui/lint/unused/unused-trait-fn.stderr          |  2 +-
 tests/ui/lint/unused/unused-var-in-match-arm.rs      |  1 +
 tests/ui/lint/unused/unused-var-in-match-arm.stderr  |  4 ++--
 tests/ui/methods/method-not-found-on-struct.rs       |  2 ++
 tests/ui/methods/method-not-found-on-struct.stderr   |  2 +-
 tests/ui/pattern/enum-variant-not-found.rs           |  2 ++
 tests/ui/pattern/enum-variant-not-found.stderr       |  2 +-
 tests/ui/pattern/match-constant-and-byte-literal.rs  |  1 +
 .../ui/privacy/auxiliary/imported-enum-is-private.rs |  2 ++
 tests/ui/privacy/imported-enum-is-private.rs         |  5 +++--
 tests/ui/privacy/imported-enum-is-private.stderr     |  8 ++++----
 tests/ui/privacy/private-struct-field-in-module.rs   |  2 ++
 .../ui/privacy/private-struct-field-in-module.stderr |  4 ++--
 .../ui/type-inference/panic-with-unspecified-type.rs |  1 +
 .../panic-with-unspecified-type.stderr               |  2 +-
 .../ui/type-inference/send-with-unspecified-type.rs  |  2 ++
 .../type-inference/send-with-unspecified-type.stderr |  2 +-
 .../ui/type-inference/swap-with-unspecified-type.rs  |  2 ++
 .../type-inference/swap-with-unspecified-type.stderr |  2 +-
 tests/ui/type/struct-constructor-as-value.rs         |  2 ++
 tests/ui/type/struct-constructor-as-value.stderr     |  2 +-
 30 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/tests/ui/associated-types/associated-type-as-value.rs b/tests/ui/associated-types/associated-type-as-value.rs
index 3d28f1936b47..ddc808236658 100644
--- a/tests/ui/associated-types/associated-type-as-value.rs
+++ b/tests/ui/associated-types/associated-type-as-value.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 fn foo() {
     T::Item; //~ ERROR no associated item named `Item` found
 }
diff --git a/tests/ui/associated-types/associated-type-as-value.stderr b/tests/ui/associated-types/associated-type-as-value.stderr
index 4a4bd2ee43d8..c553582b3907 100644
--- a/tests/ui/associated-types/associated-type-as-value.stderr
+++ b/tests/ui/associated-types/associated-type-as-value.stderr
@@ -1,5 +1,5 @@
 error[E0599]: no associated item named `Item` found for type parameter `T` in the current scope
-  --> $DIR/issue-38919.rs:2:8
+  --> $DIR/associated-type-as-value.rs:4:8
    |
 LL | fn foo() {
    |        - associated item `Item` not found for this type parameter
diff --git a/tests/ui/cast/cast-to-unsized-type.rs b/tests/ui/cast/cast-to-unsized-type.rs
index e5f83c4ebadd..4c7ca49fd98d 100644
--- a/tests/ui/cast/cast-to-unsized-type.rs
+++ b/tests/ui/cast/cast-to-unsized-type.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 fn main() {
     let _foo = &[1_usize, 2] as [usize];
     //~^ ERROR cast to unsized type: `&[usize; 2]` as `[usize]`
diff --git a/tests/ui/cast/cast-to-unsized-type.stderr b/tests/ui/cast/cast-to-unsized-type.stderr
index 96aad879e24d..087cfac77484 100644
--- a/tests/ui/cast/cast-to-unsized-type.stderr
+++ b/tests/ui/cast/cast-to-unsized-type.stderr
@@ -1,5 +1,5 @@
 error[E0620]: cast to unsized type: `&[usize; 2]` as `[usize]`
-  --> $DIR/issue-17441.rs:2:16
+  --> $DIR/cast-to-unsized-type.rs:4:16
    |
 LL |     let _foo = &[1_usize, 2] as [usize];
    |                ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -10,7 +10,7 @@ LL |     let _foo = &[1_usize, 2] as &[usize];
    |                                 +
 
 error[E0620]: cast to unsized type: `Box` as `dyn Debug`
-  --> $DIR/issue-17441.rs:5:16
+  --> $DIR/cast-to-unsized-type.rs:7:16
    |
 LL |     let _bar = Box::new(1_usize) as dyn std::fmt::Debug;
    |                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -21,25 +21,25 @@ LL |     let _bar = Box::new(1_usize) as Box;
    |                                     ++++                   +
 
 error[E0620]: cast to unsized type: `usize` as `dyn Debug`
-  --> $DIR/issue-17441.rs:8:16
+  --> $DIR/cast-to-unsized-type.rs:10:16
    |
 LL |     let _baz = 1_usize as dyn std::fmt::Debug;
    |                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
 help: consider using a box or reference as appropriate
-  --> $DIR/issue-17441.rs:8:16
+  --> $DIR/cast-to-unsized-type.rs:10:16
    |
 LL |     let _baz = 1_usize as dyn std::fmt::Debug;
    |                ^^^^^^^
 
 error[E0620]: cast to unsized type: `[usize; 2]` as `[usize]`
-  --> $DIR/issue-17441.rs:11:17
+  --> $DIR/cast-to-unsized-type.rs:13:17
    |
 LL |     let _quux = [1_usize, 2] as [usize];
    |                 ^^^^^^^^^^^^^^^^^^^^^^^
    |
 help: consider using a box or reference as appropriate
-  --> $DIR/issue-17441.rs:11:17
+  --> $DIR/cast-to-unsized-type.rs:13:17
    |
 LL |     let _quux = [1_usize, 2] as [usize];
    |                 ^^^^^^^^^^^^
diff --git a/tests/ui/cast/non-primitive-isize-ref-cast.rs b/tests/ui/cast/non-primitive-isize-ref-cast.rs
index 0da7909480d2..95259456eede 100644
--- a/tests/ui/cast/non-primitive-isize-ref-cast.rs
+++ b/tests/ui/cast/non-primitive-isize-ref-cast.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 fn bad (p: *const isize) {
     let _q: &isize = p as &isize; //~ ERROR non-primitive cast
 }
diff --git a/tests/ui/cast/non-primitive-isize-ref-cast.stderr b/tests/ui/cast/non-primitive-isize-ref-cast.stderr
index f4a08e1751fc..3f4c171d3dc7 100644
--- a/tests/ui/cast/non-primitive-isize-ref-cast.stderr
+++ b/tests/ui/cast/non-primitive-isize-ref-cast.stderr
@@ -1,5 +1,5 @@
 error[E0605]: non-primitive cast: `*const isize` as `&isize`
-  --> $DIR/issue-2995.rs:2:22
+  --> $DIR/non-primitive-isize-ref-cast.rs:4:22
    |
 LL |     let _q: &isize = p as &isize;
    |                      ^^^^^^^^^^^ invalid cast
diff --git a/tests/ui/enum/enum-nonexisting-field.rs b/tests/ui/enum/enum-nonexisting-field.rs
index fede86f22afd..837430340f33 100644
--- a/tests/ui/enum/enum-nonexisting-field.rs
+++ b/tests/ui/enum/enum-nonexisting-field.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 enum Homura {
     Akemi { madoka: () }
 }
diff --git a/tests/ui/enum/enum-nonexisting-field.stderr b/tests/ui/enum/enum-nonexisting-field.stderr
index 0355d3a89710..22bfa08dadb3 100644
--- a/tests/ui/enum/enum-nonexisting-field.stderr
+++ b/tests/ui/enum/enum-nonexisting-field.stderr
@@ -1,5 +1,5 @@
 error[E0559]: variant `Homura::Akemi` has no field named `kaname`
-  --> $DIR/issue-19922.rs:6:34
+  --> $DIR/enum-nonexisting-field.rs:8:34
    |
 LL |     let homura = Homura::Akemi { kaname: () };
    |                                  ^^^^^^ `Homura::Akemi` does not have this field
diff --git a/tests/ui/lint/unused/unused-trait-fn.rs b/tests/ui/lint/unused/unused-trait-fn.rs
index 86049377198c..57b39c0de17e 100644
--- a/tests/ui/lint/unused/unused-trait-fn.rs
+++ b/tests/ui/lint/unused/unused-trait-fn.rs
@@ -1,3 +1,4 @@
+//! regression test for 
 //@ run-pass
 
 trait Str { fn foo(&self) {} } //~ WARN method `foo` is never used
diff --git a/tests/ui/lint/unused/unused-trait-fn.stderr b/tests/ui/lint/unused/unused-trait-fn.stderr
index 043d4ffc7808..f33fed29c94c 100644
--- a/tests/ui/lint/unused/unused-trait-fn.stderr
+++ b/tests/ui/lint/unused/unused-trait-fn.stderr
@@ -1,5 +1,5 @@
 warning: method `foo` is never used
-  --> $DIR/issue-17351.rs:3:16
+  --> $DIR/unused-trait-fn.rs:4:16
    |
 LL | trait Str { fn foo(&self) {} }
    |       ---      ^^^
diff --git a/tests/ui/lint/unused/unused-var-in-match-arm.rs b/tests/ui/lint/unused/unused-var-in-match-arm.rs
index 05096e5c1853..780225f98dbb 100644
--- a/tests/ui/lint/unused/unused-var-in-match-arm.rs
+++ b/tests/ui/lint/unused/unused-var-in-match-arm.rs
@@ -1,3 +1,4 @@
+//! regression test for 
 #![deny(unused_variables)]
 
 fn f(_: i32) {}
diff --git a/tests/ui/lint/unused/unused-var-in-match-arm.stderr b/tests/ui/lint/unused/unused-var-in-match-arm.stderr
index b599f6febe31..a1b9849293ef 100644
--- a/tests/ui/lint/unused/unused-var-in-match-arm.stderr
+++ b/tests/ui/lint/unused/unused-var-in-match-arm.stderr
@@ -1,11 +1,11 @@
 error: unused variable: `a`
-  --> $DIR/issue-22599.rs:8:19
+  --> $DIR/unused-var-in-match-arm.rs:9:19
    |
 LL |     v = match 0 { a => 0 };
    |                   ^ help: if this is intentional, prefix it with an underscore: `_a`
    |
 note: the lint level is defined here
-  --> $DIR/issue-22599.rs:1:9
+  --> $DIR/unused-var-in-match-arm.rs:2:9
    |
 LL | #![deny(unused_variables)]
    |         ^^^^^^^^^^^^^^^^
diff --git a/tests/ui/methods/method-not-found-on-struct.rs b/tests/ui/methods/method-not-found-on-struct.rs
index 99eccc8a8175..b2a457ed19e5 100644
--- a/tests/ui/methods/method-not-found-on-struct.rs
+++ b/tests/ui/methods/method-not-found-on-struct.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 struct Homura;
 
 fn akemi(homura: Homura) {
diff --git a/tests/ui/methods/method-not-found-on-struct.stderr b/tests/ui/methods/method-not-found-on-struct.stderr
index 1e3d7a2e2f51..3bf775f30a7a 100644
--- a/tests/ui/methods/method-not-found-on-struct.stderr
+++ b/tests/ui/methods/method-not-found-on-struct.stderr
@@ -1,5 +1,5 @@
 error[E0599]: no method named `kaname` found for struct `Homura` in the current scope
-  --> $DIR/issue-19692.rs:4:40
+  --> $DIR/method-not-found-on-struct.rs:6:40
    |
 LL | struct Homura;
    | ------------- method `kaname` not found for this struct
diff --git a/tests/ui/pattern/enum-variant-not-found.rs b/tests/ui/pattern/enum-variant-not-found.rs
index 632ddb91b36f..e78e28abeb6e 100644
--- a/tests/ui/pattern/enum-variant-not-found.rs
+++ b/tests/ui/pattern/enum-variant-not-found.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 enum S {
     A,
 }
diff --git a/tests/ui/pattern/enum-variant-not-found.stderr b/tests/ui/pattern/enum-variant-not-found.stderr
index 83b40d0c0816..6db4cf79d6b3 100644
--- a/tests/ui/pattern/enum-variant-not-found.stderr
+++ b/tests/ui/pattern/enum-variant-not-found.stderr
@@ -1,5 +1,5 @@
 error[E0599]: no variant named `B` found for enum `S`
-  --> $DIR/issue-34209.rs:7:12
+  --> $DIR/enum-variant-not-found.rs:9:12
    |
 LL | enum S {
    | ------ variant `B` not found here
diff --git a/tests/ui/pattern/match-constant-and-byte-literal.rs b/tests/ui/pattern/match-constant-and-byte-literal.rs
index 99c85bcffcf8..7a793478016d 100644
--- a/tests/ui/pattern/match-constant-and-byte-literal.rs
+++ b/tests/ui/pattern/match-constant-and-byte-literal.rs
@@ -1,3 +1,4 @@
+//! regression test for 
 //@ run-pass
 fn main() {
     const X: u8 = 0;
diff --git a/tests/ui/privacy/auxiliary/imported-enum-is-private.rs b/tests/ui/privacy/auxiliary/imported-enum-is-private.rs
index 74abbf0bf8cd..ea847d9aff13 100644
--- a/tests/ui/privacy/auxiliary/imported-enum-is-private.rs
+++ b/tests/ui/privacy/auxiliary/imported-enum-is-private.rs
@@ -1,3 +1,5 @@
+//! auxiliary crate for 
+
 enum Foo {
     Bar(isize)
 }
diff --git a/tests/ui/privacy/imported-enum-is-private.rs b/tests/ui/privacy/imported-enum-is-private.rs
index 9f3dfebcc812..b628676a25c6 100644
--- a/tests/ui/privacy/imported-enum-is-private.rs
+++ b/tests/ui/privacy/imported-enum-is-private.rs
@@ -1,6 +1,7 @@
-//@ aux-build:issue-11680.rs
+//! regression test for 
+//@ aux-build:imported-enum-is-private.rs
 
-extern crate issue_11680 as other;
+extern crate imported_enum_is_private as other;
 
 fn main() {
     let _b = other::Foo::Bar(1);
diff --git a/tests/ui/privacy/imported-enum-is-private.stderr b/tests/ui/privacy/imported-enum-is-private.stderr
index 5bcf93de811f..cae1ebb0e29e 100644
--- a/tests/ui/privacy/imported-enum-is-private.stderr
+++ b/tests/ui/privacy/imported-enum-is-private.stderr
@@ -1,5 +1,5 @@
 error[E0603]: enum `Foo` is private
-  --> $DIR/issue-11680.rs:6:21
+  --> $DIR/imported-enum-is-private.rs:7:21
    |
 LL |     let _b = other::Foo::Bar(1);
    |                     ^^^  --- tuple variant `Bar` is not publicly re-exported
@@ -7,13 +7,13 @@ LL |     let _b = other::Foo::Bar(1);
    |                     private enum
    |
 note: the enum `Foo` is defined here
-  --> $DIR/auxiliary/issue-11680.rs:1:1
+  --> $DIR/auxiliary/imported-enum-is-private.rs:3:1
    |
 LL | enum Foo {
    | ^^^^^^^^
 
 error[E0603]: enum `Foo` is private
-  --> $DIR/issue-11680.rs:9:27
+  --> $DIR/imported-enum-is-private.rs:10:27
    |
 LL |     let _b = other::test::Foo::Bar(1);
    |                           ^^^  --- tuple variant `Bar` is not publicly re-exported
@@ -21,7 +21,7 @@ LL |     let _b = other::test::Foo::Bar(1);
    |                           private enum
    |
 note: the enum `Foo` is defined here
-  --> $DIR/auxiliary/issue-11680.rs:6:5
+  --> $DIR/auxiliary/imported-enum-is-private.rs:8:5
    |
 LL |     enum Foo {
    |     ^^^^^^^^
diff --git a/tests/ui/privacy/private-struct-field-in-module.rs b/tests/ui/privacy/private-struct-field-in-module.rs
index b100c59ad0bd..7ed8def1c691 100644
--- a/tests/ui/privacy/private-struct-field-in-module.rs
+++ b/tests/ui/privacy/private-struct-field-in-module.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 mod sub {
     pub struct S { len: usize }
     impl S {
diff --git a/tests/ui/privacy/private-struct-field-in-module.stderr b/tests/ui/privacy/private-struct-field-in-module.stderr
index d7134bff1761..2394686f69ca 100644
--- a/tests/ui/privacy/private-struct-field-in-module.stderr
+++ b/tests/ui/privacy/private-struct-field-in-module.stderr
@@ -1,5 +1,5 @@
 error[E0616]: field `len` of struct `S` is private
-  --> $DIR/issue-26472.rs:11:15
+  --> $DIR/private-struct-field-in-module.rs:13:15
    |
 LL |     let v = s.len;
    |               ^^^ private field
@@ -10,7 +10,7 @@ LL |     let v = s.len();
    |                  ++
 
 error[E0616]: field `len` of struct `S` is private
-  --> $DIR/issue-26472.rs:12:7
+  --> $DIR/private-struct-field-in-module.rs:14:7
    |
 LL |     s.len = v;
    |       ^^^ private field
diff --git a/tests/ui/type-inference/panic-with-unspecified-type.rs b/tests/ui/type-inference/panic-with-unspecified-type.rs
index 66a3fadac8d9..0f96759e8144 100644
--- a/tests/ui/type-inference/panic-with-unspecified-type.rs
+++ b/tests/ui/type-inference/panic-with-unspecified-type.rs
@@ -1,3 +1,4 @@
+//! regression test for 
 //@ edition:2015..2021
 fn main() {
     panic!(std::default::Default::default());
diff --git a/tests/ui/type-inference/panic-with-unspecified-type.stderr b/tests/ui/type-inference/panic-with-unspecified-type.stderr
index e294d8830de0..5f08a7552632 100644
--- a/tests/ui/type-inference/panic-with-unspecified-type.stderr
+++ b/tests/ui/type-inference/panic-with-unspecified-type.stderr
@@ -1,5 +1,5 @@
 error[E0283]: type annotations needed
-  --> $DIR/issue-16966.rs:3:12
+  --> $DIR/panic-with-unspecified-type.rs:4:12
    |
 LL |     panic!(std::default::Default::default());
    |     -------^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-
diff --git a/tests/ui/type-inference/send-with-unspecified-type.rs b/tests/ui/type-inference/send-with-unspecified-type.rs
index 4be83457f7a8..4c2de025d69c 100644
--- a/tests/ui/type-inference/send-with-unspecified-type.rs
+++ b/tests/ui/type-inference/send-with-unspecified-type.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 use std::sync::mpsc::channel;
 use std::thread::spawn;
 use std::marker::PhantomData;
diff --git a/tests/ui/type-inference/send-with-unspecified-type.stderr b/tests/ui/type-inference/send-with-unspecified-type.stderr
index 23f1441e69dc..85692e8ad0cd 100644
--- a/tests/ui/type-inference/send-with-unspecified-type.stderr
+++ b/tests/ui/type-inference/send-with-unspecified-type.stderr
@@ -1,5 +1,5 @@
 error[E0282]: type annotations needed
-  --> $DIR/issue-25368.rs:11:27
+  --> $DIR/send-with-unspecified-type.rs:13:27
    |
 LL |         tx.send(Foo{ foo: PhantomData });
    |                           ^^^^^^^^^^^ cannot infer type of the type parameter `T` declared on the struct `PhantomData`
diff --git a/tests/ui/type-inference/swap-with-unspecified-type.rs b/tests/ui/type-inference/swap-with-unspecified-type.rs
index c6d301671272..db415df19e99 100644
--- a/tests/ui/type-inference/swap-with-unspecified-type.rs
+++ b/tests/ui/type-inference/swap-with-unspecified-type.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 fn main() {
     use std::mem::{transmute, swap};
     let a = 1;
diff --git a/tests/ui/type-inference/swap-with-unspecified-type.stderr b/tests/ui/type-inference/swap-with-unspecified-type.stderr
index 37a86ecc5437..eaaed559ebf1 100644
--- a/tests/ui/type-inference/swap-with-unspecified-type.stderr
+++ b/tests/ui/type-inference/swap-with-unspecified-type.stderr
@@ -1,5 +1,5 @@
 error[E0282]: type annotations needed
-  --> $DIR/issue-24013.rs:5:13
+  --> $DIR/swap-with-unspecified-type.rs:7:13
    |
 LL |     unsafe {swap::<&mut _>(transmute(&a), transmute(&b))};
    |             ^^^^^^^^^^^^^^ cannot infer type of the type parameter `T` declared on the function `swap`
diff --git a/tests/ui/type/struct-constructor-as-value.rs b/tests/ui/type/struct-constructor-as-value.rs
index 2fa762475da9..9cac1caf86ec 100644
--- a/tests/ui/type/struct-constructor-as-value.rs
+++ b/tests/ui/type/struct-constructor-as-value.rs
@@ -1,3 +1,5 @@
+//! regression test for 
+
 struct Foo(u32);
 
 fn test() -> Foo { Foo } //~ ERROR mismatched types
diff --git a/tests/ui/type/struct-constructor-as-value.stderr b/tests/ui/type/struct-constructor-as-value.stderr
index 6f6602793fdb..5915f971b324 100644
--- a/tests/ui/type/struct-constructor-as-value.stderr
+++ b/tests/ui/type/struct-constructor-as-value.stderr
@@ -1,5 +1,5 @@
 error[E0308]: mismatched types
-  --> $DIR/issue-35241.rs:3:20
+  --> $DIR/struct-constructor-as-value.rs:5:20
    |
 LL | struct Foo(u32);
    | ---------- `Foo` defines a struct constructor here, which should be called

From db9f9e65013419e87af78c54727f1b4f44bf75b2 Mon Sep 17 00:00:00 2001
From: mu001999 
Date: Sun, 18 Jan 2026 13:38:19 +0800
Subject: [PATCH 0338/1843] Use find_attr instead of attr::contains_name in
 lower_const_item_rhs

---
 compiler/rustc_ast_lowering/src/lib.rs        |  2 +-
 .../type-const-assoc-const-without-body.rs    | 19 +++++++++++++++++++
 ...type-const-assoc-const-without-body.stderr | 10 ++++++++++
 ...const-inherent-assoc-const-without-body.rs | 12 ++++++++++++
 ...t-inherent-assoc-const-without-body.stderr | 17 +++++++++++++++++
 5 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs
 create mode 100644 tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr
 create mode 100644 tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs
 create mode 100644 tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 51d1fd20cec6..8c6ee9d6bc63 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -2384,7 +2384,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             Some(ConstItemRhs::TypeConst(anon)) => {
                 hir::ConstItemRhs::TypeConst(self.lower_anon_const_to_const_arg_and_alloc(anon))
             }
-            None if attr::contains_name(attrs, sym::type_const) => {
+            None if find_attr!(attrs, AttributeKind::TypeConst(_)) => {
                 let const_arg = ConstArg {
                     hir_id: self.next_id(),
                     kind: hir::ConstArgKind::Error(
diff --git a/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs
new file mode 100644
index 000000000000..158a7addd10d
--- /dev/null
+++ b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs
@@ -0,0 +1,19 @@
+//@ needs-rustc-debug-assertions
+
+#![feature(min_generic_const_args)]
+#![expect(incomplete_features)]
+
+trait Tr {
+    #[type_const]
+    const SIZE: usize;
+}
+
+struct T;
+
+impl Tr for T {
+    #[type_const]
+    const SIZE: usize;
+    //~^ ERROR associated constant in `impl` without body
+}
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr
new file mode 100644
index 000000000000..2db677aa0ca6
--- /dev/null
+++ b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr
@@ -0,0 +1,10 @@
+error: associated constant in `impl` without body
+  --> $DIR/type-const-assoc-const-without-body.rs:15:5
+   |
+LL |     const SIZE: usize;
+   |     ^^^^^^^^^^^^^^^^^-
+   |                      |
+   |                      help: provide a definition for the constant: `= ;`
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs
new file mode 100644
index 000000000000..85b2327d3351
--- /dev/null
+++ b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs
@@ -0,0 +1,12 @@
+//@ needs-rustc-debug-assertions
+
+#![feature(min_generic_const_args)]
+#![expect(incomplete_features)]
+
+impl S { //~ ERROR cannot find type `S` in this scope
+    #[type_const]
+    const SIZE: usize;
+    //~^ ERROR associated constant in `impl` without body
+}
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr
new file mode 100644
index 000000000000..ac520a4e6946
--- /dev/null
+++ b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr
@@ -0,0 +1,17 @@
+error: associated constant in `impl` without body
+  --> $DIR/type-const-inherent-assoc-const-without-body.rs:8:5
+   |
+LL |     const SIZE: usize;
+   |     ^^^^^^^^^^^^^^^^^-
+   |                      |
+   |                      help: provide a definition for the constant: `= ;`
+
+error[E0425]: cannot find type `S` in this scope
+  --> $DIR/type-const-inherent-assoc-const-without-body.rs:6:6
+   |
+LL | impl S {
+   |      ^ not found in this scope
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0425`.

From eed703ccc0d534f38e5d7eca30c6675364566b69 Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Sun, 18 Jan 2026 08:22:33 +0000
Subject: [PATCH 0339/1843] Fix typo.

Match "build-rust-analyzer" in src/building/how-to-build-and-run.md and in the default editor settings from the rustc repo.
---
 src/doc/rustc-dev-guide/src/building/suggested.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doc/rustc-dev-guide/src/building/suggested.md b/src/doc/rustc-dev-guide/src/building/suggested.md
index 1c75de3e9042..c87dc6b28d87 100644
--- a/src/doc/rustc-dev-guide/src/building/suggested.md
+++ b/src/doc/rustc-dev-guide/src/building/suggested.md
@@ -97,7 +97,7 @@ for two reasons:
   additional rebuilds in some cases.
 
 To avoid these problems:
-- Add `--build-dir=build/rust-analyzer` to all of the custom `x` commands in
+- Add `--build-dir=build-rust-analyzer` to all of the custom `x` commands in
   your editor's rust-analyzer configuration.
   (Feel free to choose a different directory name if desired.)
 - Modify the `rust-analyzer.rustfmt.overrideCommand` setting so that it points

From cbad6dd11772b3d02a4eaca4e049f66907bb72a1 Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Fri, 16 Jan 2026 16:23:56 +0100
Subject: [PATCH 0340/1843] fix: Do not show sysroot dependencies in symbol
 search

---
 src/tools/rust-analyzer/crates/base-db/src/input.rs       | 3 ++-
 src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs | 8 +++++++-
 .../rust-analyzer/crates/project-model/src/workspace.rs   | 6 +++++-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/base-db/src/input.rs b/src/tools/rust-analyzer/crates/base-db/src/input.rs
index 240f1264917a..94793a3618e1 100644
--- a/src/tools/rust-analyzer/crates/base-db/src/input.rs
+++ b/src/tools/rust-analyzer/crates/base-db/src/input.rs
@@ -221,6 +221,7 @@ pub enum LangCrateOrigin {
     ProcMacro,
     Std,
     Test,
+    Dependency,
     Other,
 }
 
@@ -245,7 +246,7 @@ impl fmt::Display for LangCrateOrigin {
             LangCrateOrigin::ProcMacro => "proc_macro",
             LangCrateOrigin::Std => "std",
             LangCrateOrigin::Test => "test",
-            LangCrateOrigin::Other => "other",
+            LangCrateOrigin::Other | LangCrateOrigin::Dependency => "other",
         };
         f.write_str(text)
     }
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
index d7f4c66f465b..183f6b649537 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
+++ b/src/tools/rust-analyzer/crates/ide-db/src/symbol_index.rs
@@ -27,7 +27,7 @@ use std::{
     ops::ControlFlow,
 };
 
-use base_db::{LibraryRoots, LocalRoots, RootQueryDb, SourceRootId};
+use base_db::{CrateOrigin, LangCrateOrigin, LibraryRoots, LocalRoots, RootQueryDb, SourceRootId};
 use fst::{Automaton, Streamer, raw::IndexedValue};
 use hir::{
     Crate, Module,
@@ -446,6 +446,12 @@ impl<'db> SymbolIndex<'db> {
                     {
                         continue;
                     }
+                    if let CrateOrigin::Lang(LangCrateOrigin::Dependency | LangCrateOrigin::Other) =
+                        krate.origin(db)
+                    {
+                        // don't show dependencies of the sysroot
+                        continue;
+                    }
                     collector.push_crate_root(krate);
                 }
 
diff --git a/src/tools/rust-analyzer/crates/project-model/src/workspace.rs b/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
index fa3a79e041e0..8f15f7e1507c 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
@@ -1161,6 +1161,8 @@ fn project_json_to_crate_graph(
                                 name: Some(name.canonical_name().to_owned()),
                             }
                         }
+                    } else if is_sysroot {
+                        CrateOrigin::Lang(LangCrateOrigin::Dependency)
                     } else {
                         CrateOrigin::Local { repo: None, name: None }
                     },
@@ -1294,6 +1296,8 @@ fn cargo_to_crate_graph(
                             name: Some(Symbol::intern(&pkg_data.name)),
                         }
                     }
+                } else if cargo.is_sysroot() {
+                    CrateOrigin::Lang(LangCrateOrigin::Dependency)
                 } else {
                     CrateOrigin::Library {
                         repo: pkg_data.repository.clone(),
@@ -1717,7 +1721,7 @@ fn extend_crate_graph_with_sysroot(
                     !matches!(lang_crate, LangCrateOrigin::Test | LangCrateOrigin::Alloc),
                 )),
                 LangCrateOrigin::ProcMacro => libproc_macro = Some(cid),
-                LangCrateOrigin::Other => (),
+                LangCrateOrigin::Other | LangCrateOrigin::Dependency => (),
             }
         }
     }

From bfbee86a2de8e281db70b08367ce8245082431a2 Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Fri, 16 Jan 2026 13:35:31 +0100
Subject: [PATCH 0341/1843] feat: Trigger flycheck if non-workspace files get
 modified

Co-authored-by: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com>
Co-authored-by: dino 
---
 .../crates/rust-analyzer/src/config.rs              |  6 ++++++
 .../rust-analyzer/src/handlers/notification.rs      | 13 +++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index 28ac94e4deb6..7382edfa96f7 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -1043,6 +1043,7 @@ pub struct Config {
     /// The workspace roots as registered by the LSP client
     workspace_roots: Vec,
     caps: ClientCapabilities,
+    /// The LSP root path, deprecated in favor of `workspace_roots`
     root_path: AbsPathBuf,
     snippets: Vec,
     client_info: Option,
@@ -1366,6 +1367,10 @@ impl Config {
 
         self.discovered_projects_from_command.push(ProjectJsonFromCommand { data, buildfile });
     }
+
+    pub fn workspace_roots(&self) -> &[AbsPathBuf] {
+        &self.workspace_roots
+    }
 }
 
 #[derive(Default, Debug)]
@@ -1742,6 +1747,7 @@ impl Config {
     }
 
     pub fn root_path(&self) -> &AbsPathBuf {
+        // We should probably use `workspace_roots` here if set
         &self.root_path
     }
 
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
index 6cc40677fb51..138310b78f62 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
@@ -289,11 +289,24 @@ pub(crate) fn handle_did_change_watched_files(
     state: &mut GlobalState,
     params: DidChangeWatchedFilesParams,
 ) -> anyhow::Result<()> {
+    // we want to trigger flycheck if a file outside of our workspaces has changed,
+    // as to reduce stale diagnostics when outside changes happen
+    let mut trigger_flycheck = false;
     for change in params.changes.iter().unique_by(|&it| &it.uri) {
         if let Ok(path) = from_proto::abs_path(&change.uri) {
+            if !trigger_flycheck {
+                trigger_flycheck =
+                    state.config.workspace_roots().iter().any(|root| !path.starts_with(root));
+            }
             state.loader.handle.invalidate(path);
         }
     }
+
+    if trigger_flycheck && state.config.check_on_save(None) {
+        for flycheck in state.flycheck.iter() {
+            flycheck.restart_workspace(None);
+        }
+    }
     Ok(())
 }
 

From 97603c0df04af46512957dda2b5d9dbba73353e3 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sun, 18 Jan 2026 18:00:08 +1100
Subject: [PATCH 0342/1843] Use `ty::Value` in more places throughout
 `const_to_pat`

---
 .../src/thir/pattern/const_to_pat.rs          | 85 +++++++++----------
 1 file changed, 39 insertions(+), 46 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index 70bc142131e4..7fbf8cd11466 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -13,9 +13,7 @@ use rustc_infer::traits::Obligation;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::span_bug;
 use rustc_middle::thir::{FieldPat, Pat, PatKind};
-use rustc_middle::ty::{
-    self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitableExt, TypeVisitor, ValTree,
-};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitableExt, TypeVisitor};
 use rustc_span::def_id::DefId;
 use rustc_span::{DUMMY_SP, Span};
 use rustc_trait_selection::traits::ObligationCause;
@@ -48,7 +46,7 @@ impl<'tcx> PatCtxt<'tcx> {
 
         match c.kind() {
             ty::ConstKind::Unevaluated(uv) => convert.unevaluated_to_pat(uv, ty),
-            ty::ConstKind::Value(cv) => convert.valtree_to_pat(cv.valtree, cv.ty),
+            ty::ConstKind::Value(value) => convert.valtree_to_pat(value),
             _ => span_bug!(span, "Invalid `ConstKind` for `const_to_pat`: {:?}", c),
         }
     }
@@ -175,7 +173,7 @@ impl<'tcx> ConstToPat<'tcx> {
         };
 
         // Lower the valtree to a THIR pattern.
-        let mut thir_pat = self.valtree_to_pat(valtree, ty);
+        let mut thir_pat = self.valtree_to_pat(ty::Value { ty, valtree });
 
         if !thir_pat.references_error() {
             // Always check for `PartialEq` if we had no other errors yet.
@@ -192,31 +190,32 @@ impl<'tcx> ConstToPat<'tcx> {
         thir_pat
     }
 
-    fn field_pats(
+    fn lower_field_values_to_fieldpats(
         &self,
-        vals: impl Iterator, Ty<'tcx>)>,
+        values: impl Iterator>,
     ) -> Vec> {
-        vals.enumerate()
-            .map(|(idx, (val, ty))| {
-                let field = FieldIdx::new(idx);
-                // Patterns can only use monomorphic types.
-                let ty = self.tcx.normalize_erasing_regions(self.typing_env, ty);
-                FieldPat { field, pattern: *self.valtree_to_pat(val, ty) }
+        values
+            .enumerate()
+            .map(|(index, value)| FieldPat {
+                field: FieldIdx::new(index),
+                pattern: *self.valtree_to_pat(value),
             })
             .collect()
     }
 
     // Recursive helper for `to_pat`; invoke that (instead of calling this directly).
-    // FIXME(valtrees): Accept `ty::Value` instead of `Ty` and `ty::ValTree` separately.
     #[instrument(skip(self), level = "debug")]
-    fn valtree_to_pat(&self, cv: ValTree<'tcx>, ty: Ty<'tcx>) -> Box> {
+    fn valtree_to_pat(&self, value: ty::Value<'tcx>) -> Box> {
         let span = self.span;
         let tcx = self.tcx;
+        let ty::Value { ty, valtree } = value;
+
         let kind = match ty.kind() {
+            // Extremely important check for all ADTs!
+            // Make sure they are eligible to be used in patterns, and if not, emit an error.
             ty::Adt(adt_def, _) if !self.type_marked_structural(ty) => {
-                // Extremely important check for all ADTs! Make sure they opted-in to be used in
-                // patterns.
-                debug!("adt_def {:?} has !type_marked_structural for cv.ty: {:?}", adt_def, ty);
+                // This ADT cannot be used as a constant in patterns.
+                debug!(?adt_def, ?value.ty, "ADT type in pattern is not `type_marked_structural`");
                 let PartialEqImplStatus {
                     is_derived, structural_partial_eq, non_blanket_impl, ..
                 } = type_has_partial_eq_impl(self.tcx, self.typing_env, ty);
@@ -239,51 +238,43 @@ impl<'tcx> ConstToPat<'tcx> {
                 return self.mk_err(tcx.dcx().create_err(err), ty);
             }
             ty::Adt(adt_def, args) if adt_def.is_enum() => {
-                let (&variant_index, fields) = cv.to_branch().split_first().unwrap();
+                let (&variant_index, fields) = valtree.to_branch().split_first().unwrap();
                 let variant_index = VariantIdx::from_u32(variant_index.to_leaf().to_u32());
                 PatKind::Variant {
                     adt_def: *adt_def,
                     args,
                     variant_index,
-                    subpatterns: self.field_pats(
-                        fields.iter().map(|ct| ct.to_value().valtree).zip(
-                            adt_def.variants()[variant_index]
-                                .fields
-                                .iter()
-                                .map(|field| field.ty(tcx, args)),
-                        ),
-                    ),
+                    subpatterns: self
+                        .lower_field_values_to_fieldpats(fields.iter().map(|ct| ct.to_value())),
                 }
             }
-            ty::Adt(def, args) => {
+            ty::Adt(def, _) => {
                 assert!(!def.is_union()); // Valtree construction would never succeed for unions.
                 PatKind::Leaf {
-                    subpatterns: self.field_pats(
-                        cv.to_branch().iter().map(|ct| ct.to_value().valtree).zip(
-                            def.non_enum_variant().fields.iter().map(|field| field.ty(tcx, args)),
-                        ),
+                    subpatterns: self.lower_field_values_to_fieldpats(
+                        valtree.to_branch().iter().map(|ct| ct.to_value()),
                     ),
                 }
             }
-            ty::Tuple(fields) => PatKind::Leaf {
-                subpatterns: self.field_pats(
-                    cv.to_branch().iter().map(|ct| ct.to_value().valtree).zip(fields.iter()),
+            ty::Tuple(_) => PatKind::Leaf {
+                subpatterns: self.lower_field_values_to_fieldpats(
+                    valtree.to_branch().iter().map(|ct| ct.to_value()),
                 ),
             },
-            ty::Slice(elem_ty) => PatKind::Slice {
-                prefix: cv
+            ty::Slice(_) => PatKind::Slice {
+                prefix: valtree
                     .to_branch()
                     .iter()
-                    .map(|val| *self.valtree_to_pat(val.to_value().valtree, *elem_ty))
+                    .map(|val| *self.valtree_to_pat(val.to_value()))
                     .collect(),
                 slice: None,
                 suffix: Box::new([]),
             },
-            ty::Array(elem_ty, _) => PatKind::Array {
-                prefix: cv
+            ty::Array(_, _) => PatKind::Array {
+                prefix: valtree
                     .to_branch()
                     .iter()
-                    .map(|val| *self.valtree_to_pat(val.to_value().valtree, *elem_ty))
+                    .map(|val| *self.valtree_to_pat(val.to_value()))
                     .collect(),
                 slice: None,
                 suffix: Box::new([]),
@@ -296,7 +287,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 // Under `feature(deref_patterns)`, string literal patterns can also
                 // have type `str` directly, without the `&`, in order to allow things
                 // like `deref!("...")` to work when the scrutinee is `String`.
-                PatKind::Constant { value: ty::Value { ty, valtree: cv } }
+                PatKind::Constant { value }
             }
             ty::Ref(_, pointee_ty, ..) => {
                 if pointee_ty.is_str()
@@ -304,7 +295,9 @@ impl<'tcx> ConstToPat<'tcx> {
                     || pointee_ty.is_sized(tcx, self.typing_env)
                 {
                     // References have the same valtree representation as their pointee.
-                    PatKind::Deref { subpattern: self.valtree_to_pat(cv, *pointee_ty) }
+                    PatKind::Deref {
+                        subpattern: self.valtree_to_pat(ty::Value { ty: *pointee_ty, valtree }),
+                    }
                 } else {
                     return self.mk_err(
                         tcx.dcx().create_err(UnsizedPattern { span, non_sm_ty: *pointee_ty }),
@@ -313,7 +306,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 }
             }
             ty::Float(flt) => {
-                let v = cv.to_leaf();
+                let v = valtree.to_leaf();
                 let is_nan = match flt {
                     ty::FloatTy::F16 => v.to_f16().is_nan(),
                     ty::FloatTy::F32 => v.to_f32().is_nan(),
@@ -325,13 +318,13 @@ impl<'tcx> ConstToPat<'tcx> {
                     // Also see .
                     return self.mk_err(tcx.dcx().create_err(NaNPattern { span }), ty);
                 } else {
-                    PatKind::Constant { value: ty::Value { ty, valtree: cv } }
+                    PatKind::Constant { value }
                 }
             }
             ty::Pat(..) | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::RawPtr(..) => {
                 // The raw pointers we see here have been "vetted" by valtree construction to be
                 // just integers, so we simply allow them.
-                PatKind::Constant { value: ty::Value { ty, valtree: cv } }
+                PatKind::Constant { value }
             }
             ty::FnPtr(..) => {
                 unreachable!(

From c5785601773223450e8d1b8d553cac32de01840f Mon Sep 17 00:00:00 2001
From: yukang 
Date: Fri, 28 Nov 2025 14:07:18 +0800
Subject: [PATCH 0343/1843] Surpress suggestion from unstable items on stable
 channel

---
 compiler/rustc_resolve/src/diagnostics.rs     |  1 +
 .../issue-149402-suggest-unresolve/foo.rs     |  6 ++++
 .../nightly.err                               | 18 ++++++++++++
 .../output.diff                               | 16 ++++++++++
 .../issue-149402-suggest-unresolve/rmake.rs   | 29 +++++++++++++++++++
 .../issue-149402-suggest-unresolve/stable.err |  9 ++++++
 6 files changed, 79 insertions(+)
 create mode 100644 tests/run-make/issue-149402-suggest-unresolve/foo.rs
 create mode 100644 tests/run-make/issue-149402-suggest-unresolve/nightly.err
 create mode 100644 tests/run-make/issue-149402-suggest-unresolve/output.diff
 create mode 100644 tests/run-make/issue-149402-suggest-unresolve/rmake.rs
 create mode 100644 tests/run-make/issue-149402-suggest-unresolve/stable.err

diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index d704280f3fa2..9fc32b31b32a 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -1610,6 +1610,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             }
         }
 
+        suggestions.retain(|suggestion| suggestion.is_stable || self.tcx.sess.is_nightly_build());
         suggestions
     }
 
diff --git a/tests/run-make/issue-149402-suggest-unresolve/foo.rs b/tests/run-make/issue-149402-suggest-unresolve/foo.rs
new file mode 100644
index 000000000000..8456990829b4
--- /dev/null
+++ b/tests/run-make/issue-149402-suggest-unresolve/foo.rs
@@ -0,0 +1,6 @@
+fn foo() {
+    let x = Vec::new();
+    x.push(Complete::Item { name: "hello" });
+}
+
+fn main() {}
diff --git a/tests/run-make/issue-149402-suggest-unresolve/nightly.err b/tests/run-make/issue-149402-suggest-unresolve/nightly.err
new file mode 100644
index 000000000000..8659f0170df3
--- /dev/null
+++ b/tests/run-make/issue-149402-suggest-unresolve/nightly.err
@@ -0,0 +1,18 @@
+error[E0433]: failed to resolve: use of undeclared type `Complete`
+ --> foo.rs:3:12
+  |
+3 |     x.push(Complete::Item { name: "hello" });
+  |            ^^^^^^^^ use of undeclared type `Complete`
+  |
+help: there is an enum variant `core::ops::CoroutineState::Complete` and 1 other; try using the variant's enum
+  |
+3 -     x.push(Complete::Item { name: "hello" });
+3 +     x.push(core::ops::CoroutineState::Item { name: "hello" });
+  |
+3 -     x.push(Complete::Item { name: "hello" });
+3 +     x.push(std::ops::CoroutineState::Item { name: "hello" });
+  |
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0433`.
diff --git a/tests/run-make/issue-149402-suggest-unresolve/output.diff b/tests/run-make/issue-149402-suggest-unresolve/output.diff
new file mode 100644
index 000000000000..196c0326a714
--- /dev/null
+++ b/tests/run-make/issue-149402-suggest-unresolve/output.diff
@@ -0,0 +1,16 @@
+@@ -3,6 +3,15 @@
+   |
+ 3 |     x.push(Complete::Item { name: "hello" });
+   |            ^^^^^^^^ use of undeclared type `Complete`
++  |
++help: there is an enum variant `core::ops::CoroutineState::Complete` and 1 other; try using the variant's enum
++  |
++3 -     x.push(Complete::Item { name: "hello" });
++3 +     x.push(core::ops::CoroutineState::Item { name: "hello" });
++  |
++3 -     x.push(Complete::Item { name: "hello" });
++3 +     x.push(std::ops::CoroutineState::Item { name: "hello" });
++  |
+ 
+ error: aborting due to 1 previous error
+ 
diff --git a/tests/run-make/issue-149402-suggest-unresolve/rmake.rs b/tests/run-make/issue-149402-suggest-unresolve/rmake.rs
new file mode 100644
index 000000000000..5bca0c0206cb
--- /dev/null
+++ b/tests/run-make/issue-149402-suggest-unresolve/rmake.rs
@@ -0,0 +1,29 @@
+//! Check that unstable name-resolution suggestions are omitted on stable.
+//!
+//! Regression test for .
+//!
+//@ only-nightly
+//@ needs-target-std
+
+use run_make_support::{diff, rustc, similar};
+
+fn main() {
+    let stable_like = rustc()
+        .env("RUSTC_BOOTSTRAP", "-1")
+        .edition("2024")
+        .input("foo.rs")
+        .run_fail()
+        .stderr_utf8();
+
+    assert!(!stable_like.contains("CoroutineState::Complete"));
+    diff().expected_file("stable.err").actual_text("stable_like", &stable_like).run();
+
+    let nightly = rustc().edition("2024").input("foo.rs").run_fail().stderr_utf8();
+
+    assert!(nightly.contains("CoroutineState::Complete"));
+    diff().expected_file("nightly.err").actual_text("nightly", &nightly).run();
+
+    let stderr_diff =
+        similar::TextDiff::from_lines(&stable_like, &nightly).unified_diff().to_string();
+    diff().expected_file("output.diff").actual_text("diff", stderr_diff).run();
+}
diff --git a/tests/run-make/issue-149402-suggest-unresolve/stable.err b/tests/run-make/issue-149402-suggest-unresolve/stable.err
new file mode 100644
index 000000000000..6e82fe1a67ea
--- /dev/null
+++ b/tests/run-make/issue-149402-suggest-unresolve/stable.err
@@ -0,0 +1,9 @@
+error[E0433]: failed to resolve: use of undeclared type `Complete`
+ --> foo.rs:3:12
+  |
+3 |     x.push(Complete::Item { name: "hello" });
+  |            ^^^^^^^^ use of undeclared type `Complete`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0433`.

From f3e73dced1144f3b7cc81b34c45fdafec66e29f2 Mon Sep 17 00:00:00 2001
From: Olivier Goffart 
Date: Sun, 30 Nov 2025 17:02:07 +0100
Subject: [PATCH 0344/1843] Supress some lookup errors if a module contains
 `compile_error!`

The problem is that when a macro expand to `compile_error!` because
its input is malformed, the actual error message from the
`compile_error!` might be hidden in a long list of other messages about
using items that should have otherwise been generated by the macro.

So suppress error about missing items in that module.

Fixes issue 68838
---
 .../rustc_builtin_macros/src/compile_error.rs |  1 +
 compiler/rustc_expand/src/base.rs             |  4 ++
 compiler/rustc_resolve/src/ident.rs           |  3 +-
 compiler/rustc_resolve/src/macros.rs          |  8 ++++
 .../compile_error_macro-suppress-errors.rs    | 40 ++++++++++++++++
 ...compile_error_macro-suppress-errors.stderr | 46 +++++++++++++++++++
 6 files changed, 101 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/macros/compile_error_macro-suppress-errors.rs
 create mode 100644 tests/ui/macros/compile_error_macro-suppress-errors.stderr

diff --git a/compiler/rustc_builtin_macros/src/compile_error.rs b/compiler/rustc_builtin_macros/src/compile_error.rs
index 7fc4b437c1d8..df64d8f314db 100644
--- a/compiler/rustc_builtin_macros/src/compile_error.rs
+++ b/compiler/rustc_builtin_macros/src/compile_error.rs
@@ -22,6 +22,7 @@ pub(crate) fn expand_compile_error<'cx>(
     #[expect(rustc::diagnostic_outside_of_impl, reason = "diagnostic message is specified by user")]
     #[expect(rustc::untranslatable_diagnostic, reason = "diagnostic message is specified by user")]
     let guar = cx.dcx().span_err(sp, var.to_string());
+    cx.resolver.mark_scope_with_compile_error(cx.current_expansion.lint_node_id);
 
     ExpandResult::Ready(DummyResult::any(sp, guar))
 }
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index bf653fac5253..a1fc9694d663 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -1173,6 +1173,10 @@ pub trait ResolverExpand {
     /// Record the name of an opaque `Ty::ImplTrait` pre-expansion so that it can be used
     /// to generate an item name later that does not reference placeholder macros.
     fn insert_impl_trait_name(&mut self, id: NodeId, name: Symbol);
+
+    /// Mark the scope as having a compile error so that error for lookup in this scope
+    /// should be suppressed
+    fn mark_scope_with_compile_error(&mut self, parent_node: NodeId);
 }
 
 pub trait LintStoreExpand {
diff --git a/compiler/rustc_resolve/src/ident.rs b/compiler/rustc_resolve/src/ident.rs
index f7e628048ccd..79d08828ccc4 100644
--- a/compiler/rustc_resolve/src/ident.rs
+++ b/compiler/rustc_resolve/src/ident.rs
@@ -1711,7 +1711,8 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         diag_metadata: Option<&DiagMetadata<'_>>,
     ) -> PathResult<'ra> {
         let mut module = None;
-        let mut module_had_parse_errors = false;
+        let mut module_had_parse_errors = !self.mods_with_parse_errors.is_empty()
+            && self.mods_with_parse_errors.contains(&parent_scope.module.nearest_parent_mod());
         let mut allow_super = true;
         let mut second_binding = None;
 
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index 38628332f43e..7246eded2e72 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -166,6 +166,14 @@ impl<'ra, 'tcx> ResolverExpand for Resolver<'ra, 'tcx> {
         self.invocation_parents[&id].parent_def
     }
 
+    fn mark_scope_with_compile_error(&mut self, id: NodeId) {
+        if let Some(id) = self.opt_local_def_id(id)
+            && self.tcx.def_kind(id).is_module_like()
+        {
+            self.mods_with_parse_errors.insert(id.to_def_id());
+        }
+    }
+
     fn resolve_dollar_crates(&self) {
         hygiene::update_dollar_crate_names(|ctxt| {
             let ident = Ident::new(kw::DollarCrate, DUMMY_SP.with_ctxt(ctxt));
diff --git a/tests/ui/macros/compile_error_macro-suppress-errors.rs b/tests/ui/macros/compile_error_macro-suppress-errors.rs
new file mode 100644
index 000000000000..b2b6a8ae0088
--- /dev/null
+++ b/tests/ui/macros/compile_error_macro-suppress-errors.rs
@@ -0,0 +1,40 @@
+pub mod some_module {
+    compile_error!("Error in a module"); //~ ERROR: Error in a module
+
+    fn abc() -> Hello {
+        let _: self::SomeType = self::Hello::new();
+        let _: SomeType = Hello::new();
+    }
+
+    mod inner_module {
+        use super::Hello;
+        use crate::another_module::NotExist;  //~ ERROR: unresolved import `crate::another_module::NotExist`
+        use crate::some_module::World;
+        struct Foo {
+            bar: crate::some_module::Xyz,
+            error: self::MissingType, //~ ERROR: cannot find type `MissingType` in module `self`
+        }
+    }
+}
+
+pub mod another_module {
+    use crate::some_module::NotExist;
+    fn error_in_this_function() {
+        compile_error!("Error in a function"); //~ ERROR: Error in a function
+    }
+}
+
+fn main() {
+    // these errors are suppressed because of the compile_error! macro
+
+    let _ = some_module::some_function();
+    let _: some_module::SomeType = some_module::Hello::new();
+
+    // these errors are not suppressed
+
+    let _ = another_module::some_function();
+    //~^ ERROR: cannot find function `some_function` in module `another_module`
+    let _: another_module::SomeType = another_module::Hello::new();
+    //~^ ERROR: cannot find type `SomeType` in module `another_module`
+    //~^^ ERROR: failed to resolve: could not find `Hello` in `another_module`
+}
diff --git a/tests/ui/macros/compile_error_macro-suppress-errors.stderr b/tests/ui/macros/compile_error_macro-suppress-errors.stderr
new file mode 100644
index 000000000000..73b156359624
--- /dev/null
+++ b/tests/ui/macros/compile_error_macro-suppress-errors.stderr
@@ -0,0 +1,46 @@
+error: Error in a module
+  --> $DIR/compile_error_macro-suppress-errors.rs:2:5
+   |
+LL |     compile_error!("Error in a module");
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: Error in a function
+  --> $DIR/compile_error_macro-suppress-errors.rs:23:9
+   |
+LL |         compile_error!("Error in a function");
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error[E0432]: unresolved import `crate::another_module::NotExist`
+  --> $DIR/compile_error_macro-suppress-errors.rs:11:13
+   |
+LL |         use crate::another_module::NotExist;
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no `NotExist` in `another_module`
+
+error[E0433]: failed to resolve: could not find `Hello` in `another_module`
+  --> $DIR/compile_error_macro-suppress-errors.rs:37:55
+   |
+LL |     let _: another_module::SomeType = another_module::Hello::new();
+   |                                                       ^^^^^ could not find `Hello` in `another_module`
+
+error[E0425]: cannot find type `MissingType` in module `self`
+  --> $DIR/compile_error_macro-suppress-errors.rs:15:26
+   |
+LL |             error: self::MissingType,
+   |                          ^^^^^^^^^^^ not found in `self`
+
+error[E0425]: cannot find function `some_function` in module `another_module`
+  --> $DIR/compile_error_macro-suppress-errors.rs:35:29
+   |
+LL |     let _ = another_module::some_function();
+   |                             ^^^^^^^^^^^^^ not found in `another_module`
+
+error[E0425]: cannot find type `SomeType` in module `another_module`
+  --> $DIR/compile_error_macro-suppress-errors.rs:37:28
+   |
+LL |     let _: another_module::SomeType = another_module::Hello::new();
+   |                            ^^^^^^^^ not found in `another_module`
+
+error: aborting due to 7 previous errors
+
+Some errors have detailed explanations: E0425, E0432, E0433.
+For more information about an error, try `rustc --explain E0425`.

From 2ef85d7894ff46efdd6ab49393ecc88aaf60b0e5 Mon Sep 17 00:00:00 2001
From: Ralf Jung 
Date: Sun, 18 Jan 2026 15:30:05 +0100
Subject: [PATCH 0345/1843] use epoll_ctl_add more often

---
 .../pass-dep/libc/libc-epoll-blocking.rs      | 50 ++++------
 .../pass-dep/libc/libc-epoll-no-blocking.rs   | 94 +++++++------------
 2 files changed, 49 insertions(+), 95 deletions(-)

diff --git a/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs b/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs
index d5a59796de02..f9615fc6e414 100644
--- a/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs
+++ b/src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs
@@ -21,9 +21,6 @@ fn main() {
     multiple_events_wake_multiple_threads();
 }
 
-// Using `as` cast since `EPOLLET` wraps around
-const EPOLL_IN_OUT_ET: i32 = (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _;
-
 // This test allows epoll_wait to block, then unblock without notification.
 fn test_epoll_block_without_notification() {
     // Create an epoll instance.
@@ -34,10 +31,10 @@ fn test_epoll_block_without_notification() {
     let fd = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap();
 
     // Register eventfd with epoll.
-    epoll_ctl_add(epfd, fd, EPOLL_IN_OUT_ET).unwrap();
+    epoll_ctl_add(epfd, fd, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // epoll_wait to clear notification.
-    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT as _, data: fd }], 0);
+    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT, data: fd }], 0);
 
     // This epoll wait blocks, and timeout without notification.
     check_epoll_wait::<1>(epfd, &[], 5);
@@ -53,21 +50,17 @@ fn test_epoll_block_then_unblock() {
     errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) });
 
     // Register one side of the socketpair with epoll.
-    epoll_ctl_add(epfd, fds[0], EPOLL_IN_OUT_ET).unwrap();
+    epoll_ctl_add(epfd, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // epoll_wait to clear notification.
-    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT as _, data: fds[0] }], 0);
+    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT, data: fds[0] }], 0);
 
     // epoll_wait before triggering notification so it will block then get unblocked before timeout.
     let thread1 = thread::spawn(move || {
         thread::yield_now();
         write_all_from_slice(fds[1], b"abcde").unwrap();
     });
-    check_epoll_wait::<1>(
-        epfd,
-        &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fds[0] }],
-        10,
-    );
+    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLIN | libc::EPOLLOUT, data: fds[0] }], 10);
     thread1.join().unwrap();
 }
 
@@ -81,10 +74,10 @@ fn test_notification_after_timeout() {
     errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) });
 
     // Register one side of the socketpair with epoll.
-    epoll_ctl_add(epfd, fds[0], EPOLL_IN_OUT_ET).unwrap();
+    epoll_ctl_add(epfd, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // epoll_wait to clear notification.
-    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT as _, data: fds[0] }], 0);
+    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLOUT, data: fds[0] }], 0);
 
     // epoll_wait timeouts without notification.
     check_epoll_wait::<1>(epfd, &[], 10);
@@ -93,11 +86,7 @@ fn test_notification_after_timeout() {
     write_all_from_slice(fds[1], b"abcde").unwrap();
 
     // Check the result of the notification.
-    check_epoll_wait::<1>(
-        epfd,
-        &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fds[0] }],
-        10,
-    );
+    check_epoll_wait::<1>(epfd, &[Ev { events: libc::EPOLLIN | libc::EPOLLOUT, data: fds[0] }], 10);
 }
 
 // This test shows a data_race before epoll had vector clocks added.
@@ -110,7 +99,7 @@ fn test_epoll_race() {
     let fd = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap();
 
     // Register eventfd with the epoll instance.
-    epoll_ctl_add(epfd, fd, EPOLL_IN_OUT_ET).unwrap();
+    epoll_ctl_add(epfd, fd, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     static mut VAL: u8 = 0;
     let thread1 = thread::spawn(move || {
@@ -121,11 +110,7 @@ fn test_epoll_race() {
     });
     thread::yield_now();
     // epoll_wait for the event to happen.
-    check_epoll_wait::<8>(
-        epfd,
-        &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fd }],
-        -1,
-    );
+    check_epoll_wait::<8>(epfd, &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT), data: fd }], -1);
     // Read from the static mut variable.
     #[allow(static_mut_refs)]
     unsafe {
@@ -151,7 +136,7 @@ fn wakeup_on_new_interest() {
     let t = std::thread::spawn(move || {
         check_epoll_wait::<8>(
             epfd,
-            &[Ev { events: (libc::EPOLLIN | libc::EPOLLOUT) as _, data: fds[1] }],
+            &[Ev { events: libc::EPOLLIN | libc::EPOLLOUT, data: fds[1] }],
             -1,
         );
     });
@@ -159,7 +144,8 @@ fn wakeup_on_new_interest() {
     std::thread::yield_now();
 
     // Register fd[1] with EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP
-    epoll_ctl_add(epfd, fds[1], EPOLL_IN_OUT_ET | libc::EPOLLRDHUP as i32).unwrap();
+    epoll_ctl_add(epfd, fds[1], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET | libc::EPOLLRDHUP)
+        .unwrap();
 
     // This should wake up the thread.
     t.join().unwrap();
@@ -178,14 +164,12 @@ fn multiple_events_wake_multiple_threads() {
     let fd2 = errno_result(unsafe { libc::dup(fd1) }).unwrap();
 
     // Register both with epoll.
-    epoll_ctl_add(epfd, fd1, EPOLL_IN_OUT_ET).unwrap();
-    epoll_ctl_add(epfd, fd2, EPOLL_IN_OUT_ET).unwrap();
+    epoll_ctl_add(epfd, fd1, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
+    epoll_ctl_add(epfd, fd2, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Consume the initial events.
-    let expected = [
-        Ev { events: libc::EPOLLOUT as _, data: fd1 },
-        Ev { events: libc::EPOLLOUT as _, data: fd2 },
-    ];
+    let expected =
+        [Ev { events: libc::EPOLLOUT, data: fd1 }, Ev { events: libc::EPOLLOUT, data: fd2 }];
     check_epoll_wait::<8>(epfd, &expected, -1);
 
     // Block two threads on the epoll, both wanting to get just one event.
diff --git a/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs b/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs
index 490895c8541b..63300c9a433c 100644
--- a/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs
+++ b/src/tools/miri/tests/pass-dep/libc/libc-epoll-no-blocking.rs
@@ -32,9 +32,6 @@ fn main() {
     test_issue_4374_reads();
 }
 
-// Using `as` cast since `EPOLLET` wraps around
-const EPOLL_IN_OUT_ET: u32 = (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _;
-
 #[track_caller]
 fn check_epoll_wait(epfd: i32, expected_notifications: &[(u32, u64)]) {
     let epoll_event = libc::epoll_event { events: 0, u64: 0 };
@@ -145,7 +142,10 @@ fn test_epoll_ctl_del() {
     assert_eq!(res, 5);
 
     // Register fd[1] with EPOLLIN|EPOLLOUT|EPOLLET
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: u64::try_from(fds[1]).unwrap() };
+    let mut ev = libc::epoll_event {
+        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as u32,
+        u64: u64::try_from(fds[1]).unwrap(),
+    };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
     assert_eq!(res, 0);
 
@@ -173,11 +173,8 @@ fn test_two_epoll_instance() {
     assert_eq!(res, 5);
 
     // Register one side of the socketpair with EPOLLIN | EPOLLOUT | EPOLLET.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: u64::try_from(fds[1]).unwrap() };
-    let res = unsafe { libc::epoll_ctl(epfd1, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
-    assert_eq!(res, 0);
-    let res = unsafe { libc::epoll_ctl(epfd2, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd1, fds[1], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
+    epoll_ctl_add(epfd2, fds[1], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Notification should be received from both instance of epoll.
     let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap();
@@ -201,7 +198,10 @@ fn test_two_same_fd_in_same_epoll_instance() {
     assert_ne!(newfd, -1);
 
     // Register both fd to the same epoll instance.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: 5 as u64 };
+    let mut ev = libc::epoll_event {
+        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).cast_unsigned(),
+        u64: 5u64,
+    };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
     assert_eq!(res, 0);
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, newfd, &mut ev) };
@@ -214,7 +214,7 @@ fn test_two_same_fd_in_same_epoll_instance() {
 
     // Two notification should be received.
     let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap();
-    let expected_value = 5 as u64;
+    let expected_value = 5u64;
     check_epoll_wait::<8>(
         epfd,
         &[(expected_event, expected_value), (expected_event, expected_value)],
@@ -233,9 +233,7 @@ fn test_epoll_eventfd() {
     let epfd = errno_result(unsafe { libc::epoll_create1(0) }).unwrap();
 
     // Register eventfd with EPOLLIN | EPOLLOUT | EPOLLET
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: u64::try_from(fd).unwrap() };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd, fd, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Check result from epoll_wait.
     let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap();
@@ -278,12 +276,8 @@ fn test_epoll_socketpair_both_sides() {
     errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) });
 
     // Register both fd to the same epoll instance.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_eq!(res, 0);
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[1] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
+    epoll_ctl_add(epfd, fds[1], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Write to fds[1].
     // (We do the write after the register here, unlike in `test_epoll_socketpair`, to ensure
@@ -326,9 +320,7 @@ fn test_closed_fd() {
     let fd = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap();
 
     // Register eventfd with EPOLLIN | EPOLLOUT | EPOLLET
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: u64::try_from(fd).unwrap() };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd, fd, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Write to the eventfd instance.
     let sized_8_data: [u8; 8] = 1_u64.to_ne_bytes();
@@ -360,9 +352,7 @@ fn test_not_fully_closed_fd() {
     let newfd = errno_result(unsafe { libc::dup(fd) }).unwrap();
 
     // Register eventfd with EPOLLIN | EPOLLOUT | EPOLLET
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: u64::try_from(fd).unwrap() };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd, fd, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Close the original fd that being used to register with epoll.
     errno_check(unsafe { libc::close(fd) });
@@ -402,7 +392,7 @@ fn test_event_overwrite() {
 
     // Register eventfd with EPOLLIN | EPOLLOUT | EPOLLET
     let mut ev = libc::epoll_event {
-        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _,
+        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).cast_unsigned(),
         u64: u64::try_from(fd).unwrap(),
     };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
@@ -431,13 +421,13 @@ fn test_socketpair_read() {
 
     // Register both fd to the same epoll instance.
     let mut ev = libc::epoll_event {
-        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _,
+        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).cast_unsigned(),
         u64: fds[0] as u64,
     };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
     assert_eq!(res, 0);
     let mut ev = libc::epoll_event {
-        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET) as _,
+        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).cast_unsigned(),
         u64: fds[1] as u64,
     };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
@@ -493,7 +483,7 @@ fn test_no_notification_for_unregister_flag() {
 
     // Register fd[0] with EPOLLOUT|EPOLLET.
     let mut ev = libc::epoll_event {
-        events: (libc::EPOLLOUT | libc::EPOLLET) as _,
+        events: (libc::EPOLLOUT | libc::EPOLLET).cast_unsigned(),
         u64: u64::try_from(fds[0]).unwrap(),
     };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
@@ -543,19 +533,15 @@ fn test_socketpair_epollerr() {
     errno_check(unsafe { libc::close(fds[1]) });
 
     // Register fd[1] with EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP
-    let mut ev = libc::epoll_event {
-        events: (libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET | libc::EPOLLRDHUP) as _,
-        u64: u64::try_from(fds[1]).unwrap(),
-    };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_ne!(res, -1);
+    epoll_ctl_add(epfd, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET | libc::EPOLLRDHUP)
+        .unwrap();
 
     // Check result from epoll_wait.
     let expected_event = u32::try_from(
         libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLHUP | libc::EPOLLRDHUP | libc::EPOLLERR,
     )
     .unwrap();
-    let expected_value = u64::try_from(fds[1]).unwrap();
+    let expected_value = u64::try_from(fds[0]).unwrap();
     check_epoll_wait::<8>(epfd, &[(expected_event, expected_value)]);
 }
 
@@ -570,12 +556,8 @@ fn test_epoll_lost_events() {
     errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) });
 
     // Register both fd to the same epoll instance.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_eq!(res, 0);
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[1] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fds[1], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
+    epoll_ctl_add(epfd, fds[1], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Two notification should be received. But we only provide buffer for one event.
     let expected_event0 = u32::try_from(libc::EPOLLOUT).unwrap();
@@ -601,12 +583,8 @@ fn test_ready_list_fetching_logic() {
     let fd1 = errno_result(unsafe { libc::eventfd(0, flags) }).unwrap();
 
     // Register both fd to the same epoll instance. At this point, both of them are on the ready list.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd0 as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd0, &mut ev) };
-    assert_eq!(res, 0);
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd1 as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd1, &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd, fd0, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
+    epoll_ctl_add(epfd, fd1, libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Close fd0 so the first entry in the ready list will be empty.
     errno_check(unsafe { libc::close(fd0) });
@@ -643,9 +621,7 @@ fn test_epoll_ctl_notification() {
     errno_check(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) });
 
     // Register one side of the socketpair with epoll.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd0, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd0, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // epoll_wait to clear notification for epfd0.
     let expected_event = u32::try_from(libc::EPOLLOUT).unwrap();
@@ -657,9 +633,7 @@ fn test_epoll_ctl_notification() {
     assert_ne!(epfd1, -1);
 
     // Register the same file description for epfd1.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd1, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd1, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
     check_epoll_wait::<1>(epfd1, &[(expected_event, expected_value)]);
 
     // Previously this epoll_wait will receive a notification, but we shouldn't return notification
@@ -683,7 +657,7 @@ fn test_issue_3858() {
 
     // Register eventfd with EPOLLIN | EPOLLET.
     let mut ev = libc::epoll_event {
-        events: (libc::EPOLLIN | libc::EPOLLET) as _,
+        events: (libc::EPOLLIN | libc::EPOLLET).cast_unsigned(),
         u64: u64::try_from(fd).unwrap(),
     };
     let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
@@ -715,9 +689,7 @@ fn test_issue_4374() {
     assert_eq!(unsafe { libc::fcntl(fds[1], libc::F_SETFL, libc::O_NONBLOCK) }, 0);
 
     // Register fds[0] with epoll while it is writable (but not readable).
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd0, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd0, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Fill up fds[0] so that it is not writable any more.
     let zeros = [0u8; 512];
@@ -754,9 +726,7 @@ fn test_issue_4374_reads() {
     assert_eq!(res, 5);
 
     // Register fds[0] with epoll while it is readable.
-    let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fds[0] as u64 };
-    let res = unsafe { libc::epoll_ctl(epfd0, libc::EPOLL_CTL_ADD, fds[0], &mut ev) };
-    assert_eq!(res, 0);
+    epoll_ctl_add(epfd0, fds[0], libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLET).unwrap();
 
     // Read fds[0] so it is no longer readable.
     let mut buf = [0u8; 512];

From b35f80f7f71336b1b22c5de4af9eb8e6b59cb8f5 Mon Sep 17 00:00:00 2001
From: oligamiq 
Date: Sun, 18 Jan 2026 23:37:51 +0900
Subject: [PATCH 0346/1843] fix: thread creation failed on the
 wasm32-wasip1-threads target.

---
 library/std/src/sys/thread/unix.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/std/src/sys/thread/unix.rs b/library/std/src/sys/thread/unix.rs
index 6f23c28c04d6..5a210703ce9b 100644
--- a/library/std/src/sys/thread/unix.rs
+++ b/library/std/src/sys/thread/unix.rs
@@ -49,7 +49,7 @@ impl Thread {
         // WASI does not support threading via pthreads. While wasi-libc provides
         // pthread stubs, pthread_create returns EAGAIN, which causes confusing
         // errors. We return UNSUPPORTED_PLATFORM directly instead.
-        if cfg!(target_os = "wasi") {
+        if cfg!(all(target_os = "wasi", not(target_feature = "atomics"))) {
             return Err(io::Error::UNSUPPORTED_PLATFORM);
         }
 

From d6362e6d0fa95ec0d6d5bdcd2b81516bf9e3a4b6 Mon Sep 17 00:00:00 2001
From: Ralf Jung 
Date: Wed, 17 Dec 2025 14:00:59 +0100
Subject: [PATCH 0347/1843] document guidelines for which shims have a place in
 Miri

---
 src/tools/miri/CONTRIBUTING.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/tools/miri/CONTRIBUTING.md b/src/tools/miri/CONTRIBUTING.md
index 1995300c5bcb..f9cb60c66d91 100644
--- a/src/tools/miri/CONTRIBUTING.md
+++ b/src/tools/miri/CONTRIBUTING.md
@@ -66,6 +66,23 @@ process for such contributions:
 This process is largely informal, and its primary goal is to more clearly communicate expectations.
 Please get in touch with us if you have any questions!
 
+## Scope of Miri shims
+
+Miri has "shims" to implement functionality that is usually implemented in C libraries which are
+invoked from Rust code, such as opening files or spawning threads, as well as for
+CPU-vendor-provided SIMD intrinsics. However, the set of C functions that Rust code invokes this way
+is enormous, and for obvious reasons we have no intention of implementing every C API ever written
+in Miri.
+
+At the moment, the general guideline for "could this function have a shim in Miri" is: we will
+generally only add shims for functions that can be implemented in a portable way using just what is
+provided by the Rust standard library. The function should also be reasonably widely-used in Rust
+code to justify the review and maintenance effort (i.e. the easier the function is to implement, the
+lower the barrier). Other than that, we might make exceptions for certain cases if (a) there is a
+good case for why Miri should support those APIs, and (b) robust and widely-used portable libraries
+exist in the Rust ecosystem. We will generally not add shims to Miri that would require Miri to
+directly interact with platform-specific APIs (such as `libc` or `windows-sys`).
+
 ## Preparing the build environment
 
 Miri heavily relies on internal and unstable rustc interfaces to execute MIR,

From ea77786cdb7e94006f74399ee67ff57068773375 Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Sun, 18 Jan 2026 16:48:45 +0000
Subject: [PATCH 0348/1843] Port #![no_main] to the attribute parser.

---
 .../src/attributes/crate_level.rs             |   9 ++
 compiler/rustc_attr_parsing/src/context.rs    |   7 +-
 .../rustc_hir/src/attrs/data_structures.rs    |   3 +
 .../rustc_hir/src/attrs/encode_cross_crate.rs |   1 +
 compiler/rustc_passes/src/check_attr.rs       |   1 +
 compiler/rustc_passes/src/entry.rs            |   5 +-
 .../issue-43106-gating-of-builtin-attrs.rs    |  12 +-
 ...issue-43106-gating-of-builtin-attrs.stderr | 131 ++++++++++--------
 8 files changed, 96 insertions(+), 73 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/crate_level.rs b/compiler/rustc_attr_parsing/src/attributes/crate_level.rs
index 5604fbd25edc..ab99186777f9 100644
--- a/compiler/rustc_attr_parsing/src/attributes/crate_level.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/crate_level.rs
@@ -136,6 +136,15 @@ impl NoArgsAttributeParser for NoStdParser {
     const CREATE: fn(Span) -> AttributeKind = AttributeKind::NoStd;
 }
 
+pub(crate) struct NoMainParser;
+
+impl NoArgsAttributeParser for NoMainParser {
+    const PATH: &[Symbol] = &[sym::no_main];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Crate)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::NoMain;
+}
+
 pub(crate) struct RustcCoherenceIsCoreParser;
 
 impl NoArgsAttributeParser for RustcCoherenceIsCoreParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 449894f7834b..c5b98139074e 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -28,9 +28,9 @@ use crate::attributes::codegen_attrs::{
 };
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::crate_level::{
-    CrateNameParser, MoveSizeLimitParser, NoCoreParser, NoStdParser, PatternComplexityLimitParser,
-    RecursionLimitParser, RustcCoherenceIsCoreParser, TypeLengthLimitParser,
-    WindowsSubsystemParser,
+    CrateNameParser, MoveSizeLimitParser, NoCoreParser, NoMainParser, NoStdParser,
+    PatternComplexityLimitParser, RecursionLimitParser, RustcCoherenceIsCoreParser,
+    TypeLengthLimitParser, WindowsSubsystemParser,
 };
 use crate::attributes::debugger::DebuggerViualizerParser;
 use crate::attributes::deprecation::DeprecationParser;
@@ -263,6 +263,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 7b7fae9fdcca..119bf554b2ae 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -852,6 +852,9 @@ pub enum AttributeKind {
     /// Represents `#[no_link]`
     NoLink,
 
+    /// Represents `#[no_main]`
+    NoMain,
+
     /// Represents `#[no_mangle]`
     NoMangle(Span),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index dff8a5727771..7ba268c16271 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -79,6 +79,7 @@ impl AttributeKind {
             NoCore(..) => No,
             NoImplicitPrelude(..) => No,
             NoLink => No,
+            NoMain => No,
             NoMangle(..) => Yes, // Needed for rustdoc
             NoStd(..) => No,
             NonExhaustive(..) => Yes, // Needed for rustdoc
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 4b71d4755cb6..50e80cabdb07 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -297,6 +297,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::PatternComplexityLimit { .. }
                     | AttributeKind::NoCore { .. }
                     | AttributeKind::NoStd { .. }
+                    | AttributeKind::NoMain
                     | AttributeKind::ObjcClass { .. }
                     | AttributeKind::ObjcSelector { .. }
                     | AttributeKind::RustcCoherenceIsCore(..)
diff --git a/compiler/rustc_passes/src/entry.rs b/compiler/rustc_passes/src/entry.rs
index c02a01c1b823..bd737518ed47 100644
--- a/compiler/rustc_passes/src/entry.rs
+++ b/compiler/rustc_passes/src/entry.rs
@@ -1,4 +1,3 @@
-use rustc_ast::attr;
 use rustc_ast::entry::EntryPointType;
 use rustc_errors::codes::*;
 use rustc_hir::attrs::AttributeKind;
@@ -7,7 +6,7 @@ use rustc_hir::{CRATE_HIR_ID, ItemId, Node, find_attr};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{CrateType, EntryFnType, sigpipe};
-use rustc_span::{RemapPathScopeComponents, Span, sym};
+use rustc_span::{RemapPathScopeComponents, Span};
 
 use crate::errors::{ExternMain, MultipleRustcMain, NoMainErr};
 
@@ -30,7 +29,7 @@ fn entry_fn(tcx: TyCtxt<'_>, (): ()) -> Option<(DefId, EntryFnType)> {
     }
 
     // If the user wants no main function at all, then stop here.
-    if attr::contains_name(tcx.hir_attrs(CRATE_HIR_ID), sym::no_main) {
+    if find_attr!(tcx.hir_attrs(CRATE_HIR_ID), AttributeKind::NoMain) {
         return None;
     }
 
diff --git a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs
index 6716e78a7197..3e3235e658f6 100644
--- a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs
+++ b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs
@@ -884,26 +884,26 @@ mod feature {
 
 #[no_main]
 //~^ WARN crate-level attribute should be an inner attribute
-//~| HELP add a `!`
 mod no_main_1 {
+    //~^ NOTE: This attribute does not have an `!`, which means it is applied to this module
     mod inner { #![no_main] }
-//~^ WARN crate-level attribute should be in the root module
+    //~^ WARN the `#![no_main]` attribute can only be used at the crate root
 
     #[no_main] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| HELP add a `!`
+    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
 
     #[no_main] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| HELP add a `!`
+    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
 
     #[no_main] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| HELP add a `!`
+    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
 
     #[no_main] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| HELP add a `!`
+    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation
 }
 
 #[no_builtins]
diff --git a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr
index 8ed39a0079ba..d89aec222be8 100644
--- a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr
+++ b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr
@@ -240,17 +240,6 @@ help: add a `!`
 LL | #![feature(x0600)]
    |  +
 
-warning: crate-level attribute should be an inner attribute
-  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:885:1
-   |
-LL | #[no_main]
-   | ^^^^^^^^^^
-   |
-help: add a `!`
-   |
-LL | #![no_main]
-   |  +
-
 warning: crate-level attribute should be an inner attribute
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:909:1
    |
@@ -476,56 +465,6 @@ help: add a `!`
 LL |     #![feature(x0600)] impl S { }
    |      +
 
-warning: crate-level attribute should be in the root module
-  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:889:17
-   |
-LL |     mod inner { #![no_main] }
-   |                 ^^^^^^^^^^^
-
-warning: crate-level attribute should be an inner attribute
-  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:892:5
-   |
-LL |     #[no_main] fn f() { }
-   |     ^^^^^^^^^^
-   |
-help: add a `!`
-   |
-LL |     #![no_main] fn f() { }
-   |      +
-
-warning: crate-level attribute should be an inner attribute
-  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:896:5
-   |
-LL |     #[no_main] struct S;
-   |     ^^^^^^^^^^
-   |
-help: add a `!`
-   |
-LL |     #![no_main] struct S;
-   |      +
-
-warning: crate-level attribute should be an inner attribute
-  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:900:5
-   |
-LL |     #[no_main] type T = S;
-   |     ^^^^^^^^^^
-   |
-help: add a `!`
-   |
-LL |     #![no_main] type T = S;
-   |      +
-
-warning: crate-level attribute should be an inner attribute
-  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:904:5
-   |
-LL |     #[no_main] impl S { }
-   |     ^^^^^^^^^^
-   |
-help: add a `!`
-   |
-LL |     #![no_main] impl S { }
-   |      +
-
 warning: crate-level attribute should be in the root module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:913:17
    |
@@ -1407,6 +1346,76 @@ note: This attribute does not have an `!`, which means it is applied to this imp
 LL |     #[crate_name = "0900"] impl S { }
    |                            ^^^^^^^^^^
 
+warning: crate-level attribute should be an inner attribute: add an exclamation mark: `#![no_main]`
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:885:1
+   |
+LL | #[no_main]
+   | ^^^^^^^^^^
+   |
+note: This attribute does not have an `!`, which means it is applied to this module
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:887:1
+   |
+LL | / mod no_main_1 {
+LL | |
+LL | |     mod inner { #![no_main] }
+...  |
+LL | | }
+   | |_^
+
+warning: the `#![no_main]` attribute can only be used at the crate root
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:889:17
+   |
+LL |     mod inner { #![no_main] }
+   |                 ^^^^^^^^^^^
+
+warning: crate-level attribute should be an inner attribute: add an exclamation mark: `#![no_main]`
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:892:5
+   |
+LL |     #[no_main] fn f() { }
+   |     ^^^^^^^^^^
+   |
+note: This attribute does not have an `!`, which means it is applied to this function
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:892:16
+   |
+LL |     #[no_main] fn f() { }
+   |                ^^^^^^^^^^
+
+warning: crate-level attribute should be an inner attribute: add an exclamation mark: `#![no_main]`
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:896:5
+   |
+LL |     #[no_main] struct S;
+   |     ^^^^^^^^^^
+   |
+note: This attribute does not have an `!`, which means it is applied to this struct
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:896:16
+   |
+LL |     #[no_main] struct S;
+   |                ^^^^^^^^^
+
+warning: crate-level attribute should be an inner attribute: add an exclamation mark: `#![no_main]`
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:900:5
+   |
+LL |     #[no_main] type T = S;
+   |     ^^^^^^^^^^
+   |
+note: This attribute does not have an `!`, which means it is applied to this type alias
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:900:16
+   |
+LL |     #[no_main] type T = S;
+   |                ^^^^^^^^^^^
+
+warning: crate-level attribute should be an inner attribute: add an exclamation mark: `#![no_main]`
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:904:5
+   |
+LL |     #[no_main] impl S { }
+   |     ^^^^^^^^^^
+   |
+note: This attribute does not have an `!`, which means it is applied to this implementation block
+  --> $DIR/issue-43106-gating-of-builtin-attrs.rs:904:16
+   |
+LL |     #[no_main] impl S { }
+   |                ^^^^^^^^^^
+
 warning: crate-level attribute should be an inner attribute: add an exclamation mark: `#![recursion_limit]`
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:933:1
    |

From c367dfde6972ba60eca9f295d68be2f80060742e Mon Sep 17 00:00:00 2001
From: Ralf Jung 
Date: Sun, 18 Jan 2026 18:08:16 +0100
Subject: [PATCH 0349/1843] shims: add FIXME for missing direct tests

---
 .../src/shims/unix/android/foreign_items.rs   |  6 ++++
 .../miri/src/shims/unix/foreign_items.rs      | 19 ++++++++++++
 .../src/shims/unix/freebsd/foreign_items.rs   |  3 ++
 .../src/shims/unix/linux/foreign_items.rs     |  5 +++
 .../src/shims/unix/macos/foreign_items.rs     | 11 +++++++
 .../src/shims/unix/solarish/foreign_items.rs  |  5 +++
 .../miri/src/shims/windows/foreign_items.rs   | 31 +++++++++++++++++++
 .../fail-dep/concurrency/windows_join_main.rs |  2 +-
 8 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/tools/miri/src/shims/unix/android/foreign_items.rs b/src/tools/miri/src/shims/unix/android/foreign_items.rs
index 2b290b68c78c..f00bfb0a2078 100644
--- a/src/tools/miri/src/shims/unix/android/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/android/foreign_items.rs
@@ -28,21 +28,25 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
         match link_name.as_str() {
             // File related shims
             "stat" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.stat(path, buf)?;
                 this.write_scalar(result, dest)?;
             }
             "lstat" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.lstat(path, buf)?;
                 this.write_scalar(result, dest)?;
             }
             "readdir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirp] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.readdir64("dirent", dirp)?;
                 this.write_scalar(result, dest)?;
             }
             "pread64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, buf, count, offset] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, *mut _, usize, libc::off64_t) -> isize),
                     link_name,
@@ -56,6 +60,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.read(fd, buf, count, Some(offset), dest)?;
             }
             "pwrite64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, buf, n, offset] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, *const _, usize, libc::off64_t) -> isize),
                     link_name,
@@ -70,6 +75,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write(fd, buf, count, Some(offset), dest)?;
             }
             "lseek64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, offset, whence] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, libc::off64_t, i32) -> libc::off64_t),
                     link_name,
diff --git a/src/tools/miri/src/shims/unix/foreign_items.rs b/src/tools/miri/src/shims/unix/foreign_items.rs
index 8eacdc3583d4..04ec260eccd0 100644
--- a/src/tools/miri/src/shims/unix/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/foreign_items.rs
@@ -143,6 +143,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "getcwd" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [buf, size] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*mut _, usize) -> *mut _),
                     link_name,
@@ -153,6 +154,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_pointer(result, dest)?;
             }
             "chdir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _) -> i32),
                     link_name,
@@ -209,6 +211,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write(fd, buf, count, None, dest)?;
             }
             "pread" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, buf, count, offset] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, *mut _, usize, libc::off_t) -> isize),
                     link_name,
@@ -222,6 +225,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.read(fd, buf, count, Some(offset), dest)?;
             }
             "pwrite" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, buf, n, offset] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, *const _, usize, libc::off_t) -> isize),
                     link_name,
@@ -299,6 +303,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "unlink" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _) -> i32),
                     link_name,
@@ -309,6 +314,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "symlink" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [target, linkpath] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _, *const _) -> i32),
                     link_name,
@@ -324,6 +330,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "rename" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [oldpath, newpath] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _, *const _) -> i32),
                     link_name,
@@ -334,6 +341,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "mkdir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, mode] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _, libc::mode_t) -> i32),
                     link_name,
@@ -344,6 +352,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "rmdir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _) -> i32),
                     link_name,
@@ -354,6 +363,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "opendir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [name] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*const _) -> *mut _),
                     link_name,
@@ -364,6 +374,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "closedir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirp] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(*mut _) -> i32),
                     link_name,
@@ -374,6 +385,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "lseek" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, offset, whence] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, libc::off_t, i32) -> libc::off_t),
                     link_name,
@@ -398,6 +410,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "fsync" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32) -> i32),
                     link_name,
@@ -408,6 +421,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "fdatasync" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32) -> i32),
                     link_name,
@@ -659,6 +673,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_null(dest)?;
             }
             "pthread_key_delete" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [key] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let key = this.read_scalar(key)?.to_bits(key.layout.size)?;
                 this.machine.tls.delete_tls_key(key)?;
@@ -666,6 +681,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_null(dest)?;
             }
             "pthread_getspecific" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [key] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let key = this.read_scalar(key)?.to_bits(key.layout.size)?;
                 let active_thread = this.active_thread();
@@ -673,6 +689,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(ptr, dest)?;
             }
             "pthread_setspecific" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [key, new_ptr] =
                     this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let key = this.read_scalar(key)?.to_bits(key.layout.size)?;
@@ -833,6 +850,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(res, dest)?;
             }
             "sched_yield" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.sched_yield()?;
                 this.write_null(dest)?;
@@ -941,6 +959,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "pthread_atfork" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [prepare, parent, child] =
                     this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.read_pointer(prepare)?;
diff --git a/src/tools/miri/src/shims/unix/freebsd/foreign_items.rs b/src/tools/miri/src/shims/unix/freebsd/foreign_items.rs
index fb2d3f758420..b94ee27c46a0 100644
--- a/src/tools/miri/src/shims/unix/freebsd/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/freebsd/foreign_items.rs
@@ -139,11 +139,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             // For those, we both intercept `func` and `call@FBSD_1.0` symbols cases
             // since freebsd 12 the former form can be expected.
             "stat" | "stat@FBSD_1.0" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.stat(path, buf)?;
                 this.write_scalar(result, dest)?;
             }
             "lstat" | "lstat@FBSD_1.0" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.lstat(path, buf)?;
                 this.write_scalar(result, dest)?;
@@ -154,6 +156,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "readdir" | "readdir@FBSD_1.0" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirp] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.readdir64("dirent", dirp)?;
                 this.write_scalar(result, dest)?;
diff --git a/src/tools/miri/src/shims/unix/linux/foreign_items.rs b/src/tools/miri/src/shims/unix/linux/foreign_items.rs
index a7cb2ed11b2b..426bc28ce887 100644
--- a/src/tools/miri/src/shims/unix/linux/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/linux/foreign_items.rs
@@ -45,6 +45,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "pread64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, buf, count, offset] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, *mut _, usize, libc::off64_t) -> isize),
                     link_name,
@@ -58,6 +59,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.read(fd, buf, count, Some(offset), dest)?;
             }
             "pwrite64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, buf, n, offset] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, *const _, usize, libc::off64_t) -> isize),
                     link_name,
@@ -72,6 +74,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write(fd, buf, count, Some(offset), dest)?;
             }
             "lseek64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [fd, offset, whence] = this.check_shim_sig(
                     shim_sig!(extern "C" fn(i32, libc::off64_t, i32) -> libc::off64_t),
                     link_name,
@@ -111,6 +114,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "readdir64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirp] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.readdir64("dirent64", dirp)?;
                 this.write_scalar(result, dest)?;
@@ -122,6 +126,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "statx" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirfd, pathname, flags, mask, statxbuf] =
                     this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.linux_statx(dirfd, pathname, flags, mask, statxbuf)?;
diff --git a/src/tools/miri/src/shims/unix/macos/foreign_items.rs b/src/tools/miri/src/shims/unix/macos/foreign_items.rs
index f798f64441b1..204934f4c41d 100644
--- a/src/tools/miri/src/shims/unix/macos/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/macos/foreign_items.rs
@@ -47,11 +47,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "stat" | "stat$INODE64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.stat(path, buf)?;
                 this.write_scalar(result, dest)?;
             }
             "lstat" | "lstat$INODE64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.lstat(path, buf)?;
                 this.write_scalar(result, dest)?;
@@ -62,11 +64,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "opendir$INODE64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [name] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.opendir(name)?;
                 this.write_scalar(result, dest)?;
             }
             "readdir_r" | "readdir_r$INODE64" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirp, entry, result] =
                     this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.macos_readdir_r(dirp, entry, result)?;
@@ -87,6 +91,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Environment related shims
             "_NSGetEnviron" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let environ = this.machine.env_vars.unix().environ();
                 this.write_pointer(environ, dest)?;
@@ -111,6 +116,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             }
 
             "mach_timebase_info" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [info] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.mach_timebase_info(info)?;
                 this.write_scalar(result, dest)?;
@@ -118,14 +124,17 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Access to command-line arguments
             "_NSGetArgc" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.write_pointer(this.machine.argc.expect("machine must be initialized"), dest)?;
             }
             "_NSGetArgv" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.write_pointer(this.machine.argv.expect("machine must be initialized"), dest)?;
             }
             "_NSGetExecutablePath" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [buf, bufsize] =
                     this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.check_no_isolation("`_NSGetExecutablePath`")?;
@@ -168,12 +177,14 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Querying system information
             "pthread_get_stackaddr_np" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [thread] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.read_target_usize(thread)?;
                 let stack_addr = Scalar::from_uint(this.machine.stack_addr, this.pointer_size());
                 this.write_scalar(stack_addr, dest)?;
             }
             "pthread_get_stacksize_np" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [thread] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 this.read_target_usize(thread)?;
                 let stack_size = Scalar::from_uint(this.machine.stack_size, this.pointer_size());
diff --git a/src/tools/miri/src/shims/unix/solarish/foreign_items.rs b/src/tools/miri/src/shims/unix/solarish/foreign_items.rs
index fa8c86b025a7..f3918fdccf12 100644
--- a/src/tools/miri/src/shims/unix/solarish/foreign_items.rs
+++ b/src/tools/miri/src/shims/unix/solarish/foreign_items.rs
@@ -91,16 +91,19 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // File related shims
             "stat" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.stat(path, buf)?;
                 this.write_scalar(result, dest)?;
             }
             "lstat" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path, buf] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.lstat(path, buf)?;
                 this.write_scalar(result, dest)?;
             }
             "readdir" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [dirp] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let result = this.readdir64("dirent", dirp)?;
                 this.write_scalar(result, dest)?;
@@ -122,6 +125,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             }
 
             "stack_getbounds" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [stack] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 let stack = this.deref_pointer_as(stack, this.libc_ty_layout("stack_t"))?;
 
@@ -140,6 +144,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             }
 
             "pset_info" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [pset, tpe, cpus, list] =
                     this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
                 // We do not need to handle the current process cpu mask, available_parallelism
diff --git a/src/tools/miri/src/shims/windows/foreign_items.rs b/src/tools/miri/src/shims/windows/foreign_items.rs
index 1905fb22e26a..0bdf6bb78505 100644
--- a/src/tools/miri/src/shims/windows/foreign_items.rs
+++ b/src/tools/miri/src/shims/windows/foreign_items.rs
@@ -148,6 +148,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
         match link_name.as_str() {
             // Environment related shims
             "GetEnvironmentVariableW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [name, buf, size] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*const _, *mut _, u32) -> u32),
                     link_name,
@@ -158,6 +159,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "SetEnvironmentVariableW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [name, value] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*const _, *const _) -> winapi::BOOL),
                     link_name,
@@ -168,6 +170,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "GetEnvironmentStringsW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig(
                     shim_sig!(extern "system" fn() -> *mut _),
                     link_name,
@@ -178,6 +181,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_pointer(result, dest)?;
             }
             "FreeEnvironmentStringsW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [env_block] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL),
                     link_name,
@@ -188,6 +192,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "GetCurrentDirectoryW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [size, buf] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(u32, *mut _) -> u32),
                     link_name,
@@ -198,6 +203,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "SetCurrentDirectoryW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [path] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*const _) -> winapi::BOOL),
                     link_name,
@@ -208,6 +214,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "GetUserProfileDirectoryW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [token, buf, size] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HANDLE, *mut _, *mut _) -> winapi::BOOL),
                     link_name,
@@ -218,6 +225,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "GetCurrentProcessId" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig(
                     shim_sig!(extern "system" fn() -> u32),
                     link_name,
@@ -314,6 +322,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 )?;
             }
             "GetFullPathNameW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [filename, size, buffer, filepart] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*const _, u32, *mut _, *mut _) -> u32),
                     link_name,
@@ -445,6 +454,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Allocation
             "HeapAlloc" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [handle, flags, size] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HANDLE, u32, usize) -> *mut _),
                     link_name,
@@ -472,6 +482,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_pointer(ptr, dest)?;
             }
             "HeapFree" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [handle, flags, ptr] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HANDLE, u32, *mut _) -> winapi::BOOL),
                     link_name,
@@ -489,6 +500,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(Scalar::from_i32(1), dest)?;
             }
             "HeapReAlloc" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [handle, flags, old_ptr, size] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HANDLE, u32, *mut _, usize) -> *mut _),
                     link_name,
@@ -514,6 +526,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_pointer(new_ptr, dest)?;
             }
             "LocalFree" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [ptr] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HLOCAL) -> winapi::HLOCAL),
                     link_name,
@@ -577,6 +590,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Querying system information
             "GetSystemInfo" => {
+                // FIXME: This does not have a direct test (#3179).
                 // Also called from `page_size` crate.
                 let [system_info] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> ()),
@@ -658,6 +672,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Access to command-line arguments
             "GetCommandLineW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [] = this.check_shim_sig(
                     shim_sig!(extern "system" fn() -> *mut _),
                     link_name,
@@ -672,6 +687,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Time related shims
             "GetSystemTimeAsFileTime" | "GetSystemTimePreciseAsFileTime" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [filetime] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> ()),
                     link_name,
@@ -681,6 +697,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.GetSystemTimeAsFileTime(link_name.as_str(), filetime)?;
             }
             "QueryPerformanceCounter" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [performance_count] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL),
                     link_name,
@@ -691,6 +708,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "QueryPerformanceFrequency" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [frequency] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> winapi::BOOL),
                     link_name,
@@ -701,6 +719,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "Sleep" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [timeout] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(u32) -> ()),
                     link_name,
@@ -711,6 +730,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.Sleep(timeout)?;
             }
             "CreateWaitableTimerExW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [attributes, name, flags, access] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _, *const _, u32, u32) -> winapi::HANDLE),
                     link_name,
@@ -748,6 +768,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(result, dest)?;
             }
             "WaitOnAddress" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [ptr_op, compare_op, size_op, timeout_op] = this.check_shim_sig(
                     // First pointer is volatile
                     shim_sig!(extern "system" fn(*mut _, *mut _, usize, u32) -> winapi::BOOL),
@@ -759,6 +780,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.WaitOnAddress(ptr_op, compare_op, size_op, timeout_op, dest)?;
             }
             "WakeByAddressSingle" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [ptr_op] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> ()),
                     link_name,
@@ -769,6 +791,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.WakeByAddressSingle(ptr_op)?;
             }
             "WakeByAddressAll" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [ptr_op] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _) -> ()),
                     link_name,
@@ -781,6 +804,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Dynamic symbol loading
             "GetProcAddress" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [module, proc_name] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HMODULE, *const _) -> winapi::FARPROC),
                     link_name,
@@ -936,6 +960,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
             // Miscellaneous
             "ExitProcess" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [code] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(u32) -> ()),
                     link_name,
@@ -962,6 +987,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(Scalar::from_bool(true), dest)?;
             }
             "ProcessPrng" => {
+                // FIXME: This does not have a direct test (#3179).
                 // used by `std`
                 let [ptr, len] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(*mut _, usize) -> winapi::BOOL),
@@ -1014,6 +1040,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_null(dest)?; // STATUS_SUCCESS
             }
             "GetConsoleScreenBufferInfo" => {
+                // FIXME: This does not have a direct test (#3179).
                 // `term` needs this, so we fake it.
                 let [console, buffer_info] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HANDLE, *mut _) -> winapi::BOOL),
@@ -1029,6 +1056,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_null(dest)?;
             }
             "GetStdHandle" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [which] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(u32) -> winapi::HANDLE),
                     link_name,
@@ -1080,6 +1108,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(ret, dest)?;
             }
             "GetModuleFileNameW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [handle, filename, size] = this.check_shim_sig(
                     shim_sig!(extern "system" fn(winapi::HMODULE, *mut _, u32) -> u32),
                     link_name,
@@ -1118,6 +1147,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 }
             }
             "FormatMessageW" => {
+                // FIXME: This does not have a direct test (#3179).
                 let [flags, module, message_id, language_id, buffer, size, arguments] = this
                     .check_shim_sig(
                         shim_sig!(
@@ -1159,6 +1189,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
             }
 
             "_Unwind_RaiseException" => {
+                // FIXME: This does not have a direct test (#3179).
                 // This is not formally part of POSIX, but it is very wide-spread on POSIX systems.
                 // It was originally specified as part of the Itanium C++ ABI:
                 // https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html#base-throw.
diff --git a/src/tools/miri/tests/fail-dep/concurrency/windows_join_main.rs b/src/tools/miri/tests/fail-dep/concurrency/windows_join_main.rs
index da549a8d117d..a71778b1d0d4 100644
--- a/src/tools/miri/tests/fail-dep/concurrency/windows_join_main.rs
+++ b/src/tools/miri/tests/fail-dep/concurrency/windows_join_main.rs
@@ -12,7 +12,7 @@ use windows_sys::Win32::System::Threading::{INFINITE, WaitForSingleObject};
 
 // XXX HACK: This is how miri represents the handle for thread 0.
 // This value can be "legitimately" obtained by using `GetCurrentThread` with `DuplicateHandle`
-// but miri does not implement `DuplicateHandle` yet.
+// but miri does not implement `DuplicateHandle` yet. (FIXME: it does now.)
 const MAIN_THREAD: HANDLE = (2i32 << 29) as HANDLE;
 
 fn main() {

From ec787b07fd34fd68a35a6d9b07927dd06dad6da4 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 18 Jan 2026 18:26:59 +0100
Subject: [PATCH 0350/1843] Remove `DiagMessage::Translated` in favour of
 `DiagMessage::Str`

---
 compiler/rustc_error_messages/src/lib.rs | 22 +++++-----------------
 compiler/rustc_errors/src/lib.rs         |  2 +-
 compiler/rustc_errors/src/translation.rs |  2 +-
 3 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 085403c8ef36..c0737edd7d65 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -244,13 +244,6 @@ type FluentId = Cow<'static, str>;
 pub enum SubdiagMessage {
     /// Non-translatable diagnostic message.
     Str(Cow<'static, str>),
-    /// Translatable message which has already been translated eagerly.
-    ///
-    /// Some diagnostics have repeated subdiagnostics where the same interpolated variables would
-    /// be instantiated multiple times with different values. These subdiagnostics' messages
-    /// are translated when they are added to the parent diagnostic, producing this variant of
-    /// `DiagMessage`.
-    Translated(Cow<'static, str>),
     /// Identifier of a Fluent message. Instances of this variant are generated by the
     /// `Subdiagnostic` derive.
     FluentIdentifier(FluentId),
@@ -285,15 +278,13 @@ impl From> for SubdiagMessage {
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)]
 #[rustc_diagnostic_item = "DiagMessage"]
 pub enum DiagMessage {
-    /// Non-translatable diagnostic message.
-    Str(Cow<'static, str>),
-    /// Translatable message which has been already translated.
+    /// Non-translatable diagnostic message or a message that has been translated eagerly.
     ///
     /// Some diagnostics have repeated subdiagnostics where the same interpolated variables would
     /// be instantiated multiple times with different values. These subdiagnostics' messages
-    /// are translated when they are added to the parent diagnostic, producing this variant of
-    /// `DiagMessage`.
-    Translated(Cow<'static, str>),
+    /// are translated when they are added to the parent diagnostic. This is one of the ways
+    /// this variant of `DiagMessage` is produced.
+    Str(Cow<'static, str>),
     /// Identifier for a Fluent message (with optional attribute) corresponding to the diagnostic
     /// message. Yet to be translated.
     ///
@@ -311,7 +302,6 @@ impl DiagMessage {
     pub fn with_subdiagnostic_message(&self, sub: SubdiagMessage) -> Self {
         let attr = match sub {
             SubdiagMessage::Str(s) => return DiagMessage::Str(s),
-            SubdiagMessage::Translated(s) => return DiagMessage::Translated(s),
             SubdiagMessage::FluentIdentifier(id) => {
                 return DiagMessage::FluentIdentifier(id, None);
             }
@@ -320,7 +310,6 @@ impl DiagMessage {
 
         match self {
             DiagMessage::Str(s) => DiagMessage::Str(s.clone()),
-            DiagMessage::Translated(s) => DiagMessage::Translated(s.clone()),
             DiagMessage::FluentIdentifier(id, _) => {
                 DiagMessage::FluentIdentifier(id.clone(), Some(attr))
             }
@@ -329,7 +318,7 @@ impl DiagMessage {
 
     pub fn as_str(&self) -> Option<&str> {
         match self {
-            DiagMessage::Translated(s) | DiagMessage::Str(s) => Some(s),
+            DiagMessage::Str(s) => Some(s),
             DiagMessage::FluentIdentifier(_, _) => None,
         }
     }
@@ -360,7 +349,6 @@ impl From for SubdiagMessage {
     fn from(val: DiagMessage) -> Self {
         match val {
             DiagMessage::Str(s) => SubdiagMessage::Str(s),
-            DiagMessage::Translated(s) => SubdiagMessage::Translated(s),
             DiagMessage::FluentIdentifier(id, None) => SubdiagMessage::FluentIdentifier(id),
             // There isn't really a sensible behaviour for this because it loses information but
             // this is the most sensible of the behaviours.
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 148368045f4f..102dd840556b 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -1745,7 +1745,7 @@ impl DiagCtxtInner {
         message: DiagMessage,
         args: impl Iterator>,
     ) -> SubdiagMessage {
-        SubdiagMessage::Translated(Cow::from(self.eagerly_translate_to_string(message, args)))
+        SubdiagMessage::Str(Cow::from(self.eagerly_translate_to_string(message, args)))
     }
 
     /// Translate `message` eagerly with `args` to `String`.
diff --git a/compiler/rustc_errors/src/translation.rs b/compiler/rustc_errors/src/translation.rs
index c0bcec093c7e..5bffd74740dc 100644
--- a/compiler/rustc_errors/src/translation.rs
+++ b/compiler/rustc_errors/src/translation.rs
@@ -76,7 +76,7 @@ impl Translator {
     ) -> Result, TranslateError<'a>> {
         trace!(?message, ?args);
         let (identifier, attr) = match message {
-            DiagMessage::Str(msg) | DiagMessage::Translated(msg) => {
+            DiagMessage::Str(msg) => {
                 return Ok(Cow::Borrowed(msg));
             }
             DiagMessage::FluentIdentifier(identifier, attr) => (identifier, attr),

From 85200a4ef0e889c35473dad54a3b04ab8540fce6 Mon Sep 17 00:00:00 2001
From: Clara Engler 
Date: Fri, 16 Jan 2026 18:10:08 +0100
Subject: [PATCH 0351/1843] option: Use Option::map in Option::cloned

This commit removes a repetitive match statement in favor of
Option::map for Option::cloned.
---
 library/core/src/option.rs | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/library/core/src/option.rs b/library/core/src/option.rs
index ed31d4efaa75..eb4f978b7c19 100644
--- a/library/core/src/option.rs
+++ b/library/core/src/option.rs
@@ -2103,10 +2103,7 @@ impl Option<&T> {
     where
         T: Clone,
     {
-        match self {
-            Some(t) => Some(t.clone()),
-            None => None,
-        }
+        self.map(T::clone)
     }
 }
 
@@ -2154,10 +2151,7 @@ impl Option<&mut T> {
     where
         T: Clone,
     {
-        match self {
-            Some(t) => Some(t.clone()),
-            None => None,
-        }
+        self.as_deref().map(T::clone)
     }
 }
 

From 43fb39f3fed61e05520aebe23d3c5815fbc896f4 Mon Sep 17 00:00:00 2001
From: chillfish8 
Date: Sun, 11 Jan 2026 21:56:30 +0000
Subject: [PATCH 0352/1843] Add avx512 `pack*` family of instructions

---
 src/tools/miri/src/shims/x86/avx512.rs        |  34 ++++-
 .../pass/shims/x86/intrinsics-x86-avx512.rs   | 130 +++++++++++++++++-
 2 files changed, 161 insertions(+), 3 deletions(-)

diff --git a/src/tools/miri/src/shims/x86/avx512.rs b/src/tools/miri/src/shims/x86/avx512.rs
index 9231fc446919..b057a78b6c8e 100644
--- a/src/tools/miri/src/shims/x86/avx512.rs
+++ b/src/tools/miri/src/shims/x86/avx512.rs
@@ -3,7 +3,7 @@ use rustc_middle::ty::Ty;
 use rustc_span::Symbol;
 use rustc_target::callconv::FnAbi;
 
-use super::{permute, pmaddbw, pmaddwd, psadbw, pshufb};
+use super::{packssdw, packsswb, packusdw, packuswb, permute, pmaddbw, pmaddwd, psadbw, pshufb};
 use crate::*;
 
 impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
@@ -130,6 +130,38 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
 
                 vpdpbusd(this, src, a, b, dest)?;
             }
+            // Used to implement the _mm512_packs_epi16 function
+            "packsswb.512" => {
+                this.expect_target_feature_for_intrinsic(link_name, "avx512bw")?;
+
+                let [a, b] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
+
+                packsswb(this, a, b, dest)?;
+            }
+            // Used to implement the _mm512_packus_epi16 function
+            "packuswb.512" => {
+                this.expect_target_feature_for_intrinsic(link_name, "avx512bw")?;
+
+                let [a, b] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
+
+                packuswb(this, a, b, dest)?;
+            }
+            // Used to implement the _mm512_packs_epi32 function
+            "packssdw.512" => {
+                this.expect_target_feature_for_intrinsic(link_name, "avx512bw")?;
+
+                let [a, b] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
+
+                packssdw(this, a, b, dest)?;
+            }
+            // Used to implement the _mm512_packus_epi32 function
+            "packusdw.512" => {
+                this.expect_target_feature_for_intrinsic(link_name, "avx512bw")?;
+
+                let [a, b] = this.check_shim_sig_lenient(abi, CanonAbi::C, link_name, args)?;
+
+                packusdw(this, a, b, dest)?;
+            }
             _ => return interp_ok(EmulateItemResult::NotSupported),
         }
         interp_ok(EmulateItemResult::NeedsReturn)
diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs
index 7cc554ef5a3c..e1e23eda8428 100644
--- a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs
+++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs
@@ -1,6 +1,6 @@
 // We're testing x86 target specific features
 //@only-target: x86_64 i686
-//@compile-flags: -C target-feature=+avx512f,+avx512vl,+avx512bitalg,+avx512vpopcntdq,+avx512vnni
+//@compile-flags: -C target-feature=+avx512f,+avx512vl,+avx512bw,+avx512bitalg,+avx512vpopcntdq,+avx512vnni
 
 #[cfg(target_arch = "x86")]
 use std::arch::x86::*;
@@ -11,12 +11,14 @@ use std::mem::transmute;
 fn main() {
     assert!(is_x86_feature_detected!("avx512f"));
     assert!(is_x86_feature_detected!("avx512vl"));
+    assert!(is_x86_feature_detected!("avx512bw"));
     assert!(is_x86_feature_detected!("avx512bitalg"));
     assert!(is_x86_feature_detected!("avx512vpopcntdq"));
     assert!(is_x86_feature_detected!("avx512vnni"));
 
     unsafe {
         test_avx512();
+        test_avx512bw();
         test_avx512bitalg();
         test_avx512vpopcntdq();
         test_avx512ternarylogic();
@@ -579,9 +581,133 @@ unsafe fn test_avx512vnni() {
     test_mm512_dpbusd_epi32();
 }
 
+#[target_feature(enable = "avx512bw")]
+unsafe fn test_avx512bw() {
+    #[target_feature(enable = "avx512bw")]
+    unsafe fn test_mm512_packs_epi16() {
+        let a = _mm512_set1_epi16(120);
+
+        // Because `packs` instructions do signed saturation, we expect
+        // that any value over `i8::MAX` will be saturated to `i8::MAX`, and any value
+        // less than `i8::MIN` will also be saturated to `i8::MIN`.
+        let b = _mm512_set_epi16(
+            200, 200, 200, 200, 200, 200, 200, 200, -200, -200, -200, -200, -200, -200, -200, -200,
+            200, 200, 200, 200, 200, 200, 200, 200, -200, -200, -200, -200, -200, -200, -200, -200,
+        );
+
+        // The pack* family of instructions in x86 operate in blocks
+        // of 128-bit lanes, meaning the first 128-bit lane in `a` is converted and written
+        // then the first 128-bit lane of `b`, followed by the second 128-bit lane in `a`, etc...
+        // Because we are going from 16-bits to 8-bits our 128-bit block becomes 64-bits in
+        // the output register.
+        // This leaves us with 8x 8-bit values interleaved in the final register.
+        #[rustfmt::skip]
+        const DST: [i8; 64] = [
+            120, 120, 120, 120, 120, 120, 120, 120,
+            i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN,
+            120, 120, 120, 120, 120, 120, 120, 120,
+            i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX,
+            120, 120, 120, 120, 120, 120, 120, 120,
+            i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN, i8::MIN,
+            120, 120, 120, 120, 120, 120, 120, 120,
+            i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX, i8::MAX,
+        ];
+        let dst = _mm512_loadu_si512(DST.as_ptr().cast::<__m512i>());
+        assert_eq_m512i(_mm512_packs_epi16(a, b), dst);
+    }
+    test_mm512_packs_epi16();
+
+    #[target_feature(enable = "avx512bw")]
+    unsafe fn test_mm512_packus_epi16() {
+        let a = _mm512_set1_epi16(120);
+
+        // Because `packus` instructions do unsigned saturation, we expect
+        // that any value over `u8::MAX` will be saturated to `u8::MAX`, and any value
+        // less than `u8::MIN` will also be saturated to `u8::MIN`.
+        let b = _mm512_set_epi16(
+            300, 300, 300, 300, 300, 300, 300, 300, -200, -200, -200, -200, -200, -200, -200, -200,
+            300, 300, 300, 300, 300, 300, 300, 300, -200, -200, -200, -200, -200, -200, -200, -200,
+        );
+
+        // See `test_mm512_packs_epi16` for an explanation of the output structure.
+        #[rustfmt::skip]
+        const DST: [u8; 64] = [
+            120, 120, 120, 120, 120, 120, 120, 120,
+            u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN,
+            120, 120, 120, 120, 120, 120, 120, 120,
+            u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX,
+            120, 120, 120, 120, 120, 120, 120, 120,
+            u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN, u8::MIN,
+            120, 120, 120, 120, 120, 120, 120, 120,
+            u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX,
+        ];
+        let dst = _mm512_loadu_si512(DST.as_ptr().cast::<__m512i>());
+        assert_eq_m512i(_mm512_packus_epi16(a, b), dst);
+    }
+    test_mm512_packus_epi16();
+
+    #[target_feature(enable = "avx512bw")]
+    unsafe fn test_mm512_packs_epi32() {
+        let a = _mm512_set1_epi32(8_000);
+
+        // Because `packs` instructions do signed saturation, we expect
+        // that any value over `i16::MAX` will be saturated to `i16::MAX`, and any value
+        // less than `i16::MIN` will also be saturated to `i16::MIN`.
+        let b = _mm512_set_epi32(
+            50_000, 50_000, 50_000, 50_000, -50_000, -50_000, -50_000, -50_000, 50_000, 50_000,
+            50_000, 50_000, -50_000, -50_000, -50_000, -50_000,
+        );
+
+        // See `test_mm512_packs_epi16` for an explanation of the output structure.
+        #[rustfmt::skip]
+        const DST: [i16; 32] = [
+            8_000, 8_000, 8_000, 8_000,
+            i16::MIN, i16::MIN, i16::MIN, i16::MIN,
+            8_000, 8_000, 8_000, 8_000,
+            i16::MAX, i16::MAX, i16::MAX, i16::MAX,
+            8_000, 8_000, 8_000, 8_000,
+            i16::MIN, i16::MIN, i16::MIN, i16::MIN,
+            8_000, 8_000, 8_000, 8_000,
+            i16::MAX, i16::MAX, i16::MAX, i16::MAX,
+        ];
+        let dst = _mm512_loadu_si512(DST.as_ptr().cast::<__m512i>());
+        assert_eq_m512i(_mm512_packs_epi32(a, b), dst);
+    }
+    test_mm512_packs_epi32();
+
+    #[target_feature(enable = "avx512bw")]
+    unsafe fn test_mm512_packus_epi32() {
+        let a = _mm512_set1_epi32(8_000);
+
+        // Because `packus` instructions do unsigned saturation, we expect
+        // that any value over `u16::MAX` will be saturated to `u16::MAX`, and any value
+        // less than `u16::MIN` will also be saturated to `u16::MIN`.
+        let b = _mm512_set_epi32(
+            80_000, 80_000, 80_000, 80_000, -50_000, -50_000, -50_000, -50_000, 80_000, 80_000,
+            80_000, 80_000, -50_000, -50_000, -50_000, -50_000,
+        );
+
+        // See `test_mm512_packs_epi16` for an explanation of the output structure.
+        #[rustfmt::skip]
+        const DST: [u16; 32] = [
+            8_000, 8_000, 8_000, 8_000,
+            u16::MIN, u16::MIN, u16::MIN, u16::MIN,
+            8_000, 8_000, 8_000, 8_000,
+            u16::MAX, u16::MAX, u16::MAX, u16::MAX,
+            8_000, 8_000, 8_000, 8_000,
+            u16::MIN, u16::MIN, u16::MIN, u16::MIN,
+            8_000, 8_000, 8_000, 8_000,
+            u16::MAX, u16::MAX, u16::MAX, u16::MAX,
+        ];
+        let dst = _mm512_loadu_si512(DST.as_ptr().cast::<__m512i>());
+        assert_eq_m512i(_mm512_packus_epi32(a, b), dst);
+    }
+    test_mm512_packus_epi32();
+}
+
 #[track_caller]
 unsafe fn assert_eq_m512i(a: __m512i, b: __m512i) {
-    assert_eq!(transmute::<_, [i32; 16]>(a), transmute::<_, [i32; 16]>(b))
+    assert_eq!(transmute::<_, [u16; 32]>(a), transmute::<_, [u16; 32]>(b))
 }
 
 #[track_caller]

From 858fb400225aca8bc3a666ba9e1a039b03cc3131 Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 20:06:15 +0100
Subject: [PATCH 0353/1843] Port #[rustc_allocator] to attr parser

---
 compiler/rustc_attr_parsing/src/attributes/mod.rs |  1 +
 .../src/attributes/rustc_allocator.rs             | 11 +++++++++++
 compiler/rustc_attr_parsing/src/context.rs        | 15 +++++++++------
 compiler/rustc_codegen_ssa/src/codegen_attrs.rs   |  4 +++-
 compiler/rustc_hir/src/attrs/data_structures.rs   |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_passes/src/check_attr.rs           |  2 +-
 7 files changed, 29 insertions(+), 8 deletions(-)
 create mode 100644 compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs

diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs
index 64aa7a66b019..0d328d5cc6a7 100644
--- a/compiler/rustc_attr_parsing/src/attributes/mod.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs
@@ -58,6 +58,7 @@ pub(crate) mod pin_v2;
 pub(crate) mod proc_macro_attrs;
 pub(crate) mod prototype;
 pub(crate) mod repr;
+pub(crate) mod rustc_allocator;
 pub(crate) mod rustc_dump;
 pub(crate) mod rustc_internal;
 pub(crate) mod semantics;
diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
new file mode 100644
index 000000000000..d7925f6fc372
--- /dev/null
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
@@ -0,0 +1,11 @@
+use super::prelude::*;
+
+pub(crate) struct RustcAllocatorParser;
+
+impl NoArgsAttributeParser for RustcAllocatorParser {
+    const PATH: &[Symbol] = &[sym::rustc_allocator];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcAllocator;
+}
\ No newline at end of file
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 449894f7834b..652a5e2cf1c6 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -64,17 +64,19 @@ use crate::attributes::proc_macro_attrs::{
 };
 use crate::attributes::prototype::CustomMirParser;
 use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
+use crate::attributes::rustc_allocator::RustcAllocatorParser;
 use crate::attributes::rustc_dump::{
     RustcDumpDefParents, RustcDumpItemBounds, RustcDumpPredicates, RustcDumpUserArgs,
     RustcDumpVtable,
 };
 use crate::attributes::rustc_internal::{
-    RustcHasIncoherentInherentImplsParser, RustcLayoutScalarValidRangeEndParser,
-    RustcLayoutScalarValidRangeStartParser, RustcLegacyConstGenericsParser,
-    RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser, RustcLintOptTyParser,
-    RustcLintQueryInstabilityParser, RustcLintUntrackedQueryInformationParser, RustcMainParser,
-    RustcMustImplementOneOfParser, RustcNeverReturnsNullPointerParser,
-    RustcNoImplicitAutorefsParser, RustcObjectLifetimeDefaultParser, RustcScalableVectorParser,
+    RustcHasIncoherentInherentImplsParser,
+    RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser,
+    RustcLegacyConstGenericsParser, RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser,
+    RustcLintOptTyParser, RustcLintQueryInstabilityParser,
+    RustcLintUntrackedQueryInformationParser, RustcMainParser, RustcMustImplementOneOfParser,
+    RustcNeverReturnsNullPointerParser, RustcNoImplicitAutorefsParser,
+    RustcObjectLifetimeDefaultParser, RustcScalableVectorParser,
     RustcSimdMonomorphizeLaneLimitParser,
 };
 use crate::attributes::semantics::MayDangleParser;
@@ -273,6 +275,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index fa02c5c51f7c..c3bf92df34a0 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -335,6 +335,9 @@ fn process_builtin_attrs(
                 AttributeKind::InstructionSet(instruction_set) => {
                     codegen_fn_attrs.instruction_set = Some(*instruction_set)
                 }
+                AttributeKind::RustcAllocator => {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR
+                }
                 _ => {}
             }
         }
@@ -344,7 +347,6 @@ fn process_builtin_attrs(
         };
 
         match name {
-            sym::rustc_allocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR,
             sym::rustc_nounwind => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND,
             sym::rustc_reallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR,
             sym::rustc_deallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 7b7fae9fdcca..b884372c2a7c 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -906,6 +906,9 @@ pub enum AttributeKind {
     /// Represents [`#[repr]`](https://doc.rust-lang.org/stable/reference/type-layout.html#representations).
     Repr { reprs: ThinVec<(ReprAttr, Span)>, first_span: Span },
 
+    /// Represents `#[rustc_allocator]`
+    RustcAllocator,
+
     /// Represents `#[rustc_builtin_macro]`.
     RustcBuiltinMacro { builtin_name: Option, helper_attrs: ThinVec, span: Span },
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index dff8a5727771..e81e74435e51 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -97,6 +97,7 @@ impl AttributeKind {
             PubTransparent(..) => Yes,
             RecursionLimit { .. } => No,
             Repr { .. } => No,
+            RustcAllocator => No,
             RustcBuiltinMacro { .. } => Yes,
             RustcCoherenceIsCore(..) => No,
             RustcDumpDefParents => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 4b71d4755cb6..c4de0f44a648 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -316,6 +316,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDumpDefParents
                     | AttributeKind::RustcDumpVtable(..)
                     | AttributeKind::NeedsAllocator
+                    | AttributeKind::RustcAllocator
                 ) => { /* do nothing  */ }
                 Attribute::Unparsed(attr_item) => {
                     style = Some(attr_item.style);
@@ -360,7 +361,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_do_not_const_check
                             | sym::rustc_reservation_impl
                             | sym::rustc_doc_primitive
-                            | sym::rustc_allocator
                             | sym::rustc_deallocator
                             | sym::rustc_reallocator
                             | sym::rustc_conversion_suggestion

From 027a6f268f4989198db21aded966c22eb48311cf Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 19:35:33 +0100
Subject: [PATCH 0354/1843] Port `#[rustc_deallocator]` to attr parser

---
 .../src/attributes/rustc_allocator.rs            | 12 +++++++++++-
 compiler/rustc_attr_parsing/src/context.rs       | 16 ++++++++--------
 compiler/rustc_codegen_ssa/src/codegen_attrs.rs  |  4 +++-
 compiler/rustc_hir/src/attrs/data_structures.rs  |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs    |  1 +
 compiler/rustc_passes/src/check_attr.rs          |  2 +-
 6 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
index d7925f6fc372..8eb252d3ff8d 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
@@ -8,4 +8,14 @@ impl NoArgsAttributeParser for RustcAllocatorParser {
     const ALLOWED_TARGETS: AllowedTargets =
         AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcAllocator;
-}
\ No newline at end of file
+}
+
+pub(crate) struct RustcDeallocatorParser;
+
+impl NoArgsAttributeParser for RustcDeallocatorParser {
+    const PATH: &[Symbol] = &[sym::rustc_deallocator];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcDeallocator;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 652a5e2cf1c6..454a2b427313 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -64,19 +64,18 @@ use crate::attributes::proc_macro_attrs::{
 };
 use crate::attributes::prototype::CustomMirParser;
 use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
-use crate::attributes::rustc_allocator::RustcAllocatorParser;
+use crate::attributes::rustc_allocator::{RustcAllocatorParser, RustcDeallocatorParser};
 use crate::attributes::rustc_dump::{
     RustcDumpDefParents, RustcDumpItemBounds, RustcDumpPredicates, RustcDumpUserArgs,
     RustcDumpVtable,
 };
 use crate::attributes::rustc_internal::{
-    RustcHasIncoherentInherentImplsParser,
-    RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser,
-    RustcLegacyConstGenericsParser, RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser,
-    RustcLintOptTyParser, RustcLintQueryInstabilityParser,
-    RustcLintUntrackedQueryInformationParser, RustcMainParser, RustcMustImplementOneOfParser,
-    RustcNeverReturnsNullPointerParser, RustcNoImplicitAutorefsParser,
-    RustcObjectLifetimeDefaultParser, RustcScalableVectorParser,
+    RustcHasIncoherentInherentImplsParser, RustcLayoutScalarValidRangeEndParser,
+    RustcLayoutScalarValidRangeStartParser, RustcLegacyConstGenericsParser,
+    RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser, RustcLintOptTyParser,
+    RustcLintQueryInstabilityParser, RustcLintUntrackedQueryInformationParser, RustcMainParser,
+    RustcMustImplementOneOfParser, RustcNeverReturnsNullPointerParser,
+    RustcNoImplicitAutorefsParser, RustcObjectLifetimeDefaultParser, RustcScalableVectorParser,
     RustcSimdMonomorphizeLaneLimitParser,
 };
 use crate::attributes::semantics::MayDangleParser;
@@ -277,6 +276,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index c3bf92df34a0..70328c14b19c 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -338,6 +338,9 @@ fn process_builtin_attrs(
                 AttributeKind::RustcAllocator => {
                     codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR
                 }
+                AttributeKind::RustcDeallocator => {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR
+                }
                 _ => {}
             }
         }
@@ -349,7 +352,6 @@ fn process_builtin_attrs(
         match name {
             sym::rustc_nounwind => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND,
             sym::rustc_reallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR,
-            sym::rustc_deallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR,
             sym::rustc_allocator_zeroed => {
                 codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
             }
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index b884372c2a7c..6f91dd48cb5a 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -915,6 +915,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_coherence_is_core]`
     RustcCoherenceIsCore(Span),
 
+    /// Represents `#[rustc_deallocator]`
+    RustcDeallocator,
+
     /// Represents `#[rustc_dump_def_parents]`
     RustcDumpDefParents,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index e81e74435e51..9fcaf77f69a4 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -100,6 +100,7 @@ impl AttributeKind {
             RustcAllocator => No,
             RustcBuiltinMacro { .. } => Yes,
             RustcCoherenceIsCore(..) => No,
+            RustcDeallocator => No,
             RustcDumpDefParents => No,
             RustcDumpItemBounds => No,
             RustcDumpPredicates => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index c4de0f44a648..83a4bd9f0cda 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -317,6 +317,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDumpVtable(..)
                     | AttributeKind::NeedsAllocator
                     | AttributeKind::RustcAllocator
+                    | AttributeKind::RustcDeallocator
                 ) => { /* do nothing  */ }
                 Attribute::Unparsed(attr_item) => {
                     style = Some(attr_item.style);
@@ -361,7 +362,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_do_not_const_check
                             | sym::rustc_reservation_impl
                             | sym::rustc_doc_primitive
-                            | sym::rustc_deallocator
                             | sym::rustc_reallocator
                             | sym::rustc_conversion_suggestion
                             | sym::rustc_allocator_zeroed

From 9a7614da04b5b3258b24ce64e197dd2989cff1d9 Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 19:40:19 +0100
Subject: [PATCH 0355/1843] Port `#[rustc_reallocator]` to attr parser

---
 .../src/attributes/rustc_allocator.rs                  | 10 ++++++++++
 compiler/rustc_attr_parsing/src/context.rs             |  5 ++++-
 compiler/rustc_codegen_ssa/src/codegen_attrs.rs        |  4 +++-
 compiler/rustc_hir/src/attrs/data_structures.rs        |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_passes/src/check_attr.rs                |  2 +-
 6 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
index 8eb252d3ff8d..9324f0c01156 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
@@ -19,3 +19,13 @@ impl NoArgsAttributeParser for RustcDeallocatorParser {
         AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcDeallocator;
 }
+
+pub(crate) struct RustcReallocatorParser;
+
+impl NoArgsAttributeParser for RustcReallocatorParser {
+    const PATH: &[Symbol] = &[sym::rustc_reallocator];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcReallocator;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 454a2b427313..19e03f1ad3eb 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -64,7 +64,9 @@ use crate::attributes::proc_macro_attrs::{
 };
 use crate::attributes::prototype::CustomMirParser;
 use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
-use crate::attributes::rustc_allocator::{RustcAllocatorParser, RustcDeallocatorParser};
+use crate::attributes::rustc_allocator::{
+    RustcAllocatorParser, RustcDeallocatorParser, RustcReallocatorParser,
+};
 use crate::attributes::rustc_dump::{
     RustcDumpDefParents, RustcDumpItemBounds, RustcDumpPredicates, RustcDumpUserArgs,
     RustcDumpVtable,
@@ -291,6 +293,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 70328c14b19c..86ea1ac94b4e 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -341,6 +341,9 @@ fn process_builtin_attrs(
                 AttributeKind::RustcDeallocator => {
                     codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR
                 }
+                AttributeKind::RustcReallocator => {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR
+                }
                 _ => {}
             }
         }
@@ -351,7 +354,6 @@ fn process_builtin_attrs(
 
         match name {
             sym::rustc_nounwind => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND,
-            sym::rustc_reallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR,
             sym::rustc_allocator_zeroed => {
                 codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
             }
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 6f91dd48cb5a..95c4b28f1421 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -978,6 +978,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_pass_indirectly_in_non_rustic_abis]`
     RustcPassIndirectlyInNonRusticAbis(Span),
 
+    /// Represents `#[rustc_reallocator]`
+    RustcReallocator,
+
     /// Represents `#[rustc_scalable_vector(N)]`
     RustcScalableVector {
         /// The base multiple of lanes that are in a scalable vector, if provided. `element_count`
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 9fcaf77f69a4..0dd2cefc83cf 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -121,6 +121,7 @@ impl AttributeKind {
             RustcNoImplicitAutorefs => Yes,
             RustcObjectLifetimeDefault => No,
             RustcPassIndirectlyInNonRusticAbis(..) => No,
+            RustcReallocator => No,
             RustcScalableVector { .. } => Yes,
             RustcShouldNotBeCalledOnConstItems(..) => Yes,
             RustcSimdMonomorphizeLaneLimit(..) => Yes, // Affects layout computation, which needs to work cross-crate
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 83a4bd9f0cda..4e904b6f11ad 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -318,6 +318,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::NeedsAllocator
                     | AttributeKind::RustcAllocator
                     | AttributeKind::RustcDeallocator
+                    | AttributeKind::RustcReallocator
                 ) => { /* do nothing  */ }
                 Attribute::Unparsed(attr_item) => {
                     style = Some(attr_item.style);
@@ -362,7 +363,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_do_not_const_check
                             | sym::rustc_reservation_impl
                             | sym::rustc_doc_primitive
-                            | sym::rustc_reallocator
                             | sym::rustc_conversion_suggestion
                             | sym::rustc_allocator_zeroed
                             | sym::rustc_allocator_zeroed_variant

From 21c9bd7692a9ed139e12c2bfc367b9c71160fcd5 Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 19:47:09 +0100
Subject: [PATCH 0356/1843] Port `#[rustc_allocator_zeroed]` to attr parser

---
 .../src/attributes/rustc_allocator.rs                  | 10 ++++++++++
 compiler/rustc_attr_parsing/src/context.rs             |  4 +++-
 compiler/rustc_codegen_ssa/src/codegen_attrs.rs        |  6 +++---
 compiler/rustc_hir/src/attrs/data_structures.rs        |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_passes/src/check_attr.rs                |  2 +-
 6 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
index 9324f0c01156..4622348ef1db 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
@@ -10,6 +10,16 @@ impl NoArgsAttributeParser for RustcAllocatorParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcAllocator;
 }
 
+pub(crate) struct RustcAllocatorZeroedParser;
+
+impl NoArgsAttributeParser for RustcAllocatorZeroedParser {
+    const PATH: &[Symbol] = &[sym::rustc_allocator_zeroed];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcAllocatorZeroed;
+}
+
 pub(crate) struct RustcDeallocatorParser;
 
 impl NoArgsAttributeParser for RustcDeallocatorParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 19e03f1ad3eb..7e69cbf1ff6d 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -65,7 +65,8 @@ use crate::attributes::proc_macro_attrs::{
 use crate::attributes::prototype::CustomMirParser;
 use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
 use crate::attributes::rustc_allocator::{
-    RustcAllocatorParser, RustcDeallocatorParser, RustcReallocatorParser,
+    RustcAllocatorParser, RustcAllocatorZeroedParser, RustcDeallocatorParser,
+    RustcReallocatorParser,
 };
 use crate::attributes::rustc_dump::{
     RustcDumpDefParents, RustcDumpItemBounds, RustcDumpPredicates, RustcDumpUserArgs,
@@ -277,6 +278,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 86ea1ac94b4e..1b0427e7e676 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -344,6 +344,9 @@ fn process_builtin_attrs(
                 AttributeKind::RustcReallocator => {
                     codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR
                 }
+                AttributeKind::RustcAllocatorZeroed => {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
+                }
                 _ => {}
             }
         }
@@ -354,9 +357,6 @@ fn process_builtin_attrs(
 
         match name {
             sym::rustc_nounwind => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND,
-            sym::rustc_allocator_zeroed => {
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
-            }
             sym::patchable_function_entry => {
                 codegen_fn_attrs.patchable_function_entry =
                     parse_patchable_function_entry(tcx, attr);
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 95c4b28f1421..1c0562d38d8a 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -909,6 +909,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_allocator]`
     RustcAllocator,
 
+    /// Represents `#[rustc_allocator_zeroed]`
+    RustcAllocatorZeroed,
+
     /// Represents `#[rustc_builtin_macro]`.
     RustcBuiltinMacro { builtin_name: Option, helper_attrs: ThinVec, span: Span },
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 0dd2cefc83cf..803aca49b406 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -98,6 +98,7 @@ impl AttributeKind {
             RecursionLimit { .. } => No,
             Repr { .. } => No,
             RustcAllocator => No,
+            RustcAllocatorZeroed => No,
             RustcBuiltinMacro { .. } => Yes,
             RustcCoherenceIsCore(..) => No,
             RustcDeallocator => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 4e904b6f11ad..ca0d6b90a1c4 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -317,6 +317,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDumpVtable(..)
                     | AttributeKind::NeedsAllocator
                     | AttributeKind::RustcAllocator
+                    | AttributeKind::RustcAllocatorZeroed
                     | AttributeKind::RustcDeallocator
                     | AttributeKind::RustcReallocator
                 ) => { /* do nothing  */ }
@@ -364,7 +365,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_reservation_impl
                             | sym::rustc_doc_primitive
                             | sym::rustc_conversion_suggestion
-                            | sym::rustc_allocator_zeroed
                             | sym::rustc_allocator_zeroed_variant
                             | sym::rustc_deprecated_safe_2024
                             | sym::rustc_test_marker

From 9a931e8bf2c4a329df8f32d587e056d7b127ec5a Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 20:02:41 +0100
Subject: [PATCH 0357/1843] Port `#[rustc_allocator_zeroed_variant]` to attr
 parser

---
 .../src/attributes/rustc_allocator.rs         | 19 +++++++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs    |  5 +++--
 compiler/rustc_codegen_llvm/src/attributes.rs |  5 ++---
 .../rustc_hir/src/attrs/data_structures.rs    |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 6 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
index 4622348ef1db..5782f9473a99 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_allocator.rs
@@ -20,6 +20,25 @@ impl NoArgsAttributeParser for RustcAllocatorZeroedParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcAllocatorZeroed;
 }
 
+pub(crate) struct RustcAllocatorZeroedVariantParser;
+
+impl SingleAttributeParser for RustcAllocatorZeroedVariantParser {
+    const PATH: &[Symbol] = &[sym::rustc_allocator_zeroed_variant];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Fn), Allow(Target::ForeignFn)]);
+    const TEMPLATE: AttributeTemplate = template!(NameValueStr: "function");
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option {
+        let Some(name) = args.name_value().and_then(NameValueParser::value_as_str) else {
+            cx.expected_name_value(cx.attr_span, None);
+            return None;
+        };
+
+        Some(AttributeKind::RustcAllocatorZeroedVariant { name })
+    }
+}
+
 pub(crate) struct RustcDeallocatorParser;
 
 impl NoArgsAttributeParser for RustcDeallocatorParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 7e69cbf1ff6d..7c0182c7e063 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -65,8 +65,8 @@ use crate::attributes::proc_macro_attrs::{
 use crate::attributes::prototype::CustomMirParser;
 use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
 use crate::attributes::rustc_allocator::{
-    RustcAllocatorParser, RustcAllocatorZeroedParser, RustcDeallocatorParser,
-    RustcReallocatorParser,
+    RustcAllocatorParser, RustcAllocatorZeroedParser, RustcAllocatorZeroedVariantParser,
+    RustcDeallocatorParser, RustcReallocatorParser,
 };
 use crate::attributes::rustc_dump::{
     RustcDumpDefParents, RustcDumpItemBounds, RustcDumpPredicates, RustcDumpUserArgs,
@@ -227,6 +227,7 @@ attribute_parsers!(
         Single,
         Single,
         Single,
+        Single,
         Single,
         Single,
         Single,
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index a25ce9e5a90a..28e91a25a21a 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -1,6 +1,7 @@
 //! Set and unset common attributes on LLVM values.
 use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr, RtsanSetting};
 use rustc_hir::def_id::DefId;
+use rustc_hir::find_attr;
 use rustc_middle::middle::codegen_fn_attrs::{
     CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, SanitizerFnAttrs,
 };
@@ -470,9 +471,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     {
         to_add.push(create_alloc_family_attr(cx.llcx));
         if let Some(instance) = instance
-            && let Some(zv) =
-                tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant)
-            && let Some(name) = zv.value_str()
+            && let Some(name) = find_attr!(tcx.get_all_attrs(instance.def_id()), rustc_hir::attrs::AttributeKind::RustcAllocatorZeroedVariant {name} => name)
         {
             to_add.push(llvm::CreateAttrStringValue(
                 cx.llcx,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 1c0562d38d8a..eff871cb8bb4 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -912,6 +912,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_allocator_zeroed]`
     RustcAllocatorZeroed,
 
+    /// Represents `#[rustc_allocator_zeroed_variant]`
+    RustcAllocatorZeroedVariant { name: Symbol },
+
     /// Represents `#[rustc_builtin_macro]`.
     RustcBuiltinMacro { builtin_name: Option, helper_attrs: ThinVec, span: Span },
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 803aca49b406..28b41ac70925 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -99,6 +99,7 @@ impl AttributeKind {
             Repr { .. } => No,
             RustcAllocator => No,
             RustcAllocatorZeroed => No,
+            RustcAllocatorZeroedVariant { .. } => Yes,
             RustcBuiltinMacro { .. } => Yes,
             RustcCoherenceIsCore(..) => No,
             RustcDeallocator => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index ca0d6b90a1c4..1e723dd6b464 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -318,6 +318,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::NeedsAllocator
                     | AttributeKind::RustcAllocator
                     | AttributeKind::RustcAllocatorZeroed
+                    | AttributeKind::RustcAllocatorZeroedVariant { .. }
                     | AttributeKind::RustcDeallocator
                     | AttributeKind::RustcReallocator
                 ) => { /* do nothing  */ }
@@ -365,7 +366,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_reservation_impl
                             | sym::rustc_doc_primitive
                             | sym::rustc_conversion_suggestion
-                            | sym::rustc_allocator_zeroed_variant
                             | sym::rustc_deprecated_safe_2024
                             | sym::rustc_test_marker
                             | sym::rustc_abi

From 2b8f4a562f24afcb45f4f12b109c13beb5b45f75 Mon Sep 17 00:00:00 2001
From: The 8472 
Date: Sun, 29 Sep 2024 00:27:50 +0200
Subject: [PATCH 0358/1843] avoid phi node for pointers flowing into Vec
 appends

---
 library/alloc/src/slice.rs                    | 11 ++++---
 library/alloc/src/vec/mod.rs                  |  6 +++-
 .../lib-optimizations/append-elements.rs      | 33 +++++++++++++++++++
 3 files changed, 45 insertions(+), 5 deletions(-)
 create mode 100644 tests/codegen-llvm/lib-optimizations/append-elements.rs

diff --git a/library/alloc/src/slice.rs b/library/alloc/src/slice.rs
index e7d0fc3454ee..634747ca3f84 100644
--- a/library/alloc/src/slice.rs
+++ b/library/alloc/src/slice.rs
@@ -444,13 +444,16 @@ impl [T] {
         impl ConvertVec for T {
             #[inline]
             fn to_vec(s: &[Self], alloc: A) -> Vec {
-                let mut v = Vec::with_capacity_in(s.len(), alloc);
+                let len = s.len();
+                let mut v = Vec::with_capacity_in(len, alloc);
                 // SAFETY:
                 // allocated above with the capacity of `s`, and initialize to `s.len()` in
                 // ptr::copy_to_non_overlapping below.
-                unsafe {
-                    s.as_ptr().copy_to_nonoverlapping(v.as_mut_ptr(), s.len());
-                    v.set_len(s.len());
+                if len > 0 {
+                    unsafe {
+                        s.as_ptr().copy_to_nonoverlapping(v.as_mut_ptr(), len);
+                        v.set_len(len);
+                    }
                 }
                 v
             }
diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs
index 379e964f0a0c..ac86399df7ab 100644
--- a/library/alloc/src/vec/mod.rs
+++ b/library/alloc/src/vec/mod.rs
@@ -2818,7 +2818,11 @@ impl Vec {
         let count = other.len();
         self.reserve(count);
         let len = self.len();
-        unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) };
+        if count > 0 {
+            unsafe {
+                ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count)
+            };
+        }
         self.len += count;
     }
 
diff --git a/tests/codegen-llvm/lib-optimizations/append-elements.rs b/tests/codegen-llvm/lib-optimizations/append-elements.rs
new file mode 100644
index 000000000000..b8657104d665
--- /dev/null
+++ b/tests/codegen-llvm/lib-optimizations/append-elements.rs
@@ -0,0 +1,33 @@
+//@ compile-flags: -O -Zmerge-functions=disabled
+//@ needs-deterministic-layouts
+//@ min-llvm-version: 21
+#![crate_type = "lib"]
+
+//! Check that a temporary intermediate allocations can eliminated and replaced
+//! with memcpy forwarding.
+//! This requires Vec code to be structured in a way that avoids phi nodes from the
+//! zero-capacity length flowing into the memcpy arguments.
+
+// CHECK-LABEL: @vec_append_with_temp_alloc
+// CHECK-SAME: ptr{{.*}}[[DST:%[a-z]+]]{{.*}}ptr{{.*}}[[SRC:%[a-z]+]]
+#[no_mangle]
+pub fn vec_append_with_temp_alloc(dst: &mut Vec, src: &[u8]) {
+    // CHECK-NOT: call void @llvm.memcpy
+    // CHECK: call void @llvm.memcpy.{{.*}}[[DST]].i{{.*}}[[SRC]]
+    // CHECK-NOT: call void @llvm.memcpy
+    let temp = src.to_vec();
+    dst.extend(&temp);
+    // CHECK: ret
+}
+
+// CHECK-LABEL: @string_append_with_temp_alloc
+// CHECK-SAME: ptr{{.*}}[[DST:%[a-z]+]]{{.*}}ptr{{.*}}[[SRC:%[a-z]+]]
+#[no_mangle]
+pub fn string_append_with_temp_alloc(dst: &mut String, src: &str) {
+    // CHECK-NOT: call void @llvm.memcpy
+    // CHECK: call void @llvm.memcpy.{{.*}}[[DST]].i{{.*}}[[SRC]]
+    // CHECK-NOT: call void @llvm.memcpy
+    let temp = src.to_string();
+    dst.push_str(&temp);
+    // CHECK: ret
+}

From 7216b035fa406b924099d1b1e8c4f392e4e02597 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 18 Jan 2026 21:27:40 +0100
Subject: [PATCH 0359/1843] Factor out diagnostic slug checking from
 `DiagnosticDerive` and `LintDiagnosticDerive`

---
 .../src/diagnostics/diagnostic.rs             | 100 +++---------------
 .../src/diagnostics/diagnostic_builder.rs     |  49 +++++++++
 2 files changed, 66 insertions(+), 83 deletions(-)

diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic.rs b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
index 185d07049669..7e784e3464e9 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
@@ -4,12 +4,10 @@ use std::cell::RefCell;
 
 use proc_macro2::TokenStream;
 use quote::quote;
-use syn::spanned::Spanned;
 use synstructure::Structure;
 
 use crate::diagnostics::diagnostic_builder::DiagnosticDeriveKind;
-use crate::diagnostics::error::{DiagnosticDeriveError, span_err};
-use crate::diagnostics::utils::SetOnce;
+use crate::diagnostics::error::DiagnosticDeriveError;
 
 /// The central struct for constructing the `into_diag` method from an annotated struct.
 pub(crate) struct DiagnosticDerive<'a> {
@@ -29,36 +27,16 @@ impl<'a> DiagnosticDerive<'a> {
             let preamble = builder.preamble(variant);
             let body = builder.body(variant);
 
-            let init = match builder.slug.value_ref() {
-                None => {
-                    span_err(builder.span, "diagnostic slug not specified")
-                        .help(
-                            "specify the slug as the first argument to the `#[diag(...)]` \
-                            attribute, such as `#[diag(hir_analysis_example_error)]`",
-                        )
-                        .emit();
-                    return DiagnosticDeriveError::ErrorHandled.to_compile_error();
-                }
-                Some(slug)
-                    if let Some(Mismatch { slug_name, crate_name, slug_prefix }) =
-                        Mismatch::check(slug) =>
-                {
-                    span_err(slug.span().unwrap(), "diagnostic slug and crate name do not match")
-                        .note(format!("slug is `{slug_name}` but the crate name is `{crate_name}`"))
-                        .help(format!("expected a slug starting with `{slug_prefix}_...`"))
-                        .emit();
-                    return DiagnosticDeriveError::ErrorHandled.to_compile_error();
-                }
-                Some(slug) => {
-                    slugs.borrow_mut().push(slug.clone());
-                    quote! {
-                        let mut diag = rustc_errors::Diag::new(
-                            dcx,
-                            level,
-                            crate::fluent_generated::#slug
-                        );
-                    }
-                }
+            let Some(slug) = builder.primary_message() else {
+                return DiagnosticDeriveError::ErrorHandled.to_compile_error();
+            };
+            slugs.borrow_mut().push(slug.clone());
+            let init = quote! {
+                let mut diag = rustc_errors::Diag::new(
+                    dcx,
+                    level,
+                    crate::fluent_generated::#slug
+                );
             };
 
             let formatting_init = &builder.formatting_init;
@@ -113,32 +91,12 @@ impl<'a> LintDiagnosticDerive<'a> {
             let preamble = builder.preamble(variant);
             let body = builder.body(variant);
 
-            let primary_message = match builder.slug.value_ref() {
-                None => {
-                    span_err(builder.span, "diagnostic slug not specified")
-                        .help(
-                            "specify the slug as the first argument to the attribute, such as \
-                            `#[diag(compiletest_example)]`",
-                        )
-                        .emit();
-                    DiagnosticDeriveError::ErrorHandled.to_compile_error()
-                }
-                Some(slug)
-                    if let Some(Mismatch { slug_name, crate_name, slug_prefix }) =
-                        Mismatch::check(slug) =>
-                {
-                    span_err(slug.span().unwrap(), "diagnostic slug and crate name do not match")
-                        .note(format!("slug is `{slug_name}` but the crate name is `{crate_name}`"))
-                        .help(format!("expected a slug starting with `{slug_prefix}_...`"))
-                        .emit();
-                    DiagnosticDeriveError::ErrorHandled.to_compile_error()
-                }
-                Some(slug) => {
-                    slugs.borrow_mut().push(slug.clone());
-                    quote! {
-                        diag.primary_message(crate::fluent_generated::#slug);
-                    }
-                }
+            let Some(slug) = builder.primary_message() else {
+                return DiagnosticDeriveError::ErrorHandled.to_compile_error();
+            };
+            slugs.borrow_mut().push(slug.clone());
+            let primary_message = quote! {
+                diag.primary_message(crate::fluent_generated::#slug);
             };
 
             let formatting_init = &builder.formatting_init;
@@ -172,30 +130,6 @@ impl<'a> LintDiagnosticDerive<'a> {
     }
 }
 
-struct Mismatch {
-    slug_name: String,
-    crate_name: String,
-    slug_prefix: String,
-}
-
-impl Mismatch {
-    /// Checks whether the slug starts with the crate name it's in.
-    fn check(slug: &syn::Path) -> Option {
-        // If this is missing we're probably in a test, so bail.
-        let crate_name = std::env::var("CARGO_CRATE_NAME").ok()?;
-
-        // If we're not in a "rustc_" crate, bail.
-        let Some(("rustc", slug_prefix)) = crate_name.split_once('_') else { return None };
-
-        let slug_name = slug.segments.first()?.ident.to_string();
-        if !slug_name.starts_with(slug_prefix) {
-            Some(Mismatch { slug_name, slug_prefix: slug_prefix.to_string(), crate_name })
-        } else {
-            None
-        }
-    }
-}
-
 /// Generates a `#[test]` that verifies that all referenced variables
 /// exist on this structure.
 fn generate_test(slug: &syn::Path, structure: &Structure<'_>) -> TokenStream {
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
index 1055f27c1e48..cbc70b55d7ee 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
@@ -110,6 +110,31 @@ impl DiagnosticDeriveKind {
 }
 
 impl DiagnosticDeriveVariantBuilder {
+    pub(crate) fn primary_message(&self) -> Option<&Path> {
+        match self.slug.value_ref() {
+            None => {
+                span_err(self.span, "diagnostic slug not specified")
+                    .help(
+                        "specify the slug as the first argument to the `#[diag(...)]` \
+                            attribute, such as `#[diag(hir_analysis_example_error)]`",
+                    )
+                    .emit();
+                None
+            }
+            Some(slug)
+                if let Some(Mismatch { slug_name, crate_name, slug_prefix }) =
+                    Mismatch::check(slug) =>
+            {
+                span_err(slug.span().unwrap(), "diagnostic slug and crate name do not match")
+                    .note(format!("slug is `{slug_name}` but the crate name is `{crate_name}`"))
+                    .help(format!("expected a slug starting with `{slug_prefix}_...`"))
+                    .emit();
+                None
+            }
+            Some(slug) => Some(slug),
+        }
+    }
+
     /// Generates calls to `code` and similar functions based on the attributes on the type or
     /// variant.
     pub(crate) fn preamble(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
@@ -504,3 +529,27 @@ impl DiagnosticDeriveVariantBuilder {
         }
     }
 }
+
+struct Mismatch {
+    slug_name: String,
+    crate_name: String,
+    slug_prefix: String,
+}
+
+impl Mismatch {
+    /// Checks whether the slug starts with the crate name it's in.
+    fn check(slug: &syn::Path) -> Option {
+        // If this is missing we're probably in a test, so bail.
+        let crate_name = std::env::var("CARGO_CRATE_NAME").ok()?;
+
+        // If we're not in a "rustc_" crate, bail.
+        let Some(("rustc", slug_prefix)) = crate_name.split_once('_') else { return None };
+
+        let slug_name = slug.segments.first()?.ident.to_string();
+        if slug_name.starts_with(slug_prefix) {
+            return None;
+        }
+
+        Some(Mismatch { slug_name, slug_prefix: slug_prefix.to_string(), crate_name })
+    }
+}

From 7ec4a8e798cc37bdf4c9cac1c4a481f6c1d22a0d Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 18 Jan 2026 22:36:39 +0100
Subject: [PATCH 0360/1843] Update uitests

---
 tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr
index 90ad21ef08f9..77c48aceca8e 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr
@@ -384,7 +384,7 @@ error: derive(Diagnostic): diagnostic slug not specified
 LL | #[lint(no_crate_example, code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the attribute, such as `#[diag(compiletest_example)]`
+   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): attribute specified multiple times
   --> $DIR/diagnostic-derive.rs:613:53

From e668836c929e1a30b0c53583567e7c6cf42fe4da Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 18 Jan 2026 22:40:55 +0100
Subject: [PATCH 0361/1843] Fix capitalization of error messages

---
 compiler/rustc_attr_parsing/messages.ftl |  2 +-
 compiler/rustc_const_eval/messages.ftl   | 12 ++++++------
 compiler/rustc_lint/messages.ftl         |  2 +-
 compiler/rustc_monomorphize/messages.ftl |  2 +-
 compiler/rustc_passes/messages.ftl       |  2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_attr_parsing/messages.ftl b/compiler/rustc_attr_parsing/messages.ftl
index 4b4358ab0a9c..3e4c1a9dfad8 100644
--- a/compiler/rustc_attr_parsing/messages.ftl
+++ b/compiler/rustc_attr_parsing/messages.ftl
@@ -230,7 +230,7 @@ attr_parsing_unstable_cfg_target_compact =
     compact `cfg(target(..))` is experimental and subject to change
 
 attr_parsing_unstable_feature_bound_incompatible_stability = item annotated with `#[unstable_feature_bound]` should not be stable
-    .help = If this item is meant to be stable, do not use any functions annotated with `#[unstable_feature_bound]`. Otherwise, mark this item as unstable with `#[unstable]`
+    .help = if this item is meant to be stable, do not use any functions annotated with `#[unstable_feature_bound]`. Otherwise, mark this item as unstable with `#[unstable]`
 
 attr_parsing_unsupported_instruction_set = target `{$current_target}` does not support `#[instruction_set({$instruction_set}::*)]`
 
diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl
index 38ab46a7bb5b..4aa0a0b2a96f 100644
--- a/compiler/rustc_const_eval/messages.ftl
+++ b/compiler/rustc_const_eval/messages.ftl
@@ -140,9 +140,9 @@ const_eval_incompatible_return_types =
 const_eval_interior_mutable_borrow_escaping =
     interior mutable shared borrows of temporaries that have their lifetime extended until the end of the program are not allowed
     .label = this borrow of an interior mutable value refers to such a temporary
-    .note = Temporaries in constants and statics can have their lifetime extended until the end of the program
-    .note2 = To avoid accidentally creating global mutable state, such temporaries must be immutable
-    .help = If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+    .note = temporaries in constants and statics can have their lifetime extended until the end of the program
+    .note2 = to avoid accidentally creating global mutable state, such temporaries must be immutable
+    .help = if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 const_eval_intern_kind = {$kind ->
     [static] static
@@ -225,9 +225,9 @@ const_eval_modified_global =
 const_eval_mutable_borrow_escaping =
     mutable borrows of temporaries that have their lifetime extended until the end of the program are not allowed
     .label = this mutable borrow refers to such a temporary
-    .note = Temporaries in constants and statics can have their lifetime extended until the end of the program
-    .note2 = To avoid accidentally creating global mutable state, such temporaries must be immutable
-    .help = If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+    .note = temporaries in constants and statics can have their lifetime extended until the end of the program
+    .note2 = to avoid accidentally creating global mutable state, such temporaries must be immutable
+    .help = if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 const_eval_mutable_ptr_in_final = encountered mutable pointer in final value of {const_eval_intern_kind}
 
diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl
index f5b882494863..e38130aa9a29 100644
--- a/compiler/rustc_lint/messages.ftl
+++ b/compiler/rustc_lint/messages.ftl
@@ -542,7 +542,7 @@ lint_invalid_style = {$is_used_as_inner ->
         [false] crate-level attribute should be an inner attribute: add an exclamation mark: `#![{$name}]`
         *[other] the `#![{$name}]` attribute can only be used at the crate root
     }
-    .note = This attribute does not have an `!`, which means it is applied to this {$target}
+    .note = this attribute does not have an `!`, which means it is applied to this {$target}
 
 lint_invalid_target = `#[{$name}]` attribute cannot be used on {$target}
     .warn = {-lint_previously_accepted}
diff --git a/compiler/rustc_monomorphize/messages.ftl b/compiler/rustc_monomorphize/messages.ftl
index 09500ba73359..9c791208c093 100644
--- a/compiler/rustc_monomorphize/messages.ftl
+++ b/compiler/rustc_monomorphize/messages.ftl
@@ -62,7 +62,7 @@ monomorphize_encountered_error_while_instantiating_global_asm =
 monomorphize_large_assignments =
     moving {$size} bytes
     .label = value moved from here
-    .note = The current maximum size is {$limit}, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+    .note = the current maximum size is {$limit}, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 monomorphize_no_optimized_mir =
     missing optimized MIR for `{$instance}` in the crate `{$crate_name}`
diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl
index 51c439b4243c..395d940ddae6 100644
--- a/compiler/rustc_passes/messages.ftl
+++ b/compiler/rustc_passes/messages.ftl
@@ -376,7 +376,7 @@ passes_no_main_function =
     }
     .consider_adding_main_to_file = consider adding a `main` function to `{$filename}`
     .consider_adding_main_at_crate = consider adding a `main` function at the crate level
-    .teach_note = If you don't know the basics of Rust, you can go look to the Rust Book to get started: https://doc.rust-lang.org/book/
+    .teach_note = if you don't know the basics of Rust, you can go look to the Rust Book to get started: https://doc.rust-lang.org/book/
     .non_function_main = non-function item at `crate::main` is found
 
 passes_non_exhaustive_with_default_field_values =

From f5a1fc75ad10186b13ad34a65e176760b7306630 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 18 Jan 2026 22:41:00 +0100
Subject: [PATCH 0362/1843] Update uitests

---
 .../ui/attributes/crate-only-as-outer.stderr  |  2 +-
 tests/ui/attributes/malformed-attrs.stderr    |  2 +-
 tests/ui/attributes/malformed-no-std.stderr   |  4 +-
 .../consts/const-mut-refs/issue-76510.stderr  |  6 +-
 .../const-mut-refs/mut_ref_in_final.stderr    | 36 +++++------
 .../const-promoted-opaque.atomic.stderr       |  6 +-
 .../issue-17718-const-bad-values.stderr       |  6 +-
 .../ui/consts/issue-17718-const-borrow.stderr | 18 +++---
 tests/ui/consts/partial_qualif.stderr         |  6 +-
 tests/ui/consts/qualif_overwrite.stderr       |  6 +-
 tests/ui/consts/qualif_overwrite_2.stderr     |  6 +-
 tests/ui/consts/refs-to-cell-in-final.stderr  | 18 +++---
 .../consts/write_to_static_via_mut_ref.stderr |  6 +-
 tests/ui/error-codes/E0017.stderr             | 12 ++--
 tests/ui/error-codes/E0492.stderr             | 12 ++--
 .../issue-43106-gating-of-builtin-attrs.rs    | 60 +++++++++----------
 ...issue-43106-gating-of-builtin-attrs.stderr | 60 +++++++++----------
 tests/ui/issues/issue-46604.stderr            |  6 +-
 .../copy_into_box_rc_arc.stderr               |  4 +-
 .../large_assignments/copy_into_fn.stderr     |  6 +-
 .../lint/large_assignments/inline_mir.stderr  |  4 +-
 .../large_future.attribute.stderr             |  4 +-
 .../large_future.option.stderr                |  4 +-
 .../move_into_box_rc_arc.stderr               |  2 +-
 .../large_assignments/move_into_fn.stderr     |  4 +-
 .../concat-in-crate-name-issue-137687.stderr  |  2 +-
 .../unused/unused-attr-macro-rules.stderr     |  2 +-
 .../statics/check-immutable-mut-slices.stderr |  6 +-
 ...eature_bound_incompatible_stability.stderr |  2 +-
 29 files changed, 156 insertions(+), 156 deletions(-)

diff --git a/tests/ui/attributes/crate-only-as-outer.stderr b/tests/ui/attributes/crate-only-as-outer.stderr
index 270f02af9873..c1787a73d290 100644
--- a/tests/ui/attributes/crate-only-as-outer.stderr
+++ b/tests/ui/attributes/crate-only-as-outer.stderr
@@ -4,7 +4,7 @@ error: crate-level attribute should be an inner attribute: add an exclamation ma
 LL | #[crate_name = "owo"]
    | ^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/crate-only-as-outer.rs:5:1
    |
 LL | fn main() {}
diff --git a/tests/ui/attributes/malformed-attrs.stderr b/tests/ui/attributes/malformed-attrs.stderr
index 04f51f54b2cd..3d51731df792 100644
--- a/tests/ui/attributes/malformed-attrs.stderr
+++ b/tests/ui/attributes/malformed-attrs.stderr
@@ -771,7 +771,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[crate_name]
    | ^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/malformed-attrs.rs:116:1
    |
 LL | / fn test() {
diff --git a/tests/ui/attributes/malformed-no-std.stderr b/tests/ui/attributes/malformed-no-std.stderr
index 89d7ee410d70..e994e28e030f 100644
--- a/tests/ui/attributes/malformed-no-std.stderr
+++ b/tests/ui/attributes/malformed-no-std.stderr
@@ -58,7 +58,7 @@ error: crate-level attribute should be an inner attribute: add an exclamation ma
 LL | #[no_std]
    | ^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this extern crate
+note: this attribute does not have an `!`, which means it is applied to this extern crate
   --> $DIR/malformed-no-std.rs:26:1
    |
 LL | extern crate core;
@@ -75,7 +75,7 @@ error: crate-level attribute should be an inner attribute: add an exclamation ma
 LL | #[no_core]
    | ^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this extern crate
+note: this attribute does not have an `!`, which means it is applied to this extern crate
   --> $DIR/malformed-no-std.rs:26:1
    |
 LL | extern crate core;
diff --git a/tests/ui/consts/const-mut-refs/issue-76510.stderr b/tests/ui/consts/const-mut-refs/issue-76510.stderr
index 3a6c95141e52..82c9d523e738 100644
--- a/tests/ui/consts/const-mut-refs/issue-76510.stderr
+++ b/tests/ui/consts/const-mut-refs/issue-76510.stderr
@@ -4,9 +4,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const S: &'static mut str = &mut " hello ";
    |                             ^^^^^^^^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/consts/const-mut-refs/mut_ref_in_final.stderr b/tests/ui/consts/const-mut-refs/mut_ref_in_final.stderr
index da6f2a28d5a8..8f54b4eda227 100644
--- a/tests/ui/consts/const-mut-refs/mut_ref_in_final.stderr
+++ b/tests/ui/consts/const-mut-refs/mut_ref_in_final.stderr
@@ -4,9 +4,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const B: *mut i32 = &mut 4;
    |                     ^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0764]: mutable borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/mut_ref_in_final.rs:21:35
@@ -14,9 +14,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const B3: Option<&mut i32> = Some(&mut 42);
    |                                   ^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0716]: temporary value dropped while borrowed
   --> $DIR/mut_ref_in_final.rs:24:42
@@ -86,9 +86,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | static RAW_MUT_CAST_S: SyncPtr = SyncPtr { x : &mut 42 as *mut _ as *const _ };
    |                                                     ^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0764]: mutable borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/mut_ref_in_final.rs:73:54
@@ -96,9 +96,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | static RAW_MUT_COERCE_S: SyncPtr = SyncPtr { x: &mut 0 };
    |                                                      ^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0764]: mutable borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/mut_ref_in_final.rs:75:52
@@ -106,9 +106,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const RAW_MUT_CAST_C: SyncPtr = SyncPtr { x : &mut 42 as *mut _ as *const _ };
    |                                                    ^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0764]: mutable borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/mut_ref_in_final.rs:77:53
@@ -116,9 +116,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const RAW_MUT_COERCE_C: SyncPtr = SyncPtr { x: &mut 0 };
    |                                                     ^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0080]: constructing invalid value at ..0: encountered a dangling reference (0x2a[noalloc] has no provenance)
   --> $DIR/mut_ref_in_final.rs:86:5
diff --git a/tests/ui/consts/const-promoted-opaque.atomic.stderr b/tests/ui/consts/const-promoted-opaque.atomic.stderr
index 64cc7b3a3292..ac31992d0631 100644
--- a/tests/ui/consts/const-promoted-opaque.atomic.stderr
+++ b/tests/ui/consts/const-promoted-opaque.atomic.stderr
@@ -13,9 +13,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | const BAZ: &Foo = &FOO;
    |                   ^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0716]: temporary value dropped while borrowed
   --> $DIR/const-promoted-opaque.rs:40:26
diff --git a/tests/ui/consts/issue-17718-const-bad-values.stderr b/tests/ui/consts/issue-17718-const-bad-values.stderr
index 11e11adcb5ae..eebfa5d6ea40 100644
--- a/tests/ui/consts/issue-17718-const-bad-values.stderr
+++ b/tests/ui/consts/issue-17718-const-bad-values.stderr
@@ -4,9 +4,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const C1: &'static mut [usize] = &mut [];
    |                                  ^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/consts/issue-17718-const-borrow.stderr b/tests/ui/consts/issue-17718-const-borrow.stderr
index 420a2c378a25..b801498c2028 100644
--- a/tests/ui/consts/issue-17718-const-borrow.stderr
+++ b/tests/ui/consts/issue-17718-const-borrow.stderr
@@ -4,9 +4,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | const B: &'static UnsafeCell = &A;
    |                                       ^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0492]: interior mutable shared borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/issue-17718-const-borrow.rs:9:39
@@ -14,9 +14,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | const E: &'static UnsafeCell = &D.a;
    |                                       ^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0492]: interior mutable shared borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/issue-17718-const-borrow.rs:11:23
@@ -24,9 +24,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | const F: &'static C = &D;
    |                       ^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/consts/partial_qualif.stderr b/tests/ui/consts/partial_qualif.stderr
index b7632eb868ac..f69fa1c46c01 100644
--- a/tests/ui/consts/partial_qualif.stderr
+++ b/tests/ui/consts/partial_qualif.stderr
@@ -4,9 +4,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL |     &{a}
    |     ^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/consts/qualif_overwrite.stderr b/tests/ui/consts/qualif_overwrite.stderr
index 4aaaa4b2ca90..1dc2bf7f1231 100644
--- a/tests/ui/consts/qualif_overwrite.stderr
+++ b/tests/ui/consts/qualif_overwrite.stderr
@@ -4,9 +4,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL |     &{a}
    |     ^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/consts/qualif_overwrite_2.stderr b/tests/ui/consts/qualif_overwrite_2.stderr
index bc1681418765..fb8ac601c67a 100644
--- a/tests/ui/consts/qualif_overwrite_2.stderr
+++ b/tests/ui/consts/qualif_overwrite_2.stderr
@@ -4,9 +4,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL |     &{a.0}
    |     ^^^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/consts/refs-to-cell-in-final.stderr b/tests/ui/consts/refs-to-cell-in-final.stderr
index ac866dbe7210..e30b5aa24e76 100644
--- a/tests/ui/consts/refs-to-cell-in-final.stderr
+++ b/tests/ui/consts/refs-to-cell-in-final.stderr
@@ -4,9 +4,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | static RAW_SYNC_S: SyncPtr> = SyncPtr { x: &Cell::new(42) };
    |                                                      ^^^^^^^^^^^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0492]: interior mutable shared borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/refs-to-cell-in-final.rs:15:53
@@ -14,9 +14,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | const RAW_SYNC_C: SyncPtr> = SyncPtr { x: &Cell::new(42) };
    |                                                     ^^^^^^^^^^^^^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0492]: interior mutable shared borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/refs-to-cell-in-final.rs:41:57
@@ -31,9 +31,9 @@ LL | |     x
 LL | | };
    | |_^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/consts/write_to_static_via_mut_ref.stderr b/tests/ui/consts/write_to_static_via_mut_ref.stderr
index ce44047f1550..be1f7178998a 100644
--- a/tests/ui/consts/write_to_static_via_mut_ref.stderr
+++ b/tests/ui/consts/write_to_static_via_mut_ref.stderr
@@ -4,9 +4,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | static OH_NO: &mut i32 = &mut 42;
    |                          ^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0594]: cannot assign to `*OH_NO`, as `OH_NO` is an immutable static item
   --> $DIR/write_to_static_via_mut_ref.rs:4:5
diff --git a/tests/ui/error-codes/E0017.stderr b/tests/ui/error-codes/E0017.stderr
index fcc57b9e5c3c..70186165d862 100644
--- a/tests/ui/error-codes/E0017.stderr
+++ b/tests/ui/error-codes/E0017.stderr
@@ -19,9 +19,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | const CR: &'static mut i32 = &mut C;
    |                              ^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0596]: cannot borrow immutable static item `X` as mutable
   --> $DIR/E0017.rs:11:39
@@ -52,9 +52,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | static CONST_REF: &'static mut i32 = &mut C;
    |                                      ^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 3 previous errors; 2 warnings emitted
 
diff --git a/tests/ui/error-codes/E0492.stderr b/tests/ui/error-codes/E0492.stderr
index 43a3a872e4e7..a5057e8baedb 100644
--- a/tests/ui/error-codes/E0492.stderr
+++ b/tests/ui/error-codes/E0492.stderr
@@ -4,9 +4,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | const B: &'static AtomicUsize = &A;
    |                                 ^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0492]: interior mutable shared borrows of temporaries that have their lifetime extended until the end of the program are not allowed
   --> $DIR/E0492.rs:5:34
@@ -14,9 +14,9 @@ error[E0492]: interior mutable shared borrows of temporaries that have their lif
 LL | static C: &'static AtomicUsize = &A;
    |                                  ^^ this borrow of an interior mutable value refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs
index 3e3235e658f6..ce2f9a4e6eeb 100644
--- a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs
+++ b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.rs
@@ -525,25 +525,25 @@ mod macro_escape {
 #[no_std]
 //~^ WARN crate-level attribute should be an inner attribute
 mod no_std {
-    //~^ NOTE This attribute does not have an `!`, which means it is applied to this module
+    //~^ NOTE this attribute does not have an `!`, which means it is applied to this module
     mod inner { #![no_std] }
 //~^ WARN the `#![no_std]` attribute can only be used at the crate root
 
     #[no_std] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this function
 
     #[no_std] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this struct
 
     #[no_std] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this type alias
 
     #[no_std] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation block
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this implementation block
 }
 
 // At time of authorship, #[proc_macro_derive = "2500"] signals error
@@ -786,25 +786,25 @@ mod must_use {
 #[windows_subsystem = "windows"]
 //~^ WARN crate-level attribute should be an inner attribute
 mod windows_subsystem {
-    //~^ NOTE This attribute does not have an `!`, which means it is applied to this module
+    //~^ NOTE this attribute does not have an `!`, which means it is applied to this module
     mod inner { #![windows_subsystem="windows"] }
     //~^ WARN the `#![windows_subsystem]` attribute can only be used at the crate root
 
     #[windows_subsystem = "windows"] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this function
 
     #[windows_subsystem = "windows"] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this struct
 
     #[windows_subsystem = "windows"] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this type alias
 
     #[windows_subsystem = "windows"] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation block
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this implementation block
 }
 
 // BROKEN USES OF CRATE-LEVEL BUILT-IN ATTRIBUTES
@@ -812,25 +812,25 @@ mod windows_subsystem {
 #[crate_name = "0900"]
 //~^ WARN crate-level attribute should be an inner attribute
 mod crate_name {
-//~^ NOTE This attribute does not have an `!`, which means it is applied to this module
+//~^ NOTE this attribute does not have an `!`, which means it is applied to this module
     mod inner { #![crate_name="0900"] }
 //~^ WARN the `#![crate_name]` attribute can only be used at the crate root
 
     #[crate_name = "0900"] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this function
 
     #[crate_name = "0900"] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this struct
 
     #[crate_name = "0900"] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this type alias
 
     #[crate_name = "0900"] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation block
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this implementation block
 }
 
 #[crate_type = "0800"]
@@ -885,25 +885,25 @@ mod feature {
 #[no_main]
 //~^ WARN crate-level attribute should be an inner attribute
 mod no_main_1 {
-    //~^ NOTE: This attribute does not have an `!`, which means it is applied to this module
+    //~^ NOTE: this attribute does not have an `!`, which means it is applied to this module
     mod inner { #![no_main] }
     //~^ WARN the `#![no_main]` attribute can only be used at the crate root
 
     #[no_main] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this function
 
     #[no_main] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this struct
 
     #[no_main] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this type alias
 
     #[no_main] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this implementation
 }
 
 #[no_builtins]
@@ -933,49 +933,49 @@ mod no_builtins {
 #[recursion_limit="0200"]
 //~^ WARN crate-level attribute should be an inner attribute
 mod recursion_limit {
-    //~^ NOTE This attribute does not have an `!`, which means it is applied to this module
+    //~^ NOTE this attribute does not have an `!`, which means it is applied to this module
     mod inner { #![recursion_limit="0200"] }
 //~^ WARN the `#![recursion_limit]` attribute can only be used at the crate root
 
     #[recursion_limit="0200"] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this function
 
     #[recursion_limit="0200"] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this struct
 
     #[recursion_limit="0200"] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this type alias
 
     #[recursion_limit="0200"] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation block
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this implementation block
 }
 
 #[type_length_limit="0100"]
 //~^ WARN crate-level attribute should be an inner attribute
 mod type_length_limit {
-    //~^ NOTE This attribute does not have an `!`, which means it is applied to this module
+    //~^ NOTE this attribute does not have an `!`, which means it is applied to this module
     mod inner { #![type_length_limit="0100"] }
 //~^ WARN the `#![type_length_limit]` attribute can only be used at the crate root
 
     #[type_length_limit="0100"] fn f() { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this function
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this function
 
     #[type_length_limit="0100"] struct S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this struct
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this struct
 
     #[type_length_limit="0100"] type T = S;
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this type alias
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this type alias
 
     #[type_length_limit="0100"] impl S { }
     //~^ WARN crate-level attribute should be an inner attribute
-    //~| NOTE This attribute does not have an `!`, which means it is applied to this implementation block
+    //~| NOTE this attribute does not have an `!`, which means it is applied to this implementation block
 }
 
 fn main() {}
diff --git a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr
index d89aec222be8..cbb80ccd753c 100644
--- a/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr
+++ b/tests/ui/feature-gates/issue-43106-gating-of-builtin-attrs.stderr
@@ -935,7 +935,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[no_std]
    | ^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this module
+note: this attribute does not have an `!`, which means it is applied to this module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:527:1
    |
 LL | / mod no_std {
@@ -957,7 +957,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_std] fn f() { }
    |     ^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:532:15
    |
 LL |     #[no_std] fn f() { }
@@ -969,7 +969,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_std] struct S;
    |     ^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this struct
+note: this attribute does not have an `!`, which means it is applied to this struct
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:536:15
    |
 LL |     #[no_std] struct S;
@@ -981,7 +981,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_std] type T = S;
    |     ^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this type alias
+note: this attribute does not have an `!`, which means it is applied to this type alias
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:540:15
    |
 LL |     #[no_std] type T = S;
@@ -993,7 +993,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_std] impl S { }
    |     ^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this implementation block
+note: this attribute does not have an `!`, which means it is applied to this implementation block
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:544:15
    |
 LL |     #[no_std] impl S { }
@@ -1212,7 +1212,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[windows_subsystem = "windows"]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this module
+note: this attribute does not have an `!`, which means it is applied to this module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:788:1
    |
 LL | / mod windows_subsystem {
@@ -1234,7 +1234,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[windows_subsystem = "windows"] fn f() { }
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:793:38
    |
 LL |     #[windows_subsystem = "windows"] fn f() { }
@@ -1246,7 +1246,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[windows_subsystem = "windows"] struct S;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this struct
+note: this attribute does not have an `!`, which means it is applied to this struct
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:797:38
    |
 LL |     #[windows_subsystem = "windows"] struct S;
@@ -1258,7 +1258,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[windows_subsystem = "windows"] type T = S;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this type alias
+note: this attribute does not have an `!`, which means it is applied to this type alias
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:801:38
    |
 LL |     #[windows_subsystem = "windows"] type T = S;
@@ -1270,7 +1270,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[windows_subsystem = "windows"] impl S { }
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this implementation block
+note: this attribute does not have an `!`, which means it is applied to this implementation block
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:805:38
    |
 LL |     #[windows_subsystem = "windows"] impl S { }
@@ -1282,7 +1282,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[crate_name = "0900"]
    | ^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this module
+note: this attribute does not have an `!`, which means it is applied to this module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:814:1
    |
 LL | / mod crate_name {
@@ -1304,7 +1304,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[crate_name = "0900"] fn f() { }
    |     ^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:819:28
    |
 LL |     #[crate_name = "0900"] fn f() { }
@@ -1316,7 +1316,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[crate_name = "0900"] struct S;
    |     ^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this struct
+note: this attribute does not have an `!`, which means it is applied to this struct
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:823:28
    |
 LL |     #[crate_name = "0900"] struct S;
@@ -1328,7 +1328,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[crate_name = "0900"] type T = S;
    |     ^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this type alias
+note: this attribute does not have an `!`, which means it is applied to this type alias
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:827:28
    |
 LL |     #[crate_name = "0900"] type T = S;
@@ -1340,7 +1340,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[crate_name = "0900"] impl S { }
    |     ^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this implementation block
+note: this attribute does not have an `!`, which means it is applied to this implementation block
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:831:28
    |
 LL |     #[crate_name = "0900"] impl S { }
@@ -1352,7 +1352,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[no_main]
    | ^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this module
+note: this attribute does not have an `!`, which means it is applied to this module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:887:1
    |
 LL | / mod no_main_1 {
@@ -1374,7 +1374,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_main] fn f() { }
    |     ^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:892:16
    |
 LL |     #[no_main] fn f() { }
@@ -1386,7 +1386,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_main] struct S;
    |     ^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this struct
+note: this attribute does not have an `!`, which means it is applied to this struct
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:896:16
    |
 LL |     #[no_main] struct S;
@@ -1398,7 +1398,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_main] type T = S;
    |     ^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this type alias
+note: this attribute does not have an `!`, which means it is applied to this type alias
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:900:16
    |
 LL |     #[no_main] type T = S;
@@ -1410,7 +1410,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[no_main] impl S { }
    |     ^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this implementation block
+note: this attribute does not have an `!`, which means it is applied to this implementation block
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:904:16
    |
 LL |     #[no_main] impl S { }
@@ -1422,7 +1422,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[recursion_limit="0200"]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this module
+note: this attribute does not have an `!`, which means it is applied to this module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:935:1
    |
 LL | / mod recursion_limit {
@@ -1444,7 +1444,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[recursion_limit="0200"] fn f() { }
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:940:31
    |
 LL |     #[recursion_limit="0200"] fn f() { }
@@ -1456,7 +1456,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[recursion_limit="0200"] struct S;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this struct
+note: this attribute does not have an `!`, which means it is applied to this struct
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:944:31
    |
 LL |     #[recursion_limit="0200"] struct S;
@@ -1468,7 +1468,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[recursion_limit="0200"] type T = S;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this type alias
+note: this attribute does not have an `!`, which means it is applied to this type alias
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:948:31
    |
 LL |     #[recursion_limit="0200"] type T = S;
@@ -1480,7 +1480,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[recursion_limit="0200"] impl S { }
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this implementation block
+note: this attribute does not have an `!`, which means it is applied to this implementation block
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:952:31
    |
 LL |     #[recursion_limit="0200"] impl S { }
@@ -1492,7 +1492,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL | #[type_length_limit="0100"]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this module
+note: this attribute does not have an `!`, which means it is applied to this module
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:959:1
    |
 LL | / mod type_length_limit {
@@ -1514,7 +1514,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[type_length_limit="0100"] fn f() { }
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this function
+note: this attribute does not have an `!`, which means it is applied to this function
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:964:33
    |
 LL |     #[type_length_limit="0100"] fn f() { }
@@ -1526,7 +1526,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[type_length_limit="0100"] struct S;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this struct
+note: this attribute does not have an `!`, which means it is applied to this struct
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:968:33
    |
 LL |     #[type_length_limit="0100"] struct S;
@@ -1538,7 +1538,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[type_length_limit="0100"] type T = S;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this type alias
+note: this attribute does not have an `!`, which means it is applied to this type alias
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:972:33
    |
 LL |     #[type_length_limit="0100"] type T = S;
@@ -1550,7 +1550,7 @@ warning: crate-level attribute should be an inner attribute: add an exclamation
 LL |     #[type_length_limit="0100"] impl S { }
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this implementation block
+note: this attribute does not have an `!`, which means it is applied to this implementation block
   --> $DIR/issue-43106-gating-of-builtin-attrs.rs:976:33
    |
 LL |     #[type_length_limit="0100"] impl S { }
diff --git a/tests/ui/issues/issue-46604.stderr b/tests/ui/issues/issue-46604.stderr
index abe3ad476c60..21abc498de12 100644
--- a/tests/ui/issues/issue-46604.stderr
+++ b/tests/ui/issues/issue-46604.stderr
@@ -4,9 +4,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | static buf: &mut [u8] = &mut [1u8,2,3,4,5,7];
    |                         ^^^^^^^^^^^^^^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error[E0594]: cannot assign to `buf[_]`, as `buf` is an immutable static item
   --> $DIR/issue-46604.rs:6:5
diff --git a/tests/ui/lint/large_assignments/copy_into_box_rc_arc.stderr b/tests/ui/lint/large_assignments/copy_into_box_rc_arc.stderr
index 6e42328a1113..b8e7abf4807c 100644
--- a/tests/ui/lint/large_assignments/copy_into_box_rc_arc.stderr
+++ b/tests/ui/lint/large_assignments/copy_into_box_rc_arc.stderr
@@ -4,7 +4,7 @@ error: moving 9999 bytes
 LL |     let _ = NotBox::new(data);
    |                         ^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/copy_into_box_rc_arc.rs:1:9
    |
@@ -19,7 +19,7 @@ LL | |             data,
 LL | |         }
    | |_________^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/lint/large_assignments/copy_into_fn.stderr b/tests/ui/lint/large_assignments/copy_into_fn.stderr
index f05fc33e17e1..a4c4800266af 100644
--- a/tests/ui/lint/large_assignments/copy_into_fn.stderr
+++ b/tests/ui/lint/large_assignments/copy_into_fn.stderr
@@ -4,7 +4,7 @@ error: moving 9999 bytes
 LL |     one_arg(Data([0; 9999]));
    |             ^^^^^^^^^^^^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/copy_into_fn.rs:5:9
    |
@@ -17,7 +17,7 @@ error: moving 9999 bytes
 LL |     many_args(Data([0; 9999]), true, Data([0; 9999]));
    |               ^^^^^^^^^^^^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: moving 9999 bytes
   --> $DIR/copy_into_fn.rs:17:38
@@ -25,7 +25,7 @@ error: moving 9999 bytes
 LL |     many_args(Data([0; 9999]), true, Data([0; 9999]));
    |                                      ^^^^^^^^^^^^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/lint/large_assignments/inline_mir.stderr b/tests/ui/lint/large_assignments/inline_mir.stderr
index 1a5fcb6c8fc1..b8170d8977d2 100644
--- a/tests/ui/lint/large_assignments/inline_mir.stderr
+++ b/tests/ui/lint/large_assignments/inline_mir.stderr
@@ -4,7 +4,7 @@ error: moving 9999 bytes
 LL |     let cell = std::cell::UnsafeCell::new(data);
    |                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/inline_mir.rs:2:9
    |
@@ -17,7 +17,7 @@ error: moving 9999 bytes
 LL |     std::hint::black_box(cell);
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/lint/large_assignments/large_future.attribute.stderr b/tests/ui/lint/large_assignments/large_future.attribute.stderr
index 734b7ff7ba22..1580c31df3c2 100644
--- a/tests/ui/lint/large_assignments/large_future.attribute.stderr
+++ b/tests/ui/lint/large_assignments/large_future.attribute.stderr
@@ -4,7 +4,7 @@ error: moving 10024 bytes
 LL |     let z = (x, 42);
    |              ^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/large_future.rs:1:9
    |
@@ -17,7 +17,7 @@ error: moving 10024 bytes
 LL |     let a = z.0;
    |             ^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/lint/large_assignments/large_future.option.stderr b/tests/ui/lint/large_assignments/large_future.option.stderr
index 734b7ff7ba22..1580c31df3c2 100644
--- a/tests/ui/lint/large_assignments/large_future.option.stderr
+++ b/tests/ui/lint/large_assignments/large_future.option.stderr
@@ -4,7 +4,7 @@ error: moving 10024 bytes
 LL |     let z = (x, 42);
    |              ^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/large_future.rs:1:9
    |
@@ -17,7 +17,7 @@ error: moving 10024 bytes
 LL |     let a = z.0;
    |             ^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/lint/large_assignments/move_into_box_rc_arc.stderr b/tests/ui/lint/large_assignments/move_into_box_rc_arc.stderr
index a386de5e5e8e..35f30a79ad99 100644
--- a/tests/ui/lint/large_assignments/move_into_box_rc_arc.stderr
+++ b/tests/ui/lint/large_assignments/move_into_box_rc_arc.stderr
@@ -4,7 +4,7 @@ error: moving 9999 bytes
 LL |             data,
    |             ^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/move_into_box_rc_arc.rs:1:9
    |
diff --git a/tests/ui/lint/large_assignments/move_into_fn.stderr b/tests/ui/lint/large_assignments/move_into_fn.stderr
index 19ec6a51d2e7..4f4c710cacef 100644
--- a/tests/ui/lint/large_assignments/move_into_fn.stderr
+++ b/tests/ui/lint/large_assignments/move_into_fn.stderr
@@ -4,7 +4,7 @@ error: moving 9999 bytes
 LL |     let data = Data([100; 9999]);
    |                ^^^^^^^^^^^^^^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 note: the lint level is defined here
   --> $DIR/move_into_fn.rs:5:9
    |
@@ -17,7 +17,7 @@ error: moving 9999 bytes
 LL |     take_data(data);
    |               ^^^^ value moved from here
    |
-   = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
+   = note: the current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/lint/unused/concat-in-crate-name-issue-137687.stderr b/tests/ui/lint/unused/concat-in-crate-name-issue-137687.stderr
index b06e65af7bc7..5928eb6c58c4 100644
--- a/tests/ui/lint/unused/concat-in-crate-name-issue-137687.stderr
+++ b/tests/ui/lint/unused/concat-in-crate-name-issue-137687.stderr
@@ -17,7 +17,7 @@ error: crate-level attribute should be an inner attribute: add an exclamation ma
 LL | #[crate_name = concat !()]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this macro def
+note: this attribute does not have an `!`, which means it is applied to this macro def
   --> $DIR/concat-in-crate-name-issue-137687.rs:5:1
    |
 LL | / macro_rules! a {
diff --git a/tests/ui/lint/unused/unused-attr-macro-rules.stderr b/tests/ui/lint/unused/unused-attr-macro-rules.stderr
index e251ec65622e..75e86d3c014f 100644
--- a/tests/ui/lint/unused/unused-attr-macro-rules.stderr
+++ b/tests/ui/lint/unused/unused-attr-macro-rules.stderr
@@ -27,7 +27,7 @@ error: crate-level attribute should be an inner attribute: add an exclamation ma
 LL | #[recursion_limit="1"]
    | ^^^^^^^^^^^^^^^^^^^^^^
    |
-note: This attribute does not have an `!`, which means it is applied to this macro def
+note: this attribute does not have an `!`, which means it is applied to this macro def
   --> $DIR/unused-attr-macro-rules.rs:12:1
    |
 LL | / macro_rules! foo {
diff --git a/tests/ui/statics/check-immutable-mut-slices.stderr b/tests/ui/statics/check-immutable-mut-slices.stderr
index a9486fc9d781..1e6dfb78c93d 100644
--- a/tests/ui/statics/check-immutable-mut-slices.stderr
+++ b/tests/ui/statics/check-immutable-mut-slices.stderr
@@ -4,9 +4,9 @@ error[E0764]: mutable borrows of temporaries that have their lifetime extended u
 LL | static TEST: &'static mut [isize] = &mut [];
    |                                     ^^^^^^^ this mutable borrow refers to such a temporary
    |
-   = note: Temporaries in constants and statics can have their lifetime extended until the end of the program
-   = note: To avoid accidentally creating global mutable state, such temporaries must be immutable
-   = help: If you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
+   = note: temporaries in constants and statics can have their lifetime extended until the end of the program
+   = note: to avoid accidentally creating global mutable state, such temporaries must be immutable
+   = help: if you really want global mutable state, try replacing the temporary by an interior mutable `static` or a `static mut`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/unstable-feature-bound/unstable_feature_bound_incompatible_stability.stderr b/tests/ui/unstable-feature-bound/unstable_feature_bound_incompatible_stability.stderr
index 9f07e63e4544..e144b981f3cd 100644
--- a/tests/ui/unstable-feature-bound/unstable_feature_bound_incompatible_stability.stderr
+++ b/tests/ui/unstable-feature-bound/unstable_feature_bound_incompatible_stability.stderr
@@ -4,7 +4,7 @@ error: item annotated with `#[unstable_feature_bound]` should not be stable
 LL | fn bar() {}
    | ^^^^^^^^^^^
    |
-   = help: If this item is meant to be stable, do not use any functions annotated with `#[unstable_feature_bound]`. Otherwise, mark this item as unstable with `#[unstable]`
+   = help: if this item is meant to be stable, do not use any functions annotated with `#[unstable_feature_bound]`. Otherwise, mark this item as unstable with `#[unstable]`
 
 error: aborting due to 1 previous error
 

From 08432c892758a06a6bab9fa0584effb7e7881303 Mon Sep 17 00:00:00 2001
From: Andreas Liljeqvist 
Date: Sun, 18 Jan 2026 22:49:37 +0100
Subject: [PATCH 0363/1843] Optimize small input path for is_ascii on x86_64

For inputs smaller than 32 bytes, use usize-at-a-time processing
instead of calling the SSE2 function. This avoids function call
overhead from #[target_feature(enable = "sse2")] which prevents
inlining.

Also moves CHUNK_SIZE to module level so it can be shared between
is_ascii and is_ascii_sse2.
---
 library/core/src/slice/ascii.rs | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs
index c9e168d6cbf8..25b8a10af355 100644
--- a/library/core/src/slice/ascii.rs
+++ b/library/core/src/slice/ascii.rs
@@ -460,6 +460,10 @@ const fn is_ascii(s: &[u8]) -> bool {
     )
 }
 
+/// Chunk size for vectorized ASCII checking (two 16-byte SSE registers).
+#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
+const CHUNK_SIZE: usize = 32;
+
 /// SSE2 implementation using `_mm_movemask_epi8` (compiles to `pmovmskb`) to
 /// avoid LLVM's broken AVX-512 auto-vectorization of counting loops.
 ///
@@ -470,8 +474,6 @@ const fn is_ascii(s: &[u8]) -> bool {
 unsafe fn is_ascii_sse2(bytes: &[u8]) -> bool {
     use crate::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128};
 
-    const CHUNK_SIZE: usize = 32;
-
     let mut i = 0;
 
     while i + CHUNK_SIZE <= bytes.len() {
@@ -518,11 +520,27 @@ unsafe fn is_ascii_sse2(bytes: &[u8]) -> bool {
 #[inline]
 #[rustc_allow_const_fn_unstable(const_eval_select)]
 const fn is_ascii(bytes: &[u8]) -> bool {
+    const USIZE_SIZE: usize = size_of::();
+    const NONASCII_MASK: usize = usize::MAX / 255 * 0x80;
+
     const_eval_select!(
         @capture { bytes: &[u8] } -> bool:
         if const {
             is_ascii_simple(bytes)
         } else {
+            // For small inputs, use usize-at-a-time processing to avoid SSE2 call overhead.
+            if bytes.len() < CHUNK_SIZE {
+                let chunks = bytes.chunks_exact(USIZE_SIZE);
+                let remainder = chunks.remainder();
+                for chunk in chunks {
+                    let word = usize::from_ne_bytes(chunk.try_into().unwrap());
+                    if (word & NONASCII_MASK) != 0 {
+                        return false;
+                    }
+                }
+                return remainder.iter().all(|b| b.is_ascii());
+            }
+
             // SAFETY: SSE2 is guaranteed available on x86_64
             unsafe { is_ascii_sse2(bytes) }
         }

From 3a8b57715f1e762fbb78edc89c767446b7fdc0ea Mon Sep 17 00:00:00 2001
From: KaiTomotake 
Date: Sun, 18 Jan 2026 22:43:35 +0900
Subject: [PATCH 0364/1843] add lint test

Co-authored-by: Redddy 
---
 .../unused_assignments_across_match_guards.rs | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 tests/ui/lint/unused/unused_assignments_across_match_guards.rs

diff --git a/tests/ui/lint/unused/unused_assignments_across_match_guards.rs b/tests/ui/lint/unused/unused_assignments_across_match_guards.rs
new file mode 100644
index 000000000000..666a529b8f85
--- /dev/null
+++ b/tests/ui/lint/unused/unused_assignments_across_match_guards.rs
@@ -0,0 +1,19 @@
+// Regression test for 
+// This test ensures that unused_assignments does not report assignments used in a match.
+//@ check-pass
+
+fn pnk(x: usize) -> &'static str {
+    let mut k1 = "k1";
+    let mut h1 = "h1";
+    match x & 3 {
+        3 if { k1 = "unused?"; false } => (),
+        _ if { h1 = k1; true } => (),
+        _ => (),
+    }
+    h1
+}
+
+#[deny(unused_assignments)]
+fn main() {
+    pnk(3);
+}

From 3327a92b4353a03c60dbb2c58564be754e93db22 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Wed, 5 Nov 2025 20:40:15 +1100
Subject: [PATCH 0365/1843] Add some tests for trimmed paths in diagnostics

---
 .../auxiliary/doc_hidden_helper.rs            | 18 +++++
 tests/ui/trimmed-paths/core-unicode.rs        | 19 ++++++
 tests/ui/trimmed-paths/core-unicode.stderr    | 14 ++++
 tests/ui/trimmed-paths/doc-hidden.rs          | 68 +++++++++++++++++++
 tests/ui/trimmed-paths/doc-hidden.stderr      | 39 +++++++++++
 5 files changed, 158 insertions(+)
 create mode 100644 tests/ui/trimmed-paths/auxiliary/doc_hidden_helper.rs
 create mode 100644 tests/ui/trimmed-paths/core-unicode.rs
 create mode 100644 tests/ui/trimmed-paths/core-unicode.stderr
 create mode 100644 tests/ui/trimmed-paths/doc-hidden.rs
 create mode 100644 tests/ui/trimmed-paths/doc-hidden.stderr

diff --git a/tests/ui/trimmed-paths/auxiliary/doc_hidden_helper.rs b/tests/ui/trimmed-paths/auxiliary/doc_hidden_helper.rs
new file mode 100644
index 000000000000..2e5e1591606e
--- /dev/null
+++ b/tests/ui/trimmed-paths/auxiliary/doc_hidden_helper.rs
@@ -0,0 +1,18 @@
+//@ edition: 2024
+
+pub struct ActuallyPub {}
+#[doc(hidden)]
+pub struct DocHidden {}
+
+pub mod pub_mod {
+    pub struct ActuallyPubInPubMod {}
+    #[doc(hidden)]
+    pub struct DocHiddenInPubMod {}
+}
+
+#[doc(hidden)]
+pub mod hidden_mod {
+    pub struct ActuallyPubInHiddenMod {}
+    #[doc(hidden)]
+    pub struct DocHiddenInHiddenMod {}
+}
diff --git a/tests/ui/trimmed-paths/core-unicode.rs b/tests/ui/trimmed-paths/core-unicode.rs
new file mode 100644
index 000000000000..54bde92a5335
--- /dev/null
+++ b/tests/ui/trimmed-paths/core-unicode.rs
@@ -0,0 +1,19 @@
+//@ edition: 2024
+
+// Test that the `#[doc(hidden)]` module `core::unicode` module does not
+// disqualify another item named `unicode` from path trimming in diagnostics.
+
+use core::marker::PhantomData;
+
+mod inner {
+    #[expect(non_camel_case_types)]
+    pub(crate) enum unicode {}
+}
+
+fn main() {
+    let PhantomData::<(inner::unicode, u32)> = PhantomData::<(u32, inner::unicode)>;
+    //~^ ERROR mismatched types [E0308]
+    //~| NOTE expected `PhantomData<(u32, unicode)>`, found `PhantomData<(unicode, u32)>`
+    //~| NOTE this expression has type `PhantomData<(u32, inner::unicode)>`
+    //~| NOTE expected struct `PhantomData<(u32, inner::unicode)>`
+}
diff --git a/tests/ui/trimmed-paths/core-unicode.stderr b/tests/ui/trimmed-paths/core-unicode.stderr
new file mode 100644
index 000000000000..9023200d1c92
--- /dev/null
+++ b/tests/ui/trimmed-paths/core-unicode.stderr
@@ -0,0 +1,14 @@
+error[E0308]: mismatched types
+  --> $DIR/core-unicode.rs:14:9
+   |
+LL |     let PhantomData::<(inner::unicode, u32)> = PhantomData::<(u32, inner::unicode)>;
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^   ------------------------------------ this expression has type `PhantomData<(u32, inner::unicode)>`
+   |         |
+   |         expected `PhantomData<(u32, unicode)>`, found `PhantomData<(unicode, u32)>`
+   |
+   = note: expected struct `PhantomData<(u32, inner::unicode)>`
+              found struct `PhantomData<(inner::unicode, u32)>`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0308`.
diff --git a/tests/ui/trimmed-paths/doc-hidden.rs b/tests/ui/trimmed-paths/doc-hidden.rs
new file mode 100644
index 000000000000..b73d04c3c6ae
--- /dev/null
+++ b/tests/ui/trimmed-paths/doc-hidden.rs
@@ -0,0 +1,68 @@
+//@ edition: 2024
+//@ aux-crate: helper=doc_hidden_helper.rs
+
+// Test that `#[doc(hidden)]` items in other crates do not disqualify another
+// item with the same name from path trimming in diagnostics.
+
+// Declare several modules and types whose short names match those in the aux crate.
+//
+// Of these, only `ActuallyPub` and `ActuallyPubInPubMod` should be disqualified
+// from path trimming, because the other names only collide with `#[doc(hidden)]`
+// names.
+mod local {
+    pub(crate) struct ActuallyPub {}
+    pub(crate) struct DocHidden {}
+
+    pub(crate) mod pub_mod {
+        pub(crate) struct ActuallyPubInPubMod {}
+        pub(crate) struct DocHiddenInPubMod {}
+    }
+
+    pub(crate) mod hidden_mod {
+        pub(crate) struct ActuallyPubInHiddenMod {}
+        pub(crate) struct DocHiddenInHiddenMod {}
+    }
+}
+
+fn main() {
+    uses_local();
+    uses_helper();
+}
+
+fn uses_local() {
+    use local::{ActuallyPub, DocHidden};
+    use local::pub_mod::{ActuallyPubInPubMod, DocHiddenInPubMod};
+    use local::hidden_mod::{ActuallyPubInHiddenMod, DocHiddenInHiddenMod};
+
+    let _: (
+        //~^ NOTE expected due to this
+        ActuallyPub,
+        DocHidden,
+        ActuallyPubInPubMod,
+        DocHiddenInPubMod,
+        ActuallyPubInHiddenMod,
+        DocHiddenInHiddenMod,
+    ) = 3u32;
+    //~^ ERROR mismatched types [E0308]
+    //~| NOTE expected `(ActuallyPub, ..., ..., ..., ..., ...)`, found `u32`
+    //~| NOTE expected tuple `(local::ActuallyPub, local::DocHidden, local::pub_mod::ActuallyPubInPubMod, local::pub_mod::DocHiddenInPubMod, local::hidden_mod::ActuallyPubInHiddenMod, local::hidden_mod::DocHiddenInHiddenMod)`
+}
+
+fn uses_helper() {
+    use helper::{ActuallyPub, DocHidden};
+    use helper::pub_mod::{ActuallyPubInPubMod, DocHiddenInPubMod};
+    use helper::hidden_mod::{ActuallyPubInHiddenMod, DocHiddenInHiddenMod};
+
+    let _: (
+        //~^ NOTE expected due to this
+        ActuallyPub,
+        DocHidden,
+        ActuallyPubInPubMod,
+        DocHiddenInPubMod,
+        ActuallyPubInHiddenMod,
+        DocHiddenInHiddenMod,
+    ) = 3u32;
+    //~^ ERROR mismatched types [E0308]
+    //~| NOTE expected `(ActuallyPub, ..., ..., ..., ..., ...)`, found `u32`
+    //~| NOTE expected tuple `(doc_hidden_helper::ActuallyPub, doc_hidden_helper::DocHidden, doc_hidden_helper::pub_mod::ActuallyPubInPubMod, doc_hidden_helper::pub_mod::DocHiddenInPubMod, doc_hidden_helper::hidden_mod::ActuallyPubInHiddenMod, doc_hidden_helper::hidden_mod::DocHiddenInHiddenMod)`
+}
diff --git a/tests/ui/trimmed-paths/doc-hidden.stderr b/tests/ui/trimmed-paths/doc-hidden.stderr
new file mode 100644
index 000000000000..6f6f8f21fbe0
--- /dev/null
+++ b/tests/ui/trimmed-paths/doc-hidden.stderr
@@ -0,0 +1,39 @@
+error[E0308]: mismatched types
+  --> $DIR/doc-hidden.rs:45:9
+   |
+LL |       let _: (
+   |  ____________-
+LL | |
+LL | |         ActuallyPub,
+LL | |         DocHidden,
+...  |
+LL | |         DocHiddenInHiddenMod,
+LL | |     ) = 3u32;
+   | |     -   ^^^^ expected `(ActuallyPub, ..., ..., ..., ..., ...)`, found `u32`
+   | |_____|
+   |       expected due to this
+   |
+   = note: expected tuple `(local::ActuallyPub, local::DocHidden, local::pub_mod::ActuallyPubInPubMod, local::pub_mod::DocHiddenInPubMod, local::hidden_mod::ActuallyPubInHiddenMod, local::hidden_mod::DocHiddenInHiddenMod)`
+               found type `u32`
+
+error[E0308]: mismatched types
+  --> $DIR/doc-hidden.rs:64:9
+   |
+LL |       let _: (
+   |  ____________-
+LL | |
+LL | |         ActuallyPub,
+LL | |         DocHidden,
+...  |
+LL | |         DocHiddenInHiddenMod,
+LL | |     ) = 3u32;
+   | |     -   ^^^^ expected `(ActuallyPub, ..., ..., ..., ..., ...)`, found `u32`
+   | |_____|
+   |       expected due to this
+   |
+   = note: expected tuple `(doc_hidden_helper::ActuallyPub, doc_hidden_helper::DocHidden, doc_hidden_helper::pub_mod::ActuallyPubInPubMod, doc_hidden_helper::pub_mod::DocHiddenInPubMod, doc_hidden_helper::hidden_mod::ActuallyPubInHiddenMod, doc_hidden_helper::hidden_mod::DocHiddenInHiddenMod)`
+               found type `u32`
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0308`.

From 2df2c72d7ae36d11313ed730960030cc5af9fb21 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Wed, 5 Nov 2025 18:56:10 +1100
Subject: [PATCH 0366/1843] Ignore `#[doc(hidden)]` items when computing
 trimmed paths

---
 compiler/rustc_middle/src/ty/print/pretty.rs  |  7 ++++++
 ...oo-{closure#0}-{closure#0}.built.after.mir |  2 +-
 ...-{closure#0}-{synthetic#0}.built.after.mir |  2 +-
 ...0}-{closure#0}-{closure#0}.built.after.mir |  2 +-
 ...-{closure#0}-{synthetic#0}.built.after.mir |  2 +-
 ...0}-{closure#1}-{closure#0}.built.after.mir |  2 +-
 ...-{closure#1}-{synthetic#0}.built.after.mir |  2 +-
 tests/mir-opt/box_expr.rs                     |  2 +-
 .../issue_101867.main.built.after.mir         |  2 +-
 .../boxes.main.GVN.panic-abort.diff           |  4 ++--
 .../boxes.main.GVN.panic-unwind.diff          |  4 ++--
 ..._simplification.hello.GVN.panic-abort.diff |  2 +-
 ...simplification.hello.GVN.panic-unwind.diff |  2 +-
 .../transmute.unreachable_box.GVN.32bit.diff  |  2 +-
 .../transmute.unreachable_box.GVN.64bit.diff  |  2 +-
 ...n.DataflowConstProp.32bit.panic-abort.diff | 10 ++++-----
 ....DataflowConstProp.32bit.panic-unwind.diff | 10 ++++-----
 ...n.DataflowConstProp.64bit.panic-abort.diff | 10 ++++-----
 ....DataflowConstProp.64bit.panic-unwind.diff | 10 ++++-----
 ...oxed_slice.main.GVN.32bit.panic-abort.diff | 12 +++++-----
 ...xed_slice.main.GVN.32bit.panic-unwind.diff | 12 +++++-----
 ...oxed_slice.main.GVN.64bit.panic-abort.diff | 12 +++++-----
 ...xed_slice.main.GVN.64bit.panic-unwind.diff | 12 +++++-----
 ...reachable_box.DataflowConstProp.32bit.diff |  2 +-
 ...reachable_box.DataflowConstProp.64bit.diff |  2 +-
 ...ng_operand.test.GVN.32bit.panic-abort.diff |  2 +-
 ...ng_operand.test.GVN.64bit.panic-abort.diff |  2 +-
 ...onential_common.GVN.32bit.panic-abort.diff |  4 ++--
 ...nential_common.GVN.32bit.panic-unwind.diff |  4 ++--
 ...onential_common.GVN.64bit.panic-abort.diff |  4 ++--
 ...nential_common.GVN.64bit.panic-unwind.diff |  4 ++--
 tests/mir-opt/gvn.slices.GVN.panic-abort.diff |  8 +++----
 .../mir-opt/gvn.slices.GVN.panic-unwind.diff  |  8 +++----
 .../gvn.wrap_unwrap.GVN.panic-abort.diff      |  2 +-
 .../gvn.wrap_unwrap.GVN.panic-unwind.diff     |  2 +-
 ...inline_diverging.g.Inline.panic-abort.diff |  2 +-
 ...nline_diverging.g.Inline.panic-unwind.diff |  2 +-
 .../inline_shims.drop.Inline.panic-abort.diff |  4 ++--
 ..._conditions.JumpThreading.panic-abort.diff |  8 +++----
 ...conditions.JumpThreading.panic-unwind.diff |  8 +++----
 ...fg-pre-optimizations.after.panic-abort.mir |  2 +-
 ...g-pre-optimizations.after.panic-unwind.mir |  2 +-
 ...ace.PreCodegen.after.32bit.panic-abort.mir |  8 +++----
 ...ce.PreCodegen.after.32bit.panic-unwind.mir |  8 +++----
 ...ace.PreCodegen.after.64bit.panic-abort.mir |  8 +++----
 ...ce.PreCodegen.after.64bit.panic-unwind.mir |  8 +++----
 .../loops.vec_move.PreCodegen.after.mir       |  4 ++--
 ..._to_slice.PreCodegen.after.panic-abort.mir |  4 ++--
 ...to_slice.PreCodegen.after.panic-unwind.mir |  4 ++--
 ...mes.foo.ScalarReplacementOfAggregates.diff |  2 +-
 .../async-closures/def-path.stderr            |  4 ++--
 .../unsizing-wfcheck-issue-126272.stderr      |  2 +-
 tests/ui/consts/const-eval/format.rs          |  2 +-
 tests/ui/consts/const-eval/format.stderr      |  2 +-
 .../contract-captures-via-closure-noncopy.rs  |  2 +-
 ...ntract-captures-via-closure-noncopy.stderr |  4 ++--
 ...derives-span-Eq-enum-struct-variant.stderr |  2 +-
 tests/ui/derives/derives-span-Eq-enum.stderr  |  2 +-
 .../ui/derives/derives-span-Eq-struct.stderr  |  2 +-
 .../derives-span-Eq-tuple-struct.stderr       |  2 +-
 tests/ui/deriving/issue-103157.stderr         |  2 +-
 tests/ui/issues/issue-27340.stderr            |  2 +-
 tests/ui/kindck/kindck-send-object.stderr     |  2 +-
 tests/ui/kindck/kindck-send-object1.stderr    |  2 +-
 tests/ui/kindck/kindck-send-object2.stderr    |  2 +-
 tests/ui/kindck/kindck-send-owned.stderr      |  2 +-
 .../type-length-limit-enforcement.stderr      |  4 +++-
 tests/ui/proc-macro/bad-projection.stderr     |  2 +-
 tests/ui/proc-macro/proc-macro-abi.stderr     | 12 +++++-----
 ...does-not-have-iter-interpolated-dup.stderr |  2 +-
 .../does-not-have-iter-interpolated.stderr    |  2 +-
 tests/ui/proc-macro/quote/not-quotable.stderr |  2 +-
 .../signature-proc-macro-attribute.stderr     | 16 +++++++-------
 .../signature-proc-macro-derive.stderr        | 22 +++++++++----------
 .../ui/proc-macro/signature-proc-macro.stderr | 22 +++++++++----------
 tests/ui/proc-macro/signature.stderr          |  2 +-
 ...lobal-variable-promotion-error-7364.stderr |  2 +-
 tests/ui/traits/const-traits/issue-79450.rs   |  2 +-
 .../ui/traits/const-traits/issue-79450.stderr |  2 +-
 tests/ui/traits/cycle-cache-err-60010.stderr  |  4 ++--
 .../negated-auto-traits-error.stderr          |  8 +++----
 tests/ui/trimmed-paths/core-unicode.rs        |  4 ++--
 tests/ui/trimmed-paths/core-unicode.stderr    |  6 ++---
 tests/ui/trimmed-paths/doc-hidden.rs          |  2 +-
 tests/ui/trimmed-paths/doc-hidden.stderr      |  2 +-
 .../panic-with-unspecified-type.stderr        |  2 +-
 .../ui/type/pattern_types/derives_fail.stderr |  2 +-
 tests/ui/union/union-derive-clone.stderr      |  2 +-
 tests/ui/union/union-derive-eq.current.stderr |  2 +-
 tests/ui/union/union-derive-eq.next.stderr    |  2 +-
 90 files changed, 209 insertions(+), 200 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 2a65517de403..fd0a5ca309a4 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -3421,6 +3421,13 @@ fn for_each_def(tcx: TyCtxt<'_>, mut collect_fn: impl for<'b> FnMut(&'b Ident, N
                 def::Res::Def(DefKind::AssocTy, _) => {}
                 def::Res::Def(DefKind::TyAlias, _) => {}
                 def::Res::Def(defkind, def_id) => {
+                    // Ignore external `#[doc(hidden)]` items and their descendants.
+                    // They shouldn't prevent other items from being considered
+                    // unique, and should be printed with a full path if necessary.
+                    if tcx.is_doc_hidden(def_id) {
+                        continue;
+                    }
+
                     if let Some(ns) = defkind.ns() {
                         collect_fn(&child.ident, ns, def_id);
                     }
diff --git a/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{closure#0}.built.after.mir b/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{closure#0}.built.after.mir
index 9ff1a90ab820..4c9ca11f8283 100644
--- a/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{closure#0}.built.after.mir
+++ b/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{closure#0}.built.after.mir
@@ -1,6 +1,6 @@
 // MIR for `foo::{closure#0}::{closure#0}` after built
 
-fn foo::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_fake_read_for_by_move.rs:12:27: 15:6}, _2: ResumeTy) -> ()
+fn foo::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_fake_read_for_by_move.rs:12:27: 15:6}, _2: std::future::ResumeTy) -> ()
 yields ()
  {
     debug _task_context => _2;
diff --git a/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{synthetic#0}.built.after.mir b/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{synthetic#0}.built.after.mir
index 4b745caf48c5..e80fdea7051d 100644
--- a/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{synthetic#0}.built.after.mir
+++ b/tests/mir-opt/async_closure_fake_read_for_by_move.foo-{closure#0}-{synthetic#0}.built.after.mir
@@ -1,6 +1,6 @@
 // MIR for `foo::{closure#0}::{synthetic#0}` after built
 
-fn foo::{closure#0}::{synthetic#0}(_1: {async closure body@$DIR/async_closure_fake_read_for_by_move.rs:12:27: 15:6}, _2: ResumeTy) -> ()
+fn foo::{closure#0}::{synthetic#0}(_1: {async closure body@$DIR/async_closure_fake_read_for_by_move.rs:12:27: 15:6}, _2: std::future::ResumeTy) -> ()
 yields ()
  {
     debug _task_context => _2;
diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.built.after.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.built.after.mir
index 4d484b16b507..075065b4c090 100644
--- a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.built.after.mir
+++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.built.after.mir
@@ -1,6 +1,6 @@
 // MIR for `main::{closure#0}::{closure#0}::{closure#0}` after built
 
-fn main::{closure#0}::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:53:53: 56:10}, _2: ResumeTy) -> ()
+fn main::{closure#0}::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:53:53: 56:10}, _2: std::future::ResumeTy) -> ()
 yields ()
  {
     debug _task_context => _2;
diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{synthetic#0}.built.after.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{synthetic#0}.built.after.mir
index ace780f773e8..0f4e5f3cb02f 100644
--- a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{synthetic#0}.built.after.mir
+++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{synthetic#0}.built.after.mir
@@ -1,6 +1,6 @@
 // MIR for `main::{closure#0}::{closure#0}::{synthetic#0}` after built
 
-fn main::{closure#0}::{closure#0}::{synthetic#0}(_1: {async closure body@$DIR/async_closure_shims.rs:53:53: 56:10}, _2: ResumeTy) -> ()
+fn main::{closure#0}::{closure#0}::{synthetic#0}(_1: {async closure body@$DIR/async_closure_shims.rs:53:53: 56:10}, _2: std::future::ResumeTy) -> ()
 yields ()
  {
     debug _task_context => _2;
diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{closure#0}.built.after.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{closure#0}.built.after.mir
index f50ad689f447..18f4e741384f 100644
--- a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{closure#0}.built.after.mir
+++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{closure#0}.built.after.mir
@@ -1,6 +1,6 @@
 // MIR for `main::{closure#0}::{closure#1}::{closure#0}` after built
 
-fn main::{closure#0}::{closure#1}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:62:48: 65:10}, _2: ResumeTy) -> ()
+fn main::{closure#0}::{closure#1}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:62:48: 65:10}, _2: std::future::ResumeTy) -> ()
 yields ()
  {
     debug _task_context => _2;
diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{synthetic#0}.built.after.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{synthetic#0}.built.after.mir
index 62d8adeedcb6..257586c4a080 100644
--- a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{synthetic#0}.built.after.mir
+++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#1}-{synthetic#0}.built.after.mir
@@ -1,6 +1,6 @@
 // MIR for `main::{closure#0}::{closure#1}::{synthetic#0}` after built
 
-fn main::{closure#0}::{closure#1}::{synthetic#0}(_1: {async closure body@$DIR/async_closure_shims.rs:62:48: 65:10}, _2: ResumeTy) -> ()
+fn main::{closure#0}::{closure#1}::{synthetic#0}(_1: {async closure body@$DIR/async_closure_shims.rs:62:48: 65:10}, _2: std::future::ResumeTy) -> ()
 yields ()
  {
     debug _task_context => _2;
diff --git a/tests/mir-opt/box_expr.rs b/tests/mir-opt/box_expr.rs
index 6299c9871809..dbb07a028a93 100644
--- a/tests/mir-opt/box_expr.rs
+++ b/tests/mir-opt/box_expr.rs
@@ -8,7 +8,7 @@ fn main() {
     // CHECK-LABEL: fn main(
     // CHECK:   [[ptr:_.*]] = move {{_.*}} as *const S (Transmute);
     // CHECK:   [[nonnull:_.*]] = NonNull:: { pointer: move [[ptr]] };
-    // CHECK:   [[unique:_.*]] = Unique:: { pointer: move [[nonnull]], _marker: const PhantomData:: };
+    // CHECK:   [[unique:_.*]] = std::ptr::Unique:: { pointer: move [[nonnull]], _marker: const PhantomData:: };
     // CHECK:   [[box:_.*]] = Box::(move [[unique]], const std::alloc::Global);
     // CHECK:   [[ptr:_.*]] = copy (([[box]].0: std::ptr::Unique).0: std::ptr::NonNull) as *const S (Transmute);
     // CHECK:   (*[[ptr]]) = S::new() -> [return: [[ret:bb.*]], unwind: [[unwind:bb.*]]];
diff --git a/tests/mir-opt/building/issue_101867.main.built.after.mir b/tests/mir-opt/building/issue_101867.main.built.after.mir
index 83281dea44db..cef4325b9a4d 100644
--- a/tests/mir-opt/building/issue_101867.main.built.after.mir
+++ b/tests/mir-opt/building/issue_101867.main.built.after.mir
@@ -32,7 +32,7 @@ fn main() -> () {
     bb1: {
         StorageLive(_3);
         StorageLive(_4);
-        _4 = begin_panic::<&str>(const "explicit panic") -> bb8;
+        _4 = std::rt::begin_panic::<&str>(const "explicit panic") -> bb8;
     }
 
     bb2: {
diff --git a/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff b/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff
index 95eaf18b4703..ecc4b35ebcb6 100644
--- a/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff
+++ b/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff
@@ -29,10 +29,10 @@
           StorageLive(_5);
 -         _6 = move _4 as *const i32 (Transmute);
 -         _7 = NonNull:: { pointer: move _6 };
--         _8 = Unique:: { pointer: move _7, _marker: const PhantomData:: };
+-         _8 = std::ptr::Unique:: { pointer: move _7, _marker: const PhantomData:: };
 +         _6 = copy _4 as *const i32 (PtrToPtr);
 +         _7 = NonNull:: { pointer: copy _6 };
-+         _8 = Unique:: { pointer: copy _7, _marker: const PhantomData:: };
++         _8 = std::ptr::Unique:: { pointer: copy _7, _marker: const PhantomData:: };
           _5 = Box::(move _8, const std::alloc::Global);
 -         _9 = copy ((_5.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
 -         (*_9) = const 42_i32;
diff --git a/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff
index 6d8d3a0dcfe2..aba1a4f1df47 100644
--- a/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff
@@ -29,10 +29,10 @@
           StorageLive(_5);
 -         _6 = move _4 as *const i32 (Transmute);
 -         _7 = NonNull:: { pointer: move _6 };
--         _8 = Unique:: { pointer: move _7, _marker: const PhantomData:: };
+-         _8 = std::ptr::Unique:: { pointer: move _7, _marker: const PhantomData:: };
 +         _6 = copy _4 as *const i32 (PtrToPtr);
 +         _7 = NonNull:: { pointer: copy _6 };
-+         _8 = Unique:: { pointer: copy _7, _marker: const PhantomData:: };
++         _8 = std::ptr::Unique:: { pointer: copy _7, _marker: const PhantomData:: };
           _5 = Box::(move _8, const std::alloc::Global);
 -         _9 = copy ((_5.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
 -         (*_9) = const 42_i32;
diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff
index 5df2232053fe..2ecf41638125 100644
--- a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff
+++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff
@@ -10,7 +10,7 @@
       }
   
       bb1: {
-          _1 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
+          _1 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
       }
   
       bb2: {
diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff
index 788a4424943e..06287b670dd4 100644
--- a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff
@@ -10,7 +10,7 @@
       }
   
       bb1: {
-          _1 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
+          _1 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind continue;
       }
   
       bb2: {
diff --git a/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.32bit.diff b/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.32bit.diff
index b698d8f37357..bd24af602c88 100644
--- a/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.32bit.diff
+++ b/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.32bit.diff
@@ -13,7 +13,7 @@
           StorageLive(_1);
 -         _1 = const 1_usize as std::boxed::Box (Transmute);
 -         _2 = copy ((_1.0: std::ptr::Unique).0: std::ptr::NonNull) as *const Never (Transmute);
-+         _1 = const Box::(Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
++         _1 = const Box::(std::ptr::Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
 +         _2 = const std::ptr::NonNull:: {{ pointer: {0x1 as *const Never} }} as *const Never (Transmute);
           unreachable;
       }
diff --git a/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.64bit.diff b/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.64bit.diff
index b698d8f37357..bd24af602c88 100644
--- a/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.64bit.diff
+++ b/tests/mir-opt/const_prop/transmute.unreachable_box.GVN.64bit.diff
@@ -13,7 +13,7 @@
           StorageLive(_1);
 -         _1 = const 1_usize as std::boxed::Box (Transmute);
 -         _2 = copy ((_1.0: std::ptr::Unique).0: std::ptr::NonNull) as *const Never (Transmute);
-+         _1 = const Box::(Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
++         _1 = const Box::(std::ptr::Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
 +         _2 = const std::ptr::NonNull:: {{ pointer: {0x1 as *const Never} }} as *const Never (Transmute);
           unreachable;
       }
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff
index 2c89670dcf7d..7a60070b7074 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -51,13 +51,13 @@
           _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
-          _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+          _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
-          _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
+          _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
-          _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
+          _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
-          _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
+          _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind unreachable];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff
index 8fecfe224cc6..d13d0d962a69 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -51,13 +51,13 @@
           _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
-          _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+          _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
-          _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
+          _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
-          _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
+          _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
-          _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
+          _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind: bb2];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff
index 976ea252c2f8..8701e879e959 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -51,13 +51,13 @@
           _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
-          _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+          _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
-          _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
+          _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
-          _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
+          _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
-          _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
+          _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind unreachable];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff
index 6c59f5e3e2e8..ac1c8d627baa 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -51,13 +51,13 @@
           _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
-          _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+          _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
-          _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
+          _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
-          _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
+          _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
-          _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
+          _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind: bb2];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff
index 1f9cf6d6aca8..0205d0cc3d16 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -54,17 +54,17 @@
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
--         _4 = Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
-+         _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+-         _4 = std::ptr::Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
++         _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
 -         _3 = move _4 as std::ptr::Unique<[bool]> (PointerCoercion(Unsize, Implicit));
-+         _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
++         _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
 -         _2 = Box::<[bool]>(copy _3, const std::alloc::Global);
-+         _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
++         _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
 -         _1 = A { foo: move _2 };
-+         _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
++         _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind unreachable];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff
index a8760285fac1..f6babe35b5a0 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -54,17 +54,17 @@
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
--         _4 = Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
-+         _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+-         _4 = std::ptr::Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
++         _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
 -         _3 = move _4 as std::ptr::Unique<[bool]> (PointerCoercion(Unsize, Implicit));
-+         _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
++         _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
 -         _2 = Box::<[bool]>(copy _3, const std::alloc::Global);
-+         _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
++         _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
 -         _1 = A { foo: move _2 };
-+         _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
++         _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind: bb2];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff
index c398ae70a1a3..204e59415c6b 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -54,17 +54,17 @@
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
--         _4 = Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
-+         _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+-         _4 = std::ptr::Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
++         _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
 -         _3 = move _4 as std::ptr::Unique<[bool]> (PointerCoercion(Unsize, Implicit));
-+         _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
++         _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
 -         _2 = Box::<[bool]>(copy _3, const std::alloc::Global);
-+         _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
++         _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
 -         _1 = A { foo: move _2 };
-+         _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
++         _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind unreachable];
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff
index 02934c02587d..0cf3f43c0464 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff
@@ -13,7 +13,7 @@
           let mut _4: std::ptr::Unique<[bool; 0]>;
           scope 3 {
           }
-          scope 4 (inlined Unique::<[bool; 0]>::dangling) {
+          scope 4 (inlined std::ptr::Unique::<[bool; 0]>::dangling) {
               let mut _5: std::ptr::NonNull<[bool; 0]>;
               scope 5 (inlined NonNull::<[bool; 0]>::dangling) {
                   let mut _6: std::num::NonZero;
@@ -54,17 +54,17 @@
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
           StorageDead(_7);
           StorageDead(_6);
--         _4 = Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
-+         _4 = const Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
+-         _4 = std::ptr::Unique::<[bool; 0]> { pointer: move _5, _marker: const PhantomData::<[bool; 0]> };
++         _4 = const std::ptr::Unique::<[bool; 0]> {{ pointer: NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }}, _marker: PhantomData::<[bool; 0]> }};
           StorageDead(_5);
 -         _3 = move _4 as std::ptr::Unique<[bool]> (PointerCoercion(Unsize, Implicit));
-+         _3 = const Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
++         _3 = const std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }};
           StorageDead(_4);
 -         _2 = Box::<[bool]>(copy _3, const std::alloc::Global);
-+         _2 = const Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
++         _2 = const Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC1, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global);
           StorageDead(_3);
 -         _1 = A { foo: move _2 };
-+         _1 = const A {{ foo: Box::<[bool]>(Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
++         _1 = const A {{ foo: Box::<[bool]>(std::ptr::Unique::<[bool]> {{ pointer: NonNull::<[bool]> {{ pointer: Indirect { alloc_id: ALLOC2, offset: Size(0 bytes) }: *const [bool] }}, _marker: PhantomData::<[bool]> }}, std::alloc::Global) }};
           StorageDead(_2);
           _0 = const ();
           drop(_1) -> [return: bb1, unwind: bb2];
diff --git a/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.32bit.diff b/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.32bit.diff
index fa6c2e29e072..3bc5f8507590 100644
--- a/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.32bit.diff
+++ b/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.32bit.diff
@@ -12,7 +12,7 @@
       bb0: {
           StorageLive(_1);
 -         _1 = const 1_usize as std::boxed::Box (Transmute);
-+         _1 = const Box::(Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
++         _1 = const Box::(std::ptr::Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
           _2 = copy ((_1.0: std::ptr::Unique).0: std::ptr::NonNull) as *const Never (Transmute);
           unreachable;
       }
diff --git a/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.64bit.diff b/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.64bit.diff
index fa6c2e29e072..3bc5f8507590 100644
--- a/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.64bit.diff
+++ b/tests/mir-opt/dataflow-const-prop/transmute.unreachable_box.DataflowConstProp.64bit.diff
@@ -12,7 +12,7 @@
       bb0: {
           StorageLive(_1);
 -         _1 = const 1_usize as std::boxed::Box (Transmute);
-+         _1 = const Box::(Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
++         _1 = const Box::(std::ptr::Unique:: {{ pointer: NonNull:: {{ pointer: {0x1 as *const Never} }}, _marker: PhantomData:: }}, std::alloc::Global);
           _2 = copy ((_1.0: std::ptr::Unique).0: std::ptr::NonNull) as *const Never (Transmute);
           unreachable;
       }
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff
index bcf0ad7c165f..2e428b778504 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff
@@ -133,7 +133,7 @@
 -         _13 = copy _12 as *const () (PtrToPtr);
 +         _13 = copy _25 as *const () (PtrToPtr);
           _14 = NonNull::<()> { pointer: copy _13 };
-          _15 = Unique::<()> { pointer: copy _14, _marker: const PhantomData::<()> };
+          _15 = std::ptr::Unique::<()> { pointer: copy _14, _marker: const PhantomData::<()> };
           _3 = Box::<()>(move _15, const std::alloc::Global);
 -         (*_13) = move _4;
 +         (*_13) = const ();
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff
index 1b75a2bcba8b..4531720ee501 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff
@@ -133,7 +133,7 @@
 -         _13 = copy _12 as *const () (PtrToPtr);
 +         _13 = copy _25 as *const () (PtrToPtr);
           _14 = NonNull::<()> { pointer: copy _13 };
-          _15 = Unique::<()> { pointer: copy _14, _marker: const PhantomData::<()> };
+          _15 = std::ptr::Unique::<()> { pointer: copy _14, _marker: const PhantomData::<()> };
           _3 = Box::<()>(move _15, const std::alloc::Global);
 -         (*_13) = move _4;
 +         (*_13) = const ();
diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff
index 2b77aa380a0f..88c77832a4e1 100644
--- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff
+++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff
@@ -56,8 +56,8 @@
       }
   
       bb1: {
--         _5 = MinusPlus;
-+         _5 = const MinusPlus;
+-         _5 = core::num::flt2dec::Sign::MinusPlus;
++         _5 = const core::num::flt2dec::Sign::MinusPlus;
           goto -> bb3;
       }
   
diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff
index ba6d2f3e155c..8a6e7fd35ccd 100644
--- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff
+++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff
@@ -56,8 +56,8 @@
       }
   
       bb1: {
--         _5 = MinusPlus;
-+         _5 = const MinusPlus;
+-         _5 = core::num::flt2dec::Sign::MinusPlus;
++         _5 = const core::num::flt2dec::Sign::MinusPlus;
           goto -> bb3;
       }
   
diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff
index bf6d9d864d57..ce10f4bb247a 100644
--- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff
+++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff
@@ -56,8 +56,8 @@
       }
   
       bb1: {
--         _5 = MinusPlus;
-+         _5 = const MinusPlus;
+-         _5 = core::num::flt2dec::Sign::MinusPlus;
++         _5 = const core::num::flt2dec::Sign::MinusPlus;
           goto -> bb3;
       }
   
diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff
index 01c87fd5317a..b19f2438d022 100644
--- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff
+++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff
@@ -56,8 +56,8 @@
       }
   
       bb1: {
--         _5 = MinusPlus;
-+         _5 = const MinusPlus;
+-         _5 = core::num::flt2dec::Sign::MinusPlus;
++         _5 = const core::num::flt2dec::Sign::MinusPlus;
           goto -> bb3;
       }
   
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
index b7872fc9952b..247ddc73ec36 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
@@ -210,9 +210,9 @@
           _26 = &(*_27);
           StorageLive(_28);
 -         _28 = Option::>::None;
--         _22 = assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind unreachable;
+-         _22 = core::panicking::assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind unreachable;
 +         _28 = const Option::>::None;
-+         _22 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, const Option::>::None) -> unwind unreachable;
++         _22 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, const Option::>::None) -> unwind unreachable;
       }
   
       bb7: {
@@ -313,9 +313,9 @@
           _52 = &(*_53);
           StorageLive(_54);
 -         _54 = Option::>::None;
--         _48 = assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind unreachable;
+-         _48 = core::panicking::assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind unreachable;
 +         _54 = const Option::>::None;
-+         _48 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, const Option::>::None) -> unwind unreachable;
++         _48 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, const Option::>::None) -> unwind unreachable;
       }
   }
   
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
index 37817b48c199..f15c16f1ce0f 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
@@ -210,9 +210,9 @@
           _26 = &(*_27);
           StorageLive(_28);
 -         _28 = Option::>::None;
--         _22 = assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind continue;
+-         _22 = core::panicking::assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind continue;
 +         _28 = const Option::>::None;
-+         _22 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, const Option::>::None) -> unwind continue;
++         _22 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, const Option::>::None) -> unwind continue;
       }
   
       bb7: {
@@ -313,9 +313,9 @@
           _52 = &(*_53);
           StorageLive(_54);
 -         _54 = Option::>::None;
--         _48 = assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind continue;
+-         _48 = core::panicking::assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind continue;
 +         _54 = const Option::>::None;
-+         _48 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, const Option::>::None) -> unwind continue;
++         _48 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, const Option::>::None) -> unwind continue;
       }
   }
   
diff --git a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
index 3bbfd3a891eb..c3eb5d9092be 100644
--- a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
@@ -32,7 +32,7 @@
   
       bb2: {
           StorageLive(_6);
-          _6 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
+          _6 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
       }
   
       bb3: {
diff --git a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
index 03464f43f81e..ea1878be8cf6 100644
--- a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
@@ -32,7 +32,7 @@
   
       bb2: {
           StorageLive(_6);
-          _6 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
+          _6 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind continue;
       }
   
       bb3: {
diff --git a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff
index 423de59e5754..66b1bc29877a 100644
--- a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff
@@ -36,7 +36,7 @@
           StorageLive(_6);
 -         _6 = panic() -> unwind unreachable;
 +         StorageLive(_7);
-+         _7 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
++         _7 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
       }
 + }
 + 
diff --git a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff
index 3689744dcb04..68dd9530137d 100644
--- a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff
@@ -36,7 +36,7 @@
           StorageLive(_6);
 -         _6 = panic() -> unwind continue;
 +         StorageLive(_7);
-+         _7 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
++         _7 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind continue;
       }
 + }
 + 
diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff
index 9509739413b7..a74309e16e88 100644
--- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff
@@ -20,13 +20,13 @@
 +                     scope 5 (inlined alloc::raw_vec::RawVecInner::ptr::) {
 +                         scope 6 (inlined alloc::raw_vec::RawVecInner::non_null::) {
 +                             let mut _11: std::ptr::NonNull;
-+                             scope 7 (inlined Unique::::cast::) {
++                             scope 7 (inlined std::ptr::Unique::::cast::) {
 +                                 scope 8 (inlined NonNull::::cast::) {
 +                                     scope 9 (inlined NonNull::::as_ptr) {
 +                                     }
 +                                 }
 +                             }
-+                             scope 10 (inlined Unique::::as_non_null_ptr) {
++                             scope 10 (inlined std::ptr::Unique::::as_non_null_ptr) {
 +                             }
 +                         }
 +                         scope 11 (inlined NonNull::::as_ptr) {
diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff
index 3cf28f4b60af..6c9cf0cf623b 100644
--- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff
+++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff
@@ -57,13 +57,13 @@
                                   scope 11 (inlined alloc::raw_vec::RawVecInner::ptr::) {
                                       scope 12 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                                           let mut _34: std::ptr::NonNull;
-                                          scope 13 (inlined Unique::::cast::) {
+                                          scope 13 (inlined std::ptr::Unique::::cast::) {
                                               scope 14 (inlined NonNull::::cast::) {
                                                   scope 15 (inlined NonNull::::as_ptr) {
                                                   }
                                               }
                                           }
-                                          scope 16 (inlined Unique::::as_non_null_ptr) {
+                                          scope 16 (inlined std::ptr::Unique::::as_non_null_ptr) {
                                           }
                                       }
                                       scope 17 (inlined NonNull::::as_ptr) {
@@ -115,13 +115,13 @@
                                   scope 34 (inlined alloc::raw_vec::RawVecInner::ptr::) {
                                       scope 35 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                                           let mut _46: std::ptr::NonNull;
-                                          scope 36 (inlined Unique::::cast::) {
+                                          scope 36 (inlined std::ptr::Unique::::cast::) {
                                               scope 37 (inlined NonNull::::cast::) {
                                                   scope 38 (inlined NonNull::::as_ptr) {
                                                   }
                                               }
                                           }
-                                          scope 39 (inlined Unique::::as_non_null_ptr) {
+                                          scope 39 (inlined std::ptr::Unique::::as_non_null_ptr) {
                                           }
                                       }
                                       scope 40 (inlined NonNull::::as_ptr) {
diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff
index 2f0d83f92792..49cd68577a12 100644
--- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff
+++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff
@@ -57,13 +57,13 @@
                                   scope 11 (inlined alloc::raw_vec::RawVecInner::ptr::) {
                                       scope 12 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                                           let mut _34: std::ptr::NonNull;
-                                          scope 13 (inlined Unique::::cast::) {
+                                          scope 13 (inlined std::ptr::Unique::::cast::) {
                                               scope 14 (inlined NonNull::::cast::) {
                                                   scope 15 (inlined NonNull::::as_ptr) {
                                                   }
                                               }
                                           }
-                                          scope 16 (inlined Unique::::as_non_null_ptr) {
+                                          scope 16 (inlined std::ptr::Unique::::as_non_null_ptr) {
                                           }
                                       }
                                       scope 17 (inlined NonNull::::as_ptr) {
@@ -115,13 +115,13 @@
                                   scope 34 (inlined alloc::raw_vec::RawVecInner::ptr::) {
                                       scope 35 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                                           let mut _46: std::ptr::NonNull;
-                                          scope 36 (inlined Unique::::cast::) {
+                                          scope 36 (inlined std::ptr::Unique::::cast::) {
                                               scope 37 (inlined NonNull::::cast::) {
                                                   scope 38 (inlined NonNull::::as_ptr) {
                                                   }
                                               }
                                           }
-                                          scope 39 (inlined Unique::::as_non_null_ptr) {
+                                          scope 39 (inlined std::ptr::Unique::::as_non_null_ptr) {
                                           }
                                       }
                                       scope 40 (inlined NonNull::::as_ptr) {
diff --git a/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-abort.mir b/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-abort.mir
index 6e5f6dc9ea89..7933a36c92c2 100644
--- a/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-abort.mir
+++ b/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-abort.mir
@@ -24,7 +24,7 @@ fn unwrap(_1: Option) -> T {
 
     bb2: {
         StorageLive(_4);
-        _4 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
+        _4 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
     }
 
     bb3: {
diff --git a/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-unwind.mir b/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-unwind.mir
index 758aa45f2a2b..04176b82ccd5 100644
--- a/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-unwind.mir
+++ b/tests/mir-opt/no_drop_for_inactive_variant.unwrap.SimplifyCfg-pre-optimizations.after.panic-unwind.mir
@@ -24,7 +24,7 @@ fn unwrap(_1: Option) -> T {
 
     bb2: {
         StorageLive(_4);
-        _4 = begin_panic::<&str>(const "explicit panic") -> bb4;
+        _4 = std::rt::begin_panic::<&str>(const "explicit panic") -> bb4;
     }
 
     bb3: {
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir
index 013361d1d2fb..9202814adec7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir
@@ -14,14 +14,14 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                 scope 4 {
                     scope 12 (inlined Layout::size) {
                     }
-                    scope 13 (inlined Unique::<[T]>::cast::) {
+                    scope 13 (inlined std::ptr::Unique::<[T]>::cast::) {
                         scope 14 (inlined NonNull::<[T]>::cast::) {
                             scope 15 (inlined NonNull::<[T]>::as_ptr) {
                             }
                         }
                     }
-                    scope 16 (inlined  as From>>::from) {
-                        scope 17 (inlined Unique::::as_non_null_ptr) {
+                    scope 16 (inlined  as From>>::from) {
+                        scope 17 (inlined std::ptr::Unique::::as_non_null_ptr) {
                         }
                     }
                     scope 18 (inlined ::deallocate) {
@@ -45,7 +45,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         }
                     }
                 }
-                scope 5 (inlined Unique::<[T]>::as_ptr) {
+                scope 5 (inlined std::ptr::Unique::<[T]>::as_ptr) {
                     scope 6 (inlined NonNull::<[T]>::as_ptr) {
                     }
                 }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir
index 013361d1d2fb..9202814adec7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir
@@ -14,14 +14,14 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                 scope 4 {
                     scope 12 (inlined Layout::size) {
                     }
-                    scope 13 (inlined Unique::<[T]>::cast::) {
+                    scope 13 (inlined std::ptr::Unique::<[T]>::cast::) {
                         scope 14 (inlined NonNull::<[T]>::cast::) {
                             scope 15 (inlined NonNull::<[T]>::as_ptr) {
                             }
                         }
                     }
-                    scope 16 (inlined  as From>>::from) {
-                        scope 17 (inlined Unique::::as_non_null_ptr) {
+                    scope 16 (inlined  as From>>::from) {
+                        scope 17 (inlined std::ptr::Unique::::as_non_null_ptr) {
                         }
                     }
                     scope 18 (inlined ::deallocate) {
@@ -45,7 +45,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         }
                     }
                 }
-                scope 5 (inlined Unique::<[T]>::as_ptr) {
+                scope 5 (inlined std::ptr::Unique::<[T]>::as_ptr) {
                     scope 6 (inlined NonNull::<[T]>::as_ptr) {
                     }
                 }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir
index 013361d1d2fb..9202814adec7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir
@@ -14,14 +14,14 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                 scope 4 {
                     scope 12 (inlined Layout::size) {
                     }
-                    scope 13 (inlined Unique::<[T]>::cast::) {
+                    scope 13 (inlined std::ptr::Unique::<[T]>::cast::) {
                         scope 14 (inlined NonNull::<[T]>::cast::) {
                             scope 15 (inlined NonNull::<[T]>::as_ptr) {
                             }
                         }
                     }
-                    scope 16 (inlined  as From>>::from) {
-                        scope 17 (inlined Unique::::as_non_null_ptr) {
+                    scope 16 (inlined  as From>>::from) {
+                        scope 17 (inlined std::ptr::Unique::::as_non_null_ptr) {
                         }
                     }
                     scope 18 (inlined ::deallocate) {
@@ -45,7 +45,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         }
                     }
                 }
-                scope 5 (inlined Unique::<[T]>::as_ptr) {
+                scope 5 (inlined std::ptr::Unique::<[T]>::as_ptr) {
                     scope 6 (inlined NonNull::<[T]>::as_ptr) {
                     }
                 }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir
index 013361d1d2fb..9202814adec7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir
@@ -14,14 +14,14 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                 scope 4 {
                     scope 12 (inlined Layout::size) {
                     }
-                    scope 13 (inlined Unique::<[T]>::cast::) {
+                    scope 13 (inlined std::ptr::Unique::<[T]>::cast::) {
                         scope 14 (inlined NonNull::<[T]>::cast::) {
                             scope 15 (inlined NonNull::<[T]>::as_ptr) {
                             }
                         }
                     }
-                    scope 16 (inlined  as From>>::from) {
-                        scope 17 (inlined Unique::::as_non_null_ptr) {
+                    scope 16 (inlined  as From>>::from) {
+                        scope 17 (inlined std::ptr::Unique::::as_non_null_ptr) {
                         }
                     }
                     scope 18 (inlined ::deallocate) {
@@ -45,7 +45,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         }
                     }
                 }
-                scope 5 (inlined Unique::<[T]>::as_ptr) {
+                scope 5 (inlined std::ptr::Unique::<[T]>::as_ptr) {
                     scope 6 (inlined NonNull::<[T]>::as_ptr) {
                     }
                 }
diff --git a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir
index 6ab4b7712306..c8b9ff1dbed9 100644
--- a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir
@@ -139,14 +139,14 @@ fn vec_move(_1: Vec) -> () {
                     debug self => _31;
                     scope 23 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                         let mut _5: std::ptr::NonNull;
-                        scope 24 (inlined Unique::::cast::) {
+                        scope 24 (inlined std::ptr::Unique::::cast::) {
                             scope 25 (inlined NonNull::::cast::) {
                                 let mut _6: *const impl Sized;
                                 scope 26 (inlined NonNull::::as_ptr) {
                                 }
                             }
                         }
-                        scope 27 (inlined Unique::::as_non_null_ptr) {
+                        scope 27 (inlined std::ptr::Unique::::as_non_null_ptr) {
                         }
                     }
                 }
diff --git a/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-abort.mir
index 8308ecbad716..730aedf4f1a3 100644
--- a/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-abort.mir
@@ -16,13 +16,13 @@ fn vec_deref_to_slice(_1: &Vec) -> &[u8] {
                     scope 5 (inlined alloc::raw_vec::RawVecInner::ptr::) {
                         scope 6 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                             let mut _2: std::ptr::NonNull;
-                            scope 7 (inlined Unique::::cast::) {
+                            scope 7 (inlined std::ptr::Unique::::cast::) {
                                 scope 8 (inlined NonNull::::cast::) {
                                     scope 9 (inlined NonNull::::as_ptr) {
                                     }
                                 }
                             }
-                            scope 10 (inlined Unique::::as_non_null_ptr) {
+                            scope 10 (inlined std::ptr::Unique::::as_non_null_ptr) {
                             }
                         }
                         scope 11 (inlined NonNull::::as_ptr) {
diff --git a/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-unwind.mir
index 8308ecbad716..730aedf4f1a3 100644
--- a/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-unwind.mir
@@ -16,13 +16,13 @@ fn vec_deref_to_slice(_1: &Vec) -> &[u8] {
                     scope 5 (inlined alloc::raw_vec::RawVecInner::ptr::) {
                         scope 6 (inlined alloc::raw_vec::RawVecInner::non_null::) {
                             let mut _2: std::ptr::NonNull;
-                            scope 7 (inlined Unique::::cast::) {
+                            scope 7 (inlined std::ptr::Unique::::cast::) {
                                 scope 8 (inlined NonNull::::cast::) {
                                     scope 9 (inlined NonNull::::as_ptr) {
                                     }
                                 }
                             }
-                            scope 10 (inlined Unique::::as_non_null_ptr) {
+                            scope 10 (inlined std::ptr::Unique::::as_non_null_ptr) {
                             }
                         }
                         scope 11 (inlined NonNull::::as_ptr) {
diff --git a/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff
index 7012cc5aa7f6..f9965a529ebc 100644
--- a/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff
+++ b/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff
@@ -152,7 +152,7 @@
           StorageDead(_22);
           StorageDead(_21);
           StorageDead(_20);
-          _10 = _eprint(move _11) -> [return: bb6, unwind unreachable];
+          _10 = std::io::_eprint(move _11) -> [return: bb6, unwind unreachable];
       }
   
       bb6: {
diff --git a/tests/ui/async-await/async-closures/def-path.stderr b/tests/ui/async-await/async-closures/def-path.stderr
index a507fa697604..e140e9b94b30 100644
--- a/tests/ui/async-await/async-closures/def-path.stderr
+++ b/tests/ui/async-await/async-closures/def-path.stderr
@@ -5,11 +5,11 @@ LL |     let x = async || {};
    |                      -- the expected `async` closure body
 LL |
 LL |     let () = x();
-   |         ^^   --- this expression has type `{static main::{closure#0}::{closure#0} upvar_tys=?14t resume_ty=ResumeTy yield_ty=() return_ty=()}`
+   |         ^^   --- this expression has type `{static main::{closure#0}::{closure#0} upvar_tys=?14t resume_ty=std::future::ResumeTy yield_ty=() return_ty=()}`
    |         |
    |         expected `async` closure body, found `()`
    |
-   = note: expected `async` closure body `{static main::{closure#0}::{closure#0} upvar_tys=?14t resume_ty=ResumeTy yield_ty=() return_ty=()}`
+   = note: expected `async` closure body `{static main::{closure#0}::{closure#0} upvar_tys=?14t resume_ty=std::future::ResumeTy yield_ty=() return_ty=()}`
                          found unit type `()`
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/const-generics/adt_const_params/unsizing-wfcheck-issue-126272.stderr b/tests/ui/const-generics/adt_const_params/unsizing-wfcheck-issue-126272.stderr
index af29eaa35cb6..aecd97ef7763 100644
--- a/tests/ui/const-generics/adt_const_params/unsizing-wfcheck-issue-126272.stderr
+++ b/tests/ui/const-generics/adt_const_params/unsizing-wfcheck-issue-126272.stderr
@@ -99,7 +99,7 @@ LL | #[derive(Debug, PartialEq, Eq, ConstParamTy)]
    |                            ^^ unsatisfied trait bound introduced in this `derive` macro
    = note: 1 redundant requirement hidden
    = note: required for `&'static Bar` to implement `Eq`
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 
 error[E0277]: the size for values of type `dyn Debug` cannot be known at compilation time
diff --git a/tests/ui/consts/const-eval/format.rs b/tests/ui/consts/const-eval/format.rs
index a8085a786e18..f25c7018f826 100644
--- a/tests/ui/consts/const-eval/format.rs
+++ b/tests/ui/consts/const-eval/format.rs
@@ -6,7 +6,7 @@ const fn failure() {
 const fn print() {
     println!("{:?}", 0);
     //~^ ERROR cannot call non-const formatting macro in constant functions
-    //~| ERROR cannot call non-const function `_print` in constant functions
+    //~| ERROR cannot call non-const function `std::io::_print` in constant functions
 }
 
 const fn format_args() {
diff --git a/tests/ui/consts/const-eval/format.stderr b/tests/ui/consts/const-eval/format.stderr
index ea191eb5c098..06bada8da011 100644
--- a/tests/ui/consts/const-eval/format.stderr
+++ b/tests/ui/consts/const-eval/format.stderr
@@ -15,7 +15,7 @@ LL |     println!("{:?}", 0);
    = note: calls in constant functions are limited to constant functions, tuple structs and tuple variants
    = note: this error originates in the macro `println` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error[E0015]: cannot call non-const function `_print` in constant functions
+error[E0015]: cannot call non-const function `std::io::_print` in constant functions
   --> $DIR/format.rs:7:5
    |
 LL |     println!("{:?}", 0);
diff --git a/tests/ui/contracts/contract-captures-via-closure-noncopy.rs b/tests/ui/contracts/contract-captures-via-closure-noncopy.rs
index c7aa72d2b0f6..5153a7b48dcd 100644
--- a/tests/ui/contracts/contract-captures-via-closure-noncopy.rs
+++ b/tests/ui/contracts/contract-captures-via-closure-noncopy.rs
@@ -13,7 +13,7 @@ struct Baz {
 #[core::contracts::ensures({let old = x; move |ret:&Baz| ret.baz == old.baz*2 })]
 // Relevant thing is this:  ^^^^^^^^^^^
 // because we are capturing state that is non-Copy.
-//~^^^ ERROR trait bound `Baz: std::marker::Copy` is not satisfied
+//~^^^ ERROR trait bound `Baz: Copy` is not satisfied
 fn doubler(x: Baz) -> Baz {
     Baz { baz: x.baz + 10 }
 }
diff --git a/tests/ui/contracts/contract-captures-via-closure-noncopy.stderr b/tests/ui/contracts/contract-captures-via-closure-noncopy.stderr
index 5f55faed80c8..20c220e98bcc 100644
--- a/tests/ui/contracts/contract-captures-via-closure-noncopy.stderr
+++ b/tests/ui/contracts/contract-captures-via-closure-noncopy.stderr
@@ -1,4 +1,4 @@
-error[E0277]: the trait bound `Baz: std::marker::Copy` is not satisfied in `{closure@$DIR/contract-captures-via-closure-noncopy.rs:13:42: 13:57}`
+error[E0277]: the trait bound `Baz: Copy` is not satisfied in `{closure@$DIR/contract-captures-via-closure-noncopy.rs:13:42: 13:57}`
   --> $DIR/contract-captures-via-closure-noncopy.rs:13:1
    |
 LL | #[core::contracts::ensures({let old = x; move |ret:&Baz| ret.baz == old.baz*2 })]
@@ -9,7 +9,7 @@ LL | #[core::contracts::ensures({let old = x; move |ret:&Baz| ret.baz == old.baz
    | unsatisfied trait bound
    | required by a bound introduced by this call
    |
-   = help: within `{closure@$DIR/contract-captures-via-closure-noncopy.rs:13:42: 13:57}`, the trait `std::marker::Copy` is not implemented for `Baz`
+   = help: within `{closure@$DIR/contract-captures-via-closure-noncopy.rs:13:42: 13:57}`, the trait `Copy` is not implemented for `Baz`
 note: required because it's used within this closure
   --> $DIR/contract-captures-via-closure-noncopy.rs:13:42
    |
diff --git a/tests/ui/derives/derives-span-Eq-enum-struct-variant.stderr b/tests/ui/derives/derives-span-Eq-enum-struct-variant.stderr
index e0cb3c1b43da..42dc8d46b575 100644
--- a/tests/ui/derives/derives-span-Eq-enum-struct-variant.stderr
+++ b/tests/ui/derives/derives-span-Eq-enum-struct-variant.stderr
@@ -7,7 +7,7 @@ LL | #[derive(Eq,PartialEq)]
 LL |      x: Error
    |      ^^^^^^^^ the trait `Eq` is not implemented for `Error`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 help: consider annotating `Error` with `#[derive(Eq)]`
    |
diff --git a/tests/ui/derives/derives-span-Eq-enum.stderr b/tests/ui/derives/derives-span-Eq-enum.stderr
index 2f09b9ea385f..ef1d9e3242ad 100644
--- a/tests/ui/derives/derives-span-Eq-enum.stderr
+++ b/tests/ui/derives/derives-span-Eq-enum.stderr
@@ -7,7 +7,7 @@ LL | #[derive(Eq,PartialEq)]
 LL |      Error
    |      ^^^^^ the trait `Eq` is not implemented for `Error`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 help: consider annotating `Error` with `#[derive(Eq)]`
    |
diff --git a/tests/ui/derives/derives-span-Eq-struct.stderr b/tests/ui/derives/derives-span-Eq-struct.stderr
index c16d9118e10f..bae7bb0361df 100644
--- a/tests/ui/derives/derives-span-Eq-struct.stderr
+++ b/tests/ui/derives/derives-span-Eq-struct.stderr
@@ -7,7 +7,7 @@ LL | struct Struct {
 LL |     x: Error
    |     ^^^^^^^^ the trait `Eq` is not implemented for `Error`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 help: consider annotating `Error` with `#[derive(Eq)]`
    |
diff --git a/tests/ui/derives/derives-span-Eq-tuple-struct.stderr b/tests/ui/derives/derives-span-Eq-tuple-struct.stderr
index dac295eed919..13396cb27246 100644
--- a/tests/ui/derives/derives-span-Eq-tuple-struct.stderr
+++ b/tests/ui/derives/derives-span-Eq-tuple-struct.stderr
@@ -7,7 +7,7 @@ LL | struct Struct(
 LL |     Error
    |     ^^^^^ the trait `Eq` is not implemented for `Error`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 help: consider annotating `Error` with `#[derive(Eq)]`
    |
diff --git a/tests/ui/deriving/issue-103157.stderr b/tests/ui/deriving/issue-103157.stderr
index 51d4d0a89745..0e4a3f75db3f 100644
--- a/tests/ui/deriving/issue-103157.stderr
+++ b/tests/ui/deriving/issue-103157.stderr
@@ -18,7 +18,7 @@ LL |     Float(Option),
              u16
            and 4 others
    = note: required for `Option` to implement `Eq`
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/issues/issue-27340.stderr b/tests/ui/issues/issue-27340.stderr
index d5ff29a618b0..7cde901ffd95 100644
--- a/tests/ui/issues/issue-27340.stderr
+++ b/tests/ui/issues/issue-27340.stderr
@@ -16,7 +16,7 @@ LL |
 LL | struct Bar(Foo);
    |            ^^^ the trait `Clone` is not implemented for `Foo`
    |
-note: required by a bound in `AssertParamIsClone`
+note: required by a bound in `std::clone::AssertParamIsClone`
   --> $SRC_DIR/core/src/clone.rs:LL:COL
 help: consider annotating `Foo` with `#[derive(Clone)]`
    |
diff --git a/tests/ui/kindck/kindck-send-object.stderr b/tests/ui/kindck/kindck-send-object.stderr
index 0e2ff1730c8d..b71d4029350e 100644
--- a/tests/ui/kindck/kindck-send-object.stderr
+++ b/tests/ui/kindck/kindck-send-object.stderr
@@ -19,7 +19,7 @@ LL |     assert_send::>();
    |                   ^^^^^^^^^^^^^^ `dyn Dummy` cannot be sent between threads safely
    |
    = help: the trait `Send` is not implemented for `dyn Dummy`
-   = note: required for `Unique` to implement `Send`
+   = note: required for `std::ptr::Unique` to implement `Send`
 note: required because it appears within the type `Box`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
 note: required by a bound in `assert_send`
diff --git a/tests/ui/kindck/kindck-send-object1.stderr b/tests/ui/kindck/kindck-send-object1.stderr
index e3ff2eb9ff4c..2184ae704673 100644
--- a/tests/ui/kindck/kindck-send-object1.stderr
+++ b/tests/ui/kindck/kindck-send-object1.stderr
@@ -19,7 +19,7 @@ LL |     assert_send::>();
    |                   ^^^^^^^^^^^^^^^^^^^ `(dyn Dummy + 'a)` cannot be sent between threads safely
    |
    = help: the trait `Send` is not implemented for `(dyn Dummy + 'a)`
-   = note: required for `Unique<(dyn Dummy + 'a)>` to implement `Send`
+   = note: required for `std::ptr::Unique<(dyn Dummy + 'a)>` to implement `Send`
 note: required because it appears within the type `Box<(dyn Dummy + 'a)>`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
 note: required by a bound in `assert_send`
diff --git a/tests/ui/kindck/kindck-send-object2.stderr b/tests/ui/kindck/kindck-send-object2.stderr
index 8898bf5b3fab..52a7055b4229 100644
--- a/tests/ui/kindck/kindck-send-object2.stderr
+++ b/tests/ui/kindck/kindck-send-object2.stderr
@@ -19,7 +19,7 @@ LL |     assert_send::>();
    |                   ^^^^^^^^^^^^^^ `dyn Dummy` cannot be sent between threads safely
    |
    = help: the trait `Send` is not implemented for `dyn Dummy`
-   = note: required for `Unique` to implement `Send`
+   = note: required for `std::ptr::Unique` to implement `Send`
 note: required because it appears within the type `Box`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
 note: required by a bound in `assert_send`
diff --git a/tests/ui/kindck/kindck-send-owned.stderr b/tests/ui/kindck/kindck-send-owned.stderr
index 860a9391bbb0..c433d80cf140 100644
--- a/tests/ui/kindck/kindck-send-owned.stderr
+++ b/tests/ui/kindck/kindck-send-owned.stderr
@@ -5,7 +5,7 @@ LL |     assert_send::>();
    |                   ^^^^^^^^^^^^ `*mut u8` cannot be sent between threads safely
    |
    = help: the trait `Send` is not implemented for `*mut u8`
-   = note: required for `Unique<*mut u8>` to implement `Send`
+   = note: required for `std::ptr::Unique<*mut u8>` to implement `Send`
 note: required because it appears within the type `Box<*mut u8>`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
 note: required by a bound in `assert_send`
diff --git a/tests/ui/limits/type-length-limit-enforcement.stderr b/tests/ui/limits/type-length-limit-enforcement.stderr
index bfea0b5a4482..82855bd75528 100644
--- a/tests/ui/limits/type-length-limit-enforcement.stderr
+++ b/tests/ui/limits/type-length-limit-enforcement.stderr
@@ -8,9 +8,11 @@ LL |     drop::>(None);
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/type-length-limit-enforcement.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
-error: reached the type-length limit while instantiating `<{closure@rt::lang_start<()>::{closure#0}} as FnMut<()>>::call_mut`
+error: reached the type-length limit while instantiating `<{closure@...} as FnMut<()>>::call_mut`
    |
    = help: consider adding a `#![type_length_limit="10"]` attribute to your crate
+   = note: the full name for the type has been written to '$TEST_BUILD_DIR/type-length-limit-enforcement.long-type-$LONG_TYPE_HASH.txt'
+   = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/proc-macro/bad-projection.stderr b/tests/ui/proc-macro/bad-projection.stderr
index f2981499367b..5b472d407044 100644
--- a/tests/ui/proc-macro/bad-projection.stderr
+++ b/tests/ui/proc-macro/bad-projection.stderr
@@ -33,7 +33,7 @@ LL | pub fn uwu() -> <() as Project>::Assoc {}
    | takes 0 arguments
    | required by a bound introduced by this call
    |
-note: required by a bound in `ProcMacro::bang`
+note: required by a bound in `proc_macro::bridge::client::ProcMacro::bang`
   --> $SRC_DIR/proc_macro/src/bridge/client.rs:LL:COL
 
 error[E0277]: the trait bound `(): Project` is not satisfied
diff --git a/tests/ui/proc-macro/proc-macro-abi.stderr b/tests/ui/proc-macro/proc-macro-abi.stderr
index ccc72e5187ed..ccefdbfa3a86 100644
--- a/tests/ui/proc-macro/proc-macro-abi.stderr
+++ b/tests/ui/proc-macro/proc-macro-abi.stderr
@@ -4,8 +4,8 @@ error: function-like proc macro has incorrect signature
 LL | pub extern "C" fn abi(a: TokenStream) -> TokenStream {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected "Rust" fn, found "C" fn
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `extern "C" fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `extern "C" fn(TokenStream) -> TokenStream`
 
 error: function-like proc macro has incorrect signature
   --> $DIR/proc-macro-abi.rs:17:1
@@ -13,8 +13,8 @@ error: function-like proc macro has incorrect signature
 LL | pub extern "system" fn abi2(a: TokenStream) -> TokenStream {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected "Rust" fn, found "system" fn
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `extern "system" fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `extern "system" fn(TokenStream) -> TokenStream`
 
 error: function-like proc macro has incorrect signature
   --> $DIR/proc-macro-abi.rs:23:1
@@ -22,8 +22,8 @@ error: function-like proc macro has incorrect signature
 LL | pub extern fn abi3(a: TokenStream) -> TokenStream {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected "Rust" fn, found "C" fn
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `extern "C" fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `extern "C" fn(TokenStream) -> TokenStream`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/proc-macro/quote/does-not-have-iter-interpolated-dup.stderr b/tests/ui/proc-macro/quote/does-not-have-iter-interpolated-dup.stderr
index 0bcea9b85f47..86a00713a456 100644
--- a/tests/ui/proc-macro/quote/does-not-have-iter-interpolated-dup.stderr
+++ b/tests/ui/proc-macro/quote/does-not-have-iter-interpolated-dup.stderr
@@ -5,7 +5,7 @@ LL |     quote!($($nonrep $nonrep)*);
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |     |
    |     expected `HasIterator`, found `ThereIsNoIteratorInRepetition`
-   |     here the type of `has_iter` is inferred to be `ThereIsNoIteratorInRepetition`
+   |     here the type of `has_iter` is inferred to be `proc_macro::ThereIsNoIteratorInRepetition`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/proc-macro/quote/does-not-have-iter-interpolated.stderr b/tests/ui/proc-macro/quote/does-not-have-iter-interpolated.stderr
index d945ab41a12e..325e50f9796a 100644
--- a/tests/ui/proc-macro/quote/does-not-have-iter-interpolated.stderr
+++ b/tests/ui/proc-macro/quote/does-not-have-iter-interpolated.stderr
@@ -5,7 +5,7 @@ LL |     quote!($($nonrep)*);
    |     ^^^^^^^^^^^^^^^^^^^
    |     |
    |     expected `HasIterator`, found `ThereIsNoIteratorInRepetition`
-   |     here the type of `has_iter` is inferred to be `ThereIsNoIteratorInRepetition`
+   |     here the type of `has_iter` is inferred to be `proc_macro::ThereIsNoIteratorInRepetition`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/proc-macro/quote/not-quotable.stderr b/tests/ui/proc-macro/quote/not-quotable.stderr
index b00d029946d6..4177d9c672b5 100644
--- a/tests/ui/proc-macro/quote/not-quotable.stderr
+++ b/tests/ui/proc-macro/quote/not-quotable.stderr
@@ -11,11 +11,11 @@ LL |     let _ = quote! { $ip };
              &T
              &mut T
              Box
+             CStr
              CString
              Cow<'_, T>
              Option
              Rc
-             bool
            and 24 others
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/proc-macro/signature-proc-macro-attribute.stderr b/tests/ui/proc-macro/signature-proc-macro-attribute.stderr
index ce832eaa5c7a..9dfb081a10e6 100644
--- a/tests/ui/proc-macro/signature-proc-macro-attribute.stderr
+++ b/tests/ui/proc-macro/signature-proc-macro-attribute.stderr
@@ -4,8 +4,8 @@ error: attribute proc macro has incorrect signature
 LL | pub fn bad_input(input: String) -> TokenStream {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ incorrect number of function parameters
    |
-   = note: expected signature `fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(std::string::String) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream, TokenStream) -> TokenStream`
+              found signature `fn(String) -> TokenStream`
 
 error: attribute proc macro has incorrect signature
   --> $DIR/signature-proc-macro-attribute.rs:16:1
@@ -13,8 +13,8 @@ error: attribute proc macro has incorrect signature
 LL | pub fn bad_output(input: TokenStream) -> String {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ incorrect number of function parameters
    |
-   = note: expected signature `fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(proc_macro::TokenStream) -> std::string::String`
+   = note: expected signature `fn(TokenStream, TokenStream) -> TokenStream`
+              found signature `fn(TokenStream) -> String`
 
 error: attribute proc macro has incorrect signature
   --> $DIR/signature-proc-macro-attribute.rs:22:1
@@ -22,8 +22,8 @@ error: attribute proc macro has incorrect signature
 LL | pub fn bad_everything(input: String) -> String {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ incorrect number of function parameters
    |
-   = note: expected signature `fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(std::string::String) -> std::string::String`
+   = note: expected signature `fn(TokenStream, TokenStream) -> TokenStream`
+              found signature `fn(String) -> String`
 
 error: attribute proc macro has incorrect signature
   --> $DIR/signature-proc-macro-attribute.rs:28:52
@@ -31,8 +31,8 @@ error: attribute proc macro has incorrect signature
 LL | pub fn too_many(a: TokenStream, b: TokenStream, c: String) -> TokenStream {
    |                                                    ^^^^^^ incorrect number of function parameters
    |
-   = note: expected signature `fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(proc_macro::TokenStream, proc_macro::TokenStream, std::string::String) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream, TokenStream) -> TokenStream`
+              found signature `fn(TokenStream, TokenStream, String) -> TokenStream`
 
 error: aborting due to 4 previous errors
 
diff --git a/tests/ui/proc-macro/signature-proc-macro-derive.stderr b/tests/ui/proc-macro/signature-proc-macro-derive.stderr
index 03c6abad17d9..3539ae7c2e17 100644
--- a/tests/ui/proc-macro/signature-proc-macro-derive.stderr
+++ b/tests/ui/proc-macro/signature-proc-macro-derive.stderr
@@ -2,28 +2,28 @@ error: derive proc macro has incorrect signature
   --> $DIR/signature-proc-macro-derive.rs:10:25
    |
 LL | pub fn bad_input(input: String) -> TokenStream {
-   |                         ^^^^^^ expected `proc_macro::TokenStream`, found `std::string::String`
+   |                         ^^^^^^ expected `TokenStream`, found `String`
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(std::string::String) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(String) -> TokenStream`
 
 error: derive proc macro has incorrect signature
   --> $DIR/signature-proc-macro-derive.rs:16:42
    |
 LL | pub fn bad_output(input: TokenStream) -> String {
-   |                                          ^^^^^^ expected `proc_macro::TokenStream`, found `std::string::String`
+   |                                          ^^^^^^ expected `TokenStream`, found `String`
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(proc_macro::TokenStream) -> std::string::String`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(TokenStream) -> String`
 
 error: derive proc macro has incorrect signature
   --> $DIR/signature-proc-macro-derive.rs:22:30
    |
 LL | pub fn bad_everything(input: String) -> String {
-   |                              ^^^^^^ expected `proc_macro::TokenStream`, found `std::string::String`
+   |                              ^^^^^^ expected `TokenStream`, found `String`
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(std::string::String) -> std::string::String`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(String) -> String`
 
 error: derive proc macro has incorrect signature
   --> $DIR/signature-proc-macro-derive.rs:28:36
@@ -31,8 +31,8 @@ error: derive proc macro has incorrect signature
 LL | pub fn too_many(a: TokenStream, b: TokenStream, c: String) -> TokenStream {
    |                                    ^^^^^^^^^^^ incorrect number of function parameters
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(proc_macro::TokenStream, proc_macro::TokenStream, std::string::String) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(TokenStream, TokenStream, String) -> TokenStream`
 
 error: aborting due to 4 previous errors
 
diff --git a/tests/ui/proc-macro/signature-proc-macro.stderr b/tests/ui/proc-macro/signature-proc-macro.stderr
index dd2cb0570daa..1959d8c6d615 100644
--- a/tests/ui/proc-macro/signature-proc-macro.stderr
+++ b/tests/ui/proc-macro/signature-proc-macro.stderr
@@ -2,28 +2,28 @@ error: function-like proc macro has incorrect signature
   --> $DIR/signature-proc-macro.rs:10:25
    |
 LL | pub fn bad_input(input: String) -> TokenStream {
-   |                         ^^^^^^ expected `proc_macro::TokenStream`, found `std::string::String`
+   |                         ^^^^^^ expected `TokenStream`, found `String`
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(std::string::String) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(String) -> TokenStream`
 
 error: function-like proc macro has incorrect signature
   --> $DIR/signature-proc-macro.rs:16:42
    |
 LL | pub fn bad_output(input: TokenStream) -> String {
-   |                                          ^^^^^^ expected `proc_macro::TokenStream`, found `std::string::String`
+   |                                          ^^^^^^ expected `TokenStream`, found `String`
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(proc_macro::TokenStream) -> std::string::String`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(TokenStream) -> String`
 
 error: function-like proc macro has incorrect signature
   --> $DIR/signature-proc-macro.rs:22:30
    |
 LL | pub fn bad_everything(input: String) -> String {
-   |                              ^^^^^^ expected `proc_macro::TokenStream`, found `std::string::String`
+   |                              ^^^^^^ expected `TokenStream`, found `String`
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(std::string::String) -> std::string::String`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(String) -> String`
 
 error: function-like proc macro has incorrect signature
   --> $DIR/signature-proc-macro.rs:28:36
@@ -31,8 +31,8 @@ error: function-like proc macro has incorrect signature
 LL | pub fn too_many(a: TokenStream, b: TokenStream, c: String) -> TokenStream {
    |                                    ^^^^^^^^^^^ incorrect number of function parameters
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
-              found signature `fn(proc_macro::TokenStream, proc_macro::TokenStream, std::string::String) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
+              found signature `fn(TokenStream, TokenStream, String) -> TokenStream`
 
 error: aborting due to 4 previous errors
 
diff --git a/tests/ui/proc-macro/signature.stderr b/tests/ui/proc-macro/signature.stderr
index fd679442b6af..2c1973eb6e6d 100644
--- a/tests/ui/proc-macro/signature.stderr
+++ b/tests/ui/proc-macro/signature.stderr
@@ -4,7 +4,7 @@ error: derive proc macro has incorrect signature
 LL | pub unsafe extern "C" fn foo(a: i32, b: u32) -> u32 {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected safe fn, found unsafe fn
    |
-   = note: expected signature `fn(proc_macro::TokenStream) -> proc_macro::TokenStream`
+   = note: expected signature `fn(TokenStream) -> TokenStream`
               found signature `unsafe extern "C" fn(i32, u32) -> u32`
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/static/global-variable-promotion-error-7364.stderr b/tests/ui/static/global-variable-promotion-error-7364.stderr
index b9d75676bef8..9f0026621c13 100644
--- a/tests/ui/static/global-variable-promotion-error-7364.stderr
+++ b/tests/ui/static/global-variable-promotion-error-7364.stderr
@@ -6,7 +6,7 @@ LL | static boxed: Box> = Box::new(RefCell::new(0));
    |
    = help: the trait `Sync` is not implemented for `RefCell`
    = note: if you want to do aliasing and mutation between multiple threads, use `std::sync::RwLock` instead
-   = note: required for `Unique>` to implement `Sync`
+   = note: required for `std::ptr::Unique>` to implement `Sync`
 note: required because it appears within the type `Box>`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
    = note: shared static variables must have a type that implements `Sync`
diff --git a/tests/ui/traits/const-traits/issue-79450.rs b/tests/ui/traits/const-traits/issue-79450.rs
index e74da811fc80..c6234f8616d7 100644
--- a/tests/ui/traits/const-traits/issue-79450.rs
+++ b/tests/ui/traits/const-traits/issue-79450.rs
@@ -5,7 +5,7 @@ const trait Tr {
     fn req(&self);
 
     fn prov(&self) {
-        println!("lul"); //~ ERROR: cannot call non-const function `_print` in constant functions
+        println!("lul"); //~ ERROR: cannot call non-const function `std::io::_print` in constant functions
         self.req();
     }
 }
diff --git a/tests/ui/traits/const-traits/issue-79450.stderr b/tests/ui/traits/const-traits/issue-79450.stderr
index c10023e9f0ef..702e93a76a8f 100644
--- a/tests/ui/traits/const-traits/issue-79450.stderr
+++ b/tests/ui/traits/const-traits/issue-79450.stderr
@@ -1,4 +1,4 @@
-error[E0015]: cannot call non-const function `_print` in constant functions
+error[E0015]: cannot call non-const function `std::io::_print` in constant functions
   --> $DIR/issue-79450.rs:8:9
    |
 LL |         println!("lul");
diff --git a/tests/ui/traits/cycle-cache-err-60010.stderr b/tests/ui/traits/cycle-cache-err-60010.stderr
index 4f5e31818321..9665d5badf59 100644
--- a/tests/ui/traits/cycle-cache-err-60010.stderr
+++ b/tests/ui/traits/cycle-cache-err-60010.stderr
@@ -6,7 +6,7 @@ LL |     _parse: >::Data,
    |
 note: required because it appears within the type `PhantomData`
   --> $SRC_DIR/core/src/marker.rs:LL:COL
-note: required because it appears within the type `Unique`
+note: required because it appears within the type `std::ptr::Unique`
   --> $SRC_DIR/core/src/ptr/unique.rs:LL:COL
 note: required because it appears within the type `Box`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
@@ -45,7 +45,7 @@ LL |     type Storage = SalsaStorage;
    |
 note: required because it appears within the type `PhantomData`
   --> $SRC_DIR/core/src/marker.rs:LL:COL
-note: required because it appears within the type `Unique`
+note: required because it appears within the type `std::ptr::Unique`
   --> $SRC_DIR/core/src/ptr/unique.rs:LL:COL
 note: required because it appears within the type `Box`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
diff --git a/tests/ui/traits/negative-impls/negated-auto-traits-error.stderr b/tests/ui/traits/negative-impls/negated-auto-traits-error.stderr
index f450f786f608..b7d8484e1041 100644
--- a/tests/ui/traits/negative-impls/negated-auto-traits-error.stderr
+++ b/tests/ui/traits/negative-impls/negated-auto-traits-error.stderr
@@ -77,12 +77,12 @@ error[E0277]: `dummy2::TestType` cannot be sent between threads safely
   --> $DIR/negated-auto-traits-error.rs:48:13
    |
 LL |     is_send(Box::new(TestType));
-   |     ------- ^^^^^^^^^^^^^^^^^^ the trait `Send` is not implemented for `Unique`
+   |     ------- ^^^^^^^^^^^^^^^^^^ the trait `Send` is not implemented for `std::ptr::Unique`
    |     |
    |     required by a bound introduced by this call
    |
-   = note: the trait bound `Unique: Send` is not satisfied
-   = note: required for `Unique` to implement `Send`
+   = note: the trait bound `std::ptr::Unique: Send` is not satisfied
+   = note: required for `std::ptr::Unique` to implement `Send`
 note: required because it appears within the type `Box`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
 note: required by a bound in `is_send`
@@ -113,7 +113,7 @@ note: required because it appears within the type `Outer2`
    |
 LL | struct Outer2(T);
    |        ^^^^^^
-   = note: required for `Unique>` to implement `Send`
+   = note: required for `std::ptr::Unique>` to implement `Send`
 note: required because it appears within the type `Box>`
   --> $SRC_DIR/alloc/src/boxed.rs:LL:COL
 note: required by a bound in `is_send`
diff --git a/tests/ui/trimmed-paths/core-unicode.rs b/tests/ui/trimmed-paths/core-unicode.rs
index 54bde92a5335..4a3eeca62970 100644
--- a/tests/ui/trimmed-paths/core-unicode.rs
+++ b/tests/ui/trimmed-paths/core-unicode.rs
@@ -14,6 +14,6 @@ fn main() {
     let PhantomData::<(inner::unicode, u32)> = PhantomData::<(u32, inner::unicode)>;
     //~^ ERROR mismatched types [E0308]
     //~| NOTE expected `PhantomData<(u32, unicode)>`, found `PhantomData<(unicode, u32)>`
-    //~| NOTE this expression has type `PhantomData<(u32, inner::unicode)>`
-    //~| NOTE expected struct `PhantomData<(u32, inner::unicode)>`
+    //~| NOTE this expression has type `PhantomData<(u32, unicode)>`
+    //~| NOTE expected struct `PhantomData<(u32, unicode)>`
 }
diff --git a/tests/ui/trimmed-paths/core-unicode.stderr b/tests/ui/trimmed-paths/core-unicode.stderr
index 9023200d1c92..7351896b0b54 100644
--- a/tests/ui/trimmed-paths/core-unicode.stderr
+++ b/tests/ui/trimmed-paths/core-unicode.stderr
@@ -2,12 +2,12 @@ error[E0308]: mismatched types
   --> $DIR/core-unicode.rs:14:9
    |
 LL |     let PhantomData::<(inner::unicode, u32)> = PhantomData::<(u32, inner::unicode)>;
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^   ------------------------------------ this expression has type `PhantomData<(u32, inner::unicode)>`
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^   ------------------------------------ this expression has type `PhantomData<(u32, unicode)>`
    |         |
    |         expected `PhantomData<(u32, unicode)>`, found `PhantomData<(unicode, u32)>`
    |
-   = note: expected struct `PhantomData<(u32, inner::unicode)>`
-              found struct `PhantomData<(inner::unicode, u32)>`
+   = note: expected struct `PhantomData<(u32, unicode)>`
+              found struct `PhantomData<(unicode, u32)>`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/trimmed-paths/doc-hidden.rs b/tests/ui/trimmed-paths/doc-hidden.rs
index b73d04c3c6ae..c3125385c7e4 100644
--- a/tests/ui/trimmed-paths/doc-hidden.rs
+++ b/tests/ui/trimmed-paths/doc-hidden.rs
@@ -45,7 +45,7 @@ fn uses_local() {
     ) = 3u32;
     //~^ ERROR mismatched types [E0308]
     //~| NOTE expected `(ActuallyPub, ..., ..., ..., ..., ...)`, found `u32`
-    //~| NOTE expected tuple `(local::ActuallyPub, local::DocHidden, local::pub_mod::ActuallyPubInPubMod, local::pub_mod::DocHiddenInPubMod, local::hidden_mod::ActuallyPubInHiddenMod, local::hidden_mod::DocHiddenInHiddenMod)`
+    //~| NOTE expected tuple `(local::ActuallyPub, DocHidden, local::pub_mod::ActuallyPubInPubMod, DocHiddenInPubMod, ActuallyPubInHiddenMod, DocHiddenInHiddenMod)`
 }
 
 fn uses_helper() {
diff --git a/tests/ui/trimmed-paths/doc-hidden.stderr b/tests/ui/trimmed-paths/doc-hidden.stderr
index 6f6f8f21fbe0..167c92c50a35 100644
--- a/tests/ui/trimmed-paths/doc-hidden.stderr
+++ b/tests/ui/trimmed-paths/doc-hidden.stderr
@@ -13,7 +13,7 @@ LL | |     ) = 3u32;
    | |_____|
    |       expected due to this
    |
-   = note: expected tuple `(local::ActuallyPub, local::DocHidden, local::pub_mod::ActuallyPubInPubMod, local::pub_mod::DocHiddenInPubMod, local::hidden_mod::ActuallyPubInHiddenMod, local::hidden_mod::DocHiddenInHiddenMod)`
+   = note: expected tuple `(local::ActuallyPub, DocHidden, local::pub_mod::ActuallyPubInPubMod, DocHiddenInPubMod, ActuallyPubInHiddenMod, DocHiddenInHiddenMod)`
                found type `u32`
 
 error[E0308]: mismatched types
diff --git a/tests/ui/type-inference/panic-with-unspecified-type.stderr b/tests/ui/type-inference/panic-with-unspecified-type.stderr
index 5f08a7552632..cd8485f392bc 100644
--- a/tests/ui/type-inference/panic-with-unspecified-type.stderr
+++ b/tests/ui/type-inference/panic-with-unspecified-type.stderr
@@ -8,7 +8,7 @@ LL |     panic!(std::default::Default::default());
    |     required by a bound introduced by this call
    |
    = note: cannot satisfy `_: Any`
-note: required by a bound in `begin_panic`
+note: required by a bound in `std::rt::begin_panic`
   --> $SRC_DIR/std/src/panicking.rs:LL:COL
 
 error: aborting due to 1 previous error
diff --git a/tests/ui/type/pattern_types/derives_fail.stderr b/tests/ui/type/pattern_types/derives_fail.stderr
index 6b2e27494f0e..45c9bae1f280 100644
--- a/tests/ui/type/pattern_types/derives_fail.stderr
+++ b/tests/ui/type/pattern_types/derives_fail.stderr
@@ -16,7 +16,7 @@ LL | #[repr(transparent)]
 LL | struct Nanoseconds(NanoI32);
    |                    ^^^^^^^ the trait `Eq` is not implemented for `(i32) is 0..=999999999`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 
 error[E0277]: `(i32) is 0..=999999999` doesn't implement `Debug`
diff --git a/tests/ui/union/union-derive-clone.stderr b/tests/ui/union/union-derive-clone.stderr
index 679ab6a38e49..18f862aaa7d6 100644
--- a/tests/ui/union/union-derive-clone.stderr
+++ b/tests/ui/union/union-derive-clone.stderr
@@ -4,7 +4,7 @@ error[E0277]: the trait bound `U1: Copy` is not satisfied
 LL | #[derive(Clone)]
    |          ^^^^^ the trait `Copy` is not implemented for `U1`
    |
-note: required by a bound in `AssertParamIsCopy`
+note: required by a bound in `std::clone::AssertParamIsCopy`
   --> $SRC_DIR/core/src/clone.rs:LL:COL
 help: consider annotating `U1` with `#[derive(Copy)]`
    |
diff --git a/tests/ui/union/union-derive-eq.current.stderr b/tests/ui/union/union-derive-eq.current.stderr
index a0339687dad4..df8e6db887bc 100644
--- a/tests/ui/union/union-derive-eq.current.stderr
+++ b/tests/ui/union/union-derive-eq.current.stderr
@@ -7,7 +7,7 @@ LL | union U2 {
 LL |     a: PartialEqNotEq,
    |     ^^^^^^^^^^^^^^^^^ the trait `Eq` is not implemented for `PartialEqNotEq`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 help: consider annotating `PartialEqNotEq` with `#[derive(Eq)]`
    |
diff --git a/tests/ui/union/union-derive-eq.next.stderr b/tests/ui/union/union-derive-eq.next.stderr
index a0339687dad4..df8e6db887bc 100644
--- a/tests/ui/union/union-derive-eq.next.stderr
+++ b/tests/ui/union/union-derive-eq.next.stderr
@@ -7,7 +7,7 @@ LL | union U2 {
 LL |     a: PartialEqNotEq,
    |     ^^^^^^^^^^^^^^^^^ the trait `Eq` is not implemented for `PartialEqNotEq`
    |
-note: required by a bound in `AssertParamIsEq`
+note: required by a bound in `std::cmp::AssertParamIsEq`
   --> $SRC_DIR/core/src/cmp.rs:LL:COL
 help: consider annotating `PartialEqNotEq` with `#[derive(Eq)]`
    |

From 40691d0ca6a20bae367533fb2b58320251b4ad17 Mon Sep 17 00:00:00 2001
From: tiif 
Date: Mon, 19 Jan 2026 01:46:08 +0000
Subject: [PATCH 0367/1843] Add myself to the review rotation

---
 triagebot.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/triagebot.toml b/triagebot.toml
index e51622f1e5a9..a25c2a0a388c 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1465,6 +1465,7 @@ compiler = [
     "@oli-obk",
     "@petrochenkov",
     "@SparrowLii",
+    "@tiif",
     "@WaffleLapkin",
     "@wesleywiser",
 ]

From 37135122179974e9073f4a885b70044cedfa3f80 Mon Sep 17 00:00:00 2001
From: mu001999 
Date: Sat, 17 Jan 2026 22:53:08 +0800
Subject: [PATCH 0368/1843] Parse ident with allowing recovery when trying to
 recover in diagnosing

---
 .../rustc_parse/src/parser/diagnostics.rs     |  2 +-
 compiler/rustc_parse/src/parser/item.rs       |  3 +-
 compiler/rustc_parse/src/parser/mod.rs        |  2 +-
 .../kw-in-const-item-pos-recovery-149692.rs   |  3 +-
 ...w-in-const-item-pos-recovery-149692.stderr | 19 ++++++--
 .../macro/kw-in-item-pos-recovery-149692.rs   |  9 ++--
 .../kw-in-item-pos-recovery-149692.stderr     | 48 +++++++++++++++----
 .../macro/kw-in-item-pos-recovery-151238.rs   | 13 +++++
 .../kw-in-item-pos-recovery-151238.stderr     | 25 ++++++++++
 9 files changed, 102 insertions(+), 22 deletions(-)
 create mode 100644 tests/ui/parser/macro/kw-in-item-pos-recovery-151238.rs
 create mode 100644 tests/ui/parser/macro/kw-in-item-pos-recovery-151238.stderr

diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 2fd1d146b1f6..60e12fa05adf 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -2264,7 +2264,7 @@ impl<'a> Parser<'a> {
             && self.look_ahead(1, |t| *t == token::Comma || *t == token::CloseParen)
         {
             // `fn foo(String s) {}`
-            let ident = self.parse_ident().unwrap();
+            let ident = self.parse_ident_common(true).unwrap();
             let span = pat.span.with_hi(ident.span.hi());
 
             err.span_suggestion(
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index bc76418429aa..c7d09b834540 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -408,12 +408,11 @@ impl<'a> Parser<'a> {
         let insert_span = ident_span.shrink_to_lo();
 
         let ident = if self.token.is_ident()
-            && self.token.is_non_reserved_ident()
             && (!is_const || self.look_ahead(1, |t| *t == token::OpenParen))
             && self.look_ahead(1, |t| {
                 matches!(t.kind, token::Lt | token::OpenBrace | token::OpenParen)
             }) {
-            self.parse_ident().unwrap()
+            self.parse_ident_common(true).unwrap()
         } else {
             return Ok(());
         };
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index d6e99bc540f7..c39e8351f0bd 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -469,7 +469,7 @@ impl<'a> Parser<'a> {
         self.parse_ident_common(self.may_recover())
     }
 
-    fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, Ident> {
+    pub(crate) fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, Ident> {
         let (ident, is_raw) = self.ident_or_err(recover)?;
 
         if is_raw == IdentIsRaw::No && ident.is_reserved() {
diff --git a/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.rs b/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.rs
index 58bb62bc4bf8..8619856cad67 100644
--- a/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.rs
+++ b/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.rs
@@ -6,6 +6,7 @@ macro_rules! m {
 }
 
 m!(const Self());
-//~^ ERROR expected one of `!` or `::`, found `(`
+//~^ ERROR expected identifier, found keyword `Self`
+//~^^ ERROR missing `fn` or `struct` for function or struct definition
 
 fn main() {}
diff --git a/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.stderr b/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.stderr
index f9b73109dbb4..d412c59c4372 100644
--- a/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.stderr
+++ b/tests/ui/parser/macro/kw-in-const-item-pos-recovery-149692.stderr
@@ -1,11 +1,22 @@
-error: expected one of `!` or `::`, found `(`
-  --> $DIR/kw-in-const-item-pos-recovery-149692.rs:8:14
+error: expected identifier, found keyword `Self`
+  --> $DIR/kw-in-const-item-pos-recovery-149692.rs:8:10
+   |
+LL | m!(const Self());
+   |          ^^^^ expected identifier, found keyword
+
+error: missing `fn` or `struct` for function or struct definition
+  --> $DIR/kw-in-const-item-pos-recovery-149692.rs:8:10
    |
 LL |     (const $id:item()) => {}
    |            -------- while parsing argument for this `item` macro fragment
 ...
 LL | m!(const Self());
-   |              ^ expected one of `!` or `::`
+   |          ^^^^
+   |
+help: if you meant to call a macro, try
+   |
+LL | m!(const Self!());
+   |              +
 
-error: aborting due to 1 previous error
+error: aborting due to 2 previous errors
 
diff --git a/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.rs b/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.rs
index 223864e33296..77372ced09c4 100644
--- a/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.rs
+++ b/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.rs
@@ -8,12 +8,15 @@ macro_rules! m {
 }
 
 m!(Self());
-//~^ ERROR expected one of `!` or `::`, found `(`
+//~^ ERROR expected identifier, found keyword `Self`
+//~^^ ERROR missing `fn` or `struct` for function or struct definition
 
 m!(Self{});
-//~^ ERROR expected one of `!` or `::`, found `{`
+//~^ ERROR expected identifier, found keyword `Self`
+//~^^ ERROR missing `enum` or `struct` for enum or struct definition
 
 m!(crate());
-//~^ ERROR expected one of `!` or `::`, found `(`
+//~^ ERROR expected identifier, found keyword `crate`
+//~^^ ERROR missing `fn` or `struct` for function or struct definition
 
 fn main() {}
diff --git a/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.stderr b/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.stderr
index a65214b0d1f9..39f3e2d3a9ea 100644
--- a/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.stderr
+++ b/tests/ui/parser/macro/kw-in-item-pos-recovery-149692.stderr
@@ -1,29 +1,57 @@
-error: expected one of `!` or `::`, found `(`
-  --> $DIR/kw-in-item-pos-recovery-149692.rs:10:8
+error: expected identifier, found keyword `Self`
+  --> $DIR/kw-in-item-pos-recovery-149692.rs:10:4
+   |
+LL | m!(Self());
+   |    ^^^^ expected identifier, found keyword
+
+error: missing `fn` or `struct` for function or struct definition
+  --> $DIR/kw-in-item-pos-recovery-149692.rs:10:4
    |
 LL |     ($id:item()) => {}
    |      -------- while parsing argument for this `item` macro fragment
 ...
 LL | m!(Self());
-   |        ^ expected one of `!` or `::`
+   |    ^^^^
+   |
+help: if you meant to call a macro, try
+   |
+LL | m!(Self!());
+   |        +
 
-error: expected one of `!` or `::`, found `{`
-  --> $DIR/kw-in-item-pos-recovery-149692.rs:13:8
+error: expected identifier, found keyword `Self`
+  --> $DIR/kw-in-item-pos-recovery-149692.rs:14:4
+   |
+LL | m!(Self{});
+   |    ^^^^ expected identifier, found keyword
+
+error: missing `enum` or `struct` for enum or struct definition
+  --> $DIR/kw-in-item-pos-recovery-149692.rs:14:4
    |
 LL |     ($id:item()) => {}
    |      -------- while parsing argument for this `item` macro fragment
 ...
 LL | m!(Self{});
-   |        ^ expected one of `!` or `::`
+   |    ^^^^
 
-error: expected one of `!` or `::`, found `(`
-  --> $DIR/kw-in-item-pos-recovery-149692.rs:16:9
+error: expected identifier, found keyword `crate`
+  --> $DIR/kw-in-item-pos-recovery-149692.rs:18:4
+   |
+LL | m!(crate());
+   |    ^^^^^ expected identifier, found keyword
+
+error: missing `fn` or `struct` for function or struct definition
+  --> $DIR/kw-in-item-pos-recovery-149692.rs:18:4
    |
 LL |     ($id:item()) => {}
    |      -------- while parsing argument for this `item` macro fragment
 ...
 LL | m!(crate());
-   |         ^ expected one of `!` or `::`
+   |    ^^^^^
+   |
+help: if you meant to call a macro, try
+   |
+LL | m!(crate!());
+   |         +
 
-error: aborting due to 3 previous errors
+error: aborting due to 6 previous errors
 
diff --git a/tests/ui/parser/macro/kw-in-item-pos-recovery-151238.rs b/tests/ui/parser/macro/kw-in-item-pos-recovery-151238.rs
new file mode 100644
index 000000000000..bd1785ba5b3c
--- /dev/null
+++ b/tests/ui/parser/macro/kw-in-item-pos-recovery-151238.rs
@@ -0,0 +1,13 @@
+//@ edition: 2021
+
+macro_rules! x {
+    ($ty : item) => {};
+}
+x! {
+  trait MyTrait { fn bar(c self) }
+  //~^ ERROR expected identifier, found keyword `self`
+  //~^^ ERROR expected one of `:`, `@`, or `|`, found keyword `self`
+  //~^^^ ERROR expected one of `->`, `;`, `where`, or `{`, found `}`
+}
+
+fn main() {}
diff --git a/tests/ui/parser/macro/kw-in-item-pos-recovery-151238.stderr b/tests/ui/parser/macro/kw-in-item-pos-recovery-151238.stderr
new file mode 100644
index 000000000000..81151edaf0c0
--- /dev/null
+++ b/tests/ui/parser/macro/kw-in-item-pos-recovery-151238.stderr
@@ -0,0 +1,25 @@
+error: expected identifier, found keyword `self`
+  --> $DIR/kw-in-item-pos-recovery-151238.rs:7:28
+   |
+LL |   trait MyTrait { fn bar(c self) }
+   |                            ^^^^ expected identifier, found keyword
+
+error: expected one of `:`, `@`, or `|`, found keyword `self`
+  --> $DIR/kw-in-item-pos-recovery-151238.rs:7:28
+   |
+LL |   trait MyTrait { fn bar(c self) }
+   |                          --^^^^
+   |                          | |
+   |                          | expected one of `:`, `@`, or `|`
+   |                          help: declare the type after the parameter binding: `: `
+
+error: expected one of `->`, `;`, `where`, or `{`, found `}`
+  --> $DIR/kw-in-item-pos-recovery-151238.rs:7:34
+   |
+LL |   trait MyTrait { fn bar(c self) }
+   |                      ---         ^ expected one of `->`, `;`, `where`, or `{`
+   |                      |
+   |                      while parsing this `fn`
+
+error: aborting due to 3 previous errors
+

From ebcbff2a2e79d0c36442bb19a9eb154dc202f2d5 Mon Sep 17 00:00:00 2001
From: Chayim Refael Friedman 
Date: Mon, 19 Jan 2026 04:50:23 +0200
Subject: [PATCH 0369/1843] Do not mix the order of builtin/regular derives in
 "Expand macro recursively"

---
 .../rust-analyzer/crates/hir/src/semantics.rs   |  7 +++++--
 .../crates/ide/src/expand_macro.rs              | 17 ++++++++++++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir/src/semantics.rs b/src/tools/rust-analyzer/crates/hir/src/semantics.rs
index e55b693ef018..98f5739600f3 100644
--- a/src/tools/rust-analyzer/crates/hir/src/semantics.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/semantics.rs
@@ -641,11 +641,14 @@ impl<'db> SemanticsImpl<'db> {
         })
     }
 
-    pub fn expand_derive_macro(&self, attr: &ast::Attr) -> Option>> {
+    pub fn expand_derive_macro(
+        &self,
+        attr: &ast::Attr,
+    ) -> Option>>> {
         let res: Vec<_> = self
             .derive_macro_calls(attr)?
             .into_iter()
-            .flat_map(|call| {
+            .map(|call| {
                 let file_id = call?.left()?;
                 let ExpandResult { value, err } = self.db.parse_macro_expansion(file_id);
                 let root_node = value.0.syntax_node();
diff --git a/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs b/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs
index ba8b3aa9cafe..44285d9315af 100644
--- a/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/expand_macro.rs
@@ -63,7 +63,7 @@ pub(crate) fn expand_macro(db: &RootDatabase, position: FilePosition) -> Option<
             .take_while(|it| it != &token)
             .filter(|it| it.kind() == T![,])
             .count();
-        let ExpandResult { err, value: expansion } = expansions.get(idx)?.clone();
+        let ExpandResult { err, value: expansion } = expansions.get(idx)?.clone()?;
         let expansion_file_id = sema.hir_file_for(&expansion).macro_file()?;
         let expansion_span_map = db.expansion_span_map(expansion_file_id);
         let mut expansion = format(
@@ -848,4 +848,19 @@ struct S {
                 u32"#]],
         );
     }
+
+    #[test]
+    fn regression_21489() {
+        check(
+            r#"
+//- proc_macros: derive_identity
+//- minicore: derive, fmt
+#[derive(Debug, proc_macros::DeriveIdentity$0)]
+struct Foo;
+        "#,
+            expect![[r#"
+                proc_macros::DeriveIdentity
+                struct Foo;"#]],
+        );
+    }
 }

From e519c686cfe9220609929e014642bf99eebb48c4 Mon Sep 17 00:00:00 2001
From: yukang 
Date: Mon, 19 Jan 2026 11:15:20 +0800
Subject: [PATCH 0370/1843] Deduplicate diagnostics for const trait supertraits

---
 .../traits/fulfillment_errors.rs              | 31 +++++++++++++++++++
 tests/ui/consts/issue-94675.rs                |  1 -
 tests/ui/consts/issue-94675.stderr            | 13 +-------
 tests/ui/traits/const-traits/call.rs          |  1 -
 tests/ui/traits/const-traits/call.stderr      | 11 +------
 ...closure-const_trait_impl-ice-113381.stderr | 11 +------
 ...-const-op-const-closure-non-const-outer.rs |  3 +-
 ...st-op-const-closure-non-const-outer.stderr | 15 ++-------
 8 files changed, 39 insertions(+), 47 deletions(-)

diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
index 00d06779e652..6872d038fb7f 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
@@ -1442,6 +1442,31 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         self.can_eq(param_env, goal.trait_ref, trait_assumption.trait_ref)
     }
 
+    fn can_match_host_effect(
+        &self,
+        param_env: ty::ParamEnv<'tcx>,
+        goal: ty::HostEffectPredicate<'tcx>,
+        assumption: ty::Binder<'tcx, ty::HostEffectPredicate<'tcx>>,
+    ) -> bool {
+        let assumption = self.instantiate_binder_with_fresh_vars(
+            DUMMY_SP,
+            infer::BoundRegionConversionTime::HigherRankedType,
+            assumption,
+        );
+
+        assumption.constness.satisfies(goal.constness)
+            && self.can_eq(param_env, goal.trait_ref, assumption.trait_ref)
+    }
+
+    fn as_host_effect_clause(
+        predicate: ty::Predicate<'tcx>,
+    ) -> Option>> {
+        predicate.as_clause().and_then(|clause| match clause.kind().skip_binder() {
+            ty::ClauseKind::HostEffect(pred) => Some(clause.kind().rebind(pred)),
+            _ => None,
+        })
+    }
+
     fn can_match_projection(
         &self,
         param_env: ty::ParamEnv<'tcx>,
@@ -1484,6 +1509,12 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     .filter_map(|implied| implied.as_trait_clause())
                     .any(|implied| self.can_match_trait(param_env, error, implied))
             })
+        } else if let Some(error) = Self::as_host_effect_clause(error.predicate) {
+            self.enter_forall(error, |error| {
+                elaborate(self.tcx, std::iter::once(cond.predicate))
+                    .filter_map(Self::as_host_effect_clause)
+                    .any(|implied| self.can_match_host_effect(param_env, error, implied))
+            })
         } else if let Some(error) = error.predicate.as_projection_clause() {
             self.enter_forall(error, |error| {
                 elaborate(self.tcx, std::iter::once(cond.predicate))
diff --git a/tests/ui/consts/issue-94675.rs b/tests/ui/consts/issue-94675.rs
index f2ddc928d122..0553b676bc3e 100644
--- a/tests/ui/consts/issue-94675.rs
+++ b/tests/ui/consts/issue-94675.rs
@@ -10,7 +10,6 @@ impl<'a> Foo<'a> {
     const fn spam(&mut self, baz: &mut Vec) {
         self.bar[0] = baz.len();
         //~^ ERROR: `Vec: [const] Index<_>` is not satisfied
-        //~| ERROR: `Vec: [const] Index` is not satisfied
         //~| ERROR: `Vec: [const] IndexMut` is not satisfied
     }
 }
diff --git a/tests/ui/consts/issue-94675.stderr b/tests/ui/consts/issue-94675.stderr
index 771f2a14c305..ab7a76a90e02 100644
--- a/tests/ui/consts/issue-94675.stderr
+++ b/tests/ui/consts/issue-94675.stderr
@@ -16,17 +16,6 @@ LL |         self.bar[0] = baz.len();
 note: trait `IndexMut` is implemented but not `const`
   --> $SRC_DIR/alloc/src/vec/mod.rs:LL:COL
 
-error[E0277]: the trait bound `Vec: [const] Index` is not satisfied
-  --> $DIR/issue-94675.rs:11:9
-   |
-LL |         self.bar[0] = baz.len();
-   |         ^^^^^^^^^^^
-   |
-note: trait `Index` is implemented but not `const`
-  --> $SRC_DIR/alloc/src/vec/mod.rs:LL:COL
-note: required by a bound in `std::ops::IndexMut::index_mut`
-  --> $SRC_DIR/core/src/ops/index.rs:LL:COL
-
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/traits/const-traits/call.rs b/tests/ui/traits/const-traits/call.rs
index c36adc4248f0..360c08e1b7fe 100644
--- a/tests/ui/traits/const-traits/call.rs
+++ b/tests/ui/traits/const-traits/call.rs
@@ -6,7 +6,6 @@
 const _: () = {
     assert!((const || true)());
     //~^ ERROR }: [const] Fn()` is not satisfied
-    //~| ERROR }: [const] FnMut()` is not satisfied
 };
 
 fn main() {}
diff --git a/tests/ui/traits/const-traits/call.stderr b/tests/ui/traits/const-traits/call.stderr
index b688746e2506..8e32cab6dfcf 100644
--- a/tests/ui/traits/const-traits/call.stderr
+++ b/tests/ui/traits/const-traits/call.stderr
@@ -4,15 +4,6 @@ error[E0277]: the trait bound `{closure@$DIR/call.rs:7:14: 7:22}: [const] Fn()`
 LL |     assert!((const || true)());
    |             ^^^^^^^^^^^^^^^^^
 
-error[E0277]: the trait bound `{closure@$DIR/call.rs:7:14: 7:22}: [const] FnMut()` is not satisfied
-  --> $DIR/call.rs:7:13
-   |
-LL |     assert!((const || true)());
-   |             ^^^^^^^^^^^^^^^^^
-   |
-note: required by a bound in `call`
-  --> $SRC_DIR/core/src/ops/function.rs:LL:COL
-
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/traits/const-traits/const_closure-const_trait_impl-ice-113381.stderr b/tests/ui/traits/const-traits/const_closure-const_trait_impl-ice-113381.stderr
index 90e87c724f54..dab3f14161fa 100644
--- a/tests/ui/traits/const-traits/const_closure-const_trait_impl-ice-113381.stderr
+++ b/tests/ui/traits/const-traits/const_closure-const_trait_impl-ice-113381.stderr
@@ -4,15 +4,6 @@ error[E0277]: the trait bound `{closure@$DIR/const_closure-const_trait_impl-ice-
 LL |     (const || (()).foo())();
    |     ^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0277]: the trait bound `{closure@$DIR/const_closure-const_trait_impl-ice-113381.rs:15:6: 15:14}: [const] FnMut()` is not satisfied
-  --> $DIR/const_closure-const_trait_impl-ice-113381.rs:15:5
-   |
-LL |     (const || (()).foo())();
-   |     ^^^^^^^^^^^^^^^^^^^^^^^
-   |
-note: required by a bound in `call`
-  --> $SRC_DIR/core/src/ops/function.rs:LL:COL
-
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.rs b/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.rs
index ee4ff02f4c7c..de5bedf0ace7 100644
--- a/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.rs
+++ b/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.rs
@@ -10,7 +10,8 @@ impl Foo for () {
 }
 
 fn main() {
+    // #150052 deduplicate diagnostics for const trait supertraits
+    // so we only get one error here
     (const || { (()).foo() })();
     //~^ ERROR: }: [const] Fn()` is not satisfied
-    //~| ERROR: }: [const] FnMut()` is not satisfied
 }
diff --git a/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.stderr b/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.stderr
index 69d289537da1..efbedca1c7e7 100644
--- a/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.stderr
+++ b/tests/ui/traits/const-traits/non-const-op-const-closure-non-const-outer.stderr
@@ -1,18 +1,9 @@
-error[E0277]: the trait bound `{closure@$DIR/non-const-op-const-closure-non-const-outer.rs:13:6: 13:14}: [const] Fn()` is not satisfied
-  --> $DIR/non-const-op-const-closure-non-const-outer.rs:13:5
+error[E0277]: the trait bound `{closure@$DIR/non-const-op-const-closure-non-const-outer.rs:15:6: 15:14}: [const] Fn()` is not satisfied
+  --> $DIR/non-const-op-const-closure-non-const-outer.rs:15:5
    |
 LL |     (const || { (()).foo() })();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0277]: the trait bound `{closure@$DIR/non-const-op-const-closure-non-const-outer.rs:13:6: 13:14}: [const] FnMut()` is not satisfied
-  --> $DIR/non-const-op-const-closure-non-const-outer.rs:13:5
-   |
-LL |     (const || { (()).foo() })();
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-note: required by a bound in `call`
-  --> $SRC_DIR/core/src/ops/function.rs:LL:COL
-
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0277`.

From 4dad9b90819bac5aeb50182d77ee0a90c5a196d5 Mon Sep 17 00:00:00 2001
From: Chayim Refael Friedman 
Date: Mon, 19 Jan 2026 05:21:54 +0200
Subject: [PATCH 0371/1843] Insert type vars and normalize for the type of a
 used `static`

They have their own special path, so they slipped through.
---
 .../crates/hir-ty/src/infer/path.rs           |  1 +
 .../crates/hir-ty/src/tests/simple.rs         | 43 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/infer/path.rs b/src/tools/rust-analyzer/crates/hir-ty/src/infer/path.rs
index b11650bbcd9a..ef1a610a323d 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/infer/path.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/infer/path.rs
@@ -93,6 +93,7 @@ impl<'db> InferenceContext<'_, 'db> {
         if let GenericDefId::StaticId(_) = generic_def {
             // `Static` is the kind of item that can never be generic currently. We can just skip the binders to get its type.
             let ty = self.db.value_ty(value_def)?.skip_binder();
+            let ty = self.process_remote_user_written_ty(ty);
             return Some(ValuePathResolution::NonGeneric(ty));
         };
 
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
index 28759bcbae61..d2a4149bc630 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
@@ -3997,3 +3997,46 @@ extern "C" fn foo() -> ! {
     "#,
     );
 }
+
+#[test]
+fn regression_21478() {
+    check_infer(
+        r#"
+//- minicore: unsize, coerce_unsized
+struct LazyLock(T);
+
+impl LazyLock {
+    const fn new() -> Self {
+        loop {}
+    }
+
+    fn force(this: &Self) -> &T {
+        loop {}
+    }
+}
+
+static VALUES_LAZY_LOCK: LazyLock<[u32; { 0 }]> = LazyLock::new();
+
+fn foo() {
+    let _ = LazyLock::force(&VALUES_LAZY_LOCK);
+}
+    "#,
+        expect![[r#"
+            73..96 '{     ...     }': LazyLock
+            83..90 'loop {}': !
+            88..90 '{}': ()
+            111..115 'this': &'? LazyLock
+            130..153 '{     ...     }': &'? T
+            140..147 'loop {}': !
+            145..147 '{}': ()
+            207..220 'LazyLock::new': fn new<[u32; _]>() -> LazyLock<[u32; _]>
+            207..222 'LazyLock::new()': LazyLock<[u32; _]>
+            234..285 '{     ...CK); }': ()
+            244..245 '_': &'? [u32; _]
+            248..263 'LazyLock::force': fn force<[u32; _]>(&'? LazyLock<[u32; _]>) -> &'? [u32; _]
+            248..282 'LazyLo..._LOCK)': &'? [u32; _]
+            264..281 '&VALUE...Y_LOCK': &'? LazyLock<[u32; _]>
+            265..281 'VALUES...Y_LOCK': LazyLock<[u32; _]>
+        "#]],
+    );
+}

From 1285b1b13c28e5e11637884e741b6b7e8f36efc8 Mon Sep 17 00:00:00 2001
From: Chayim Refael Friedman 
Date: Mon, 19 Jan 2026 07:14:48 +0200
Subject: [PATCH 0372/1843] Ensure correct capturing of async fn params even
 when they use weird patterns

rustc does the same.
---
 .../crates/hir-def/src/expr_store/lower.rs    | 66 ++++++++++++++-----
 .../hir-def/src/expr_store/tests/body.rs      | 18 +++++
 .../crates/hir-ty/src/tests/regression.rs     |  1 -
 .../crates/hir-ty/src/tests/simple.rs         |  1 -
 .../crates/hir-ty/src/tests/traits.rs         |  3 -
 5 files changed, 68 insertions(+), 21 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-def/src/expr_store/lower.rs b/src/tools/rust-analyzer/crates/hir-def/src/expr_store/lower.rs
index 4ae4271b92f5..79222615929f 100644
--- a/src/tools/rust-analyzer/crates/hir-def/src/expr_store/lower.rs
+++ b/src/tools/rust-analyzer/crates/hir-def/src/expr_store/lower.rs
@@ -150,6 +150,7 @@ pub(super) fn lower_body(
     };
 
     let body_expr = collector.collect(
+        &mut params,
         body,
         if is_async_fn {
             Awaitable::Yes
@@ -903,24 +904,57 @@ impl<'db> ExprCollector<'db> {
         })
     }
 
-    fn collect(&mut self, expr: Option, awaitable: Awaitable) -> ExprId {
+    /// An `async fn` needs to capture all parameters in the generated `async` block, even if they have
+    /// non-captured patterns such as wildcards (to ensure consistent drop order).
+    fn lower_async_fn(&mut self, params: &mut Vec, body: ExprId) -> ExprId {
+        let mut statements = Vec::new();
+        for param in params {
+            let name = match self.store.pats[*param] {
+                Pat::Bind { id, .. }
+                    if matches!(
+                        self.store.bindings[id].mode,
+                        BindingAnnotation::Unannotated | BindingAnnotation::Mutable
+                    ) =>
+                {
+                    // If this is a direct binding, we can leave it as-is, as it'll always be captured anyway.
+                    continue;
+                }
+                Pat::Bind { id, .. } => {
+                    // If this is a `ref` binding, we can't leave it as is but we can at least reuse the name, for better display.
+                    self.store.bindings[id].name.clone()
+                }
+                _ => self.generate_new_name(),
+            };
+            let binding_id =
+                self.alloc_binding(name.clone(), BindingAnnotation::Mutable, HygieneId::ROOT);
+            let pat_id = self.alloc_pat_desugared(Pat::Bind { id: binding_id, subpat: None });
+            let expr = self.alloc_expr_desugared(Expr::Path(name.into()));
+            statements.push(Statement::Let {
+                pat: *param,
+                type_ref: None,
+                initializer: Some(expr),
+                else_branch: None,
+            });
+            *param = pat_id;
+        }
+
+        self.alloc_expr_desugared(Expr::Async {
+            id: None,
+            statements: statements.into_boxed_slice(),
+            tail: Some(body),
+        })
+    }
+
+    fn collect(
+        &mut self,
+        params: &mut Vec,
+        expr: Option,
+        awaitable: Awaitable,
+    ) -> ExprId {
         self.awaitable_context.replace(awaitable);
         self.with_label_rib(RibKind::Closure, |this| {
-            if awaitable == Awaitable::Yes {
-                match expr {
-                    Some(e) => {
-                        let syntax_ptr = AstPtr::new(&e);
-                        let expr = this.collect_expr(e);
-                        this.alloc_expr_desugared_with_ptr(
-                            Expr::Async { id: None, statements: Box::new([]), tail: Some(expr) },
-                            syntax_ptr,
-                        )
-                    }
-                    None => this.missing_expr(),
-                }
-            } else {
-                this.collect_expr_opt(expr)
-            }
+            let body = this.collect_expr_opt(expr);
+            if awaitable == Awaitable::Yes { this.lower_async_fn(params, body) } else { body }
         })
     }
 
diff --git a/src/tools/rust-analyzer/crates/hir-def/src/expr_store/tests/body.rs b/src/tools/rust-analyzer/crates/hir-def/src/expr_store/tests/body.rs
index 504c310684d6..8f857aeeff95 100644
--- a/src/tools/rust-analyzer/crates/hir-def/src/expr_store/tests/body.rs
+++ b/src/tools/rust-analyzer/crates/hir-def/src/expr_store/tests/body.rs
@@ -659,3 +659,21 @@ fn main() {
         }"#]]
     .assert_eq(&body.pretty_print(&db, def, Edition::CURRENT))
 }
+
+#[test]
+fn async_fn_weird_param_patterns() {
+    let (db, body, def) = lower(
+        r#"
+async fn main(&self, param1: i32, ref mut param2: i32, _: i32, param4 @ _: i32, 123: i32) {}
+"#,
+    );
+
+    expect![[r#"
+        fn main(self, param1, mut param2, mut 0, param4 @ _, mut 1) async {
+            let ref mut param2 = param2;
+            let _ = 0;
+            let 123 = 1;
+            {}
+        }"#]]
+    .assert_eq(&body.pretty_print(&db, def, Edition::CURRENT))
+}
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
index a04c46f8eabd..4f1480c39366 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
@@ -2235,7 +2235,6 @@ async fn f() -> Bar {}
 "#,
         expect![[r#"
             64..66 '{}': ()
-            64..66 '{}': impl Future
         "#]],
     );
 }
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
index 28759bcbae61..80e21450c7a9 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/simple.rs
@@ -2139,7 +2139,6 @@ async fn main() {
         "#,
         expect![[r#"
             16..193 '{     ...2 }; }': ()
-            16..193 '{     ...2 }; }': impl Future
             26..27 'x': i32
             30..43 'unsafe { 92 }': i32
             39..41 '92': i32
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs
index b825a0a8f0e5..390553c0d7a9 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/traits.rs
@@ -4869,7 +4869,6 @@ async fn baz i32>(c: T) {
         expect![[r#"
             37..38 'a': T
             43..83 '{     ...ait; }': ()
-            43..83 '{     ...ait; }': impl Future
             53..57 'fut1': >::CallRefFuture<'?>
             60..61 'a': T
             60..64 'a(0)': >::CallRefFuture<'?>
@@ -4878,7 +4877,6 @@ async fn baz i32>(c: T) {
             70..80 'fut1.await': i32
             124..129 'mut b': T
             134..174 '{     ...ait; }': ()
-            134..174 '{     ...ait; }': impl Future
             144..148 'fut2': >::CallRefFuture<'?>
             151..152 'b': T
             151..155 'b(0)': >::CallRefFuture<'?>
@@ -4887,7 +4885,6 @@ async fn baz i32>(c: T) {
             161..171 'fut2.await': i32
             216..217 'c': T
             222..262 '{     ...ait; }': ()
-            222..262 '{     ...ait; }': impl Future
             232..236 'fut3': >::CallOnceFuture
             239..240 'c': T
             239..243 'c(0)': >::CallOnceFuture

From 0895c4cbe6ff960b23626a538d1691d2ebf51311 Mon Sep 17 00:00:00 2001
From: Jamie Hill-Daniel 
Date: Mon, 19 Jan 2026 04:48:23 +0000
Subject: [PATCH 0373/1843] ci: Move lockfile updates to a script

---
 .github/workflows/dependencies.yml | 16 +++-------------
 src/tools/update-lockfile.sh       | 18 ++++++++++++++++++
 2 files changed, 21 insertions(+), 13 deletions(-)
 create mode 100755 src/tools/update-lockfile.sh

diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
index 80ffd67e04e1..7c721c7abeaa 100644
--- a/.github/workflows/dependencies.yml
+++ b/.github/workflows/dependencies.yml
@@ -62,19 +62,9 @@ jobs:
           rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN
           rustup default $TOOLCHAIN
 
-      - name: cargo update compiler & tools
-        # Remove first line that always just says "Updating crates.io index"
-        run: |
-          echo -e "\ncompiler & tools dependencies:" >> cargo_update.log
-          cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: cargo update library
-        run: |
-          echo -e "\nlibrary dependencies:" >> cargo_update.log
-          cargo update --manifest-path library/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: cargo update rustbook
-        run: |
-          echo -e "\nrustbook dependencies:" >> cargo_update.log
-          cargo update --manifest-path src/tools/rustbook/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
+      - name: cargo update
+        run: ./src/tools/update-lockfile.sh
+
       - name: upload Cargo.lock artifact for use in PR
         uses: actions/upload-artifact@v4
         with:
diff --git a/src/tools/update-lockfile.sh b/src/tools/update-lockfile.sh
new file mode 100755
index 000000000000..a968d83d8152
--- /dev/null
+++ b/src/tools/update-lockfile.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Updates the workspaces in `.`, `library` and `src/tools/rustbook`
+# Logs are written to `cargo_update.log`
+# Used as part of regular dependency bumps
+
+set -euo pipefail
+
+echo -e "\ncompiler & tools dependencies:" > cargo_update.log
+# Remove first line that always just says "Updating crates.io index"
+cargo update 2>&1 | sed '/crates.io index/d' | \
+    tee -a cargo_update.log
+echo -e "\nlibrary dependencies:" >> cargo_update.log
+cargo update --manifest-path library/Cargo.toml 2>&1 | sed '/crates.io index/d' | \
+    tee -a cargo_update.log
+echo -e "\nrustbook dependencies:" >> cargo_update.log
+cargo update --manifest-path src/tools/rustbook/Cargo.toml 2>&1 | sed '/crates.io index/d' | \
+    tee -a cargo_update.log

From 740eb6b59fa663e9f1b721d7f6995dcec4af4bc9 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Fri, 16 Jan 2026 01:16:57 +0530
Subject: [PATCH 0374/1843] remove non-describing field annotation from
 bidirectional message definition

---
 .../proc-macro-api/src/bidirectional_protocol/msg.rs   | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
index c56ed5191694..2644cd406b63 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
@@ -84,18 +84,14 @@ pub struct ExpandMacroData {
     pub macro_body: FlatTree,
     pub macro_name: String,
     pub attributes: Option,
-    #[serde(skip_serializing_if = "ExpnGlobals::skip_serializing_if")]
     #[serde(default)]
     pub has_global_spans: ExpnGlobals,
-
-    #[serde(skip_serializing_if = "Vec::is_empty")]
     #[serde(default)]
     pub span_data_table: Vec,
 }
 
 #[derive(Clone, Copy, Default, Debug, Serialize, Deserialize)]
 pub struct ExpnGlobals {
-    #[serde(skip_serializing)]
     #[serde(default)]
     pub serialize: bool,
     pub def_site: usize,
@@ -103,10 +99,4 @@ pub struct ExpnGlobals {
     pub mixed_site: usize,
 }
 
-impl ExpnGlobals {
-    fn skip_serializing_if(&self) -> bool {
-        !self.serialize
-    }
-}
-
 impl Message for BidirectionalMessage {}

From ffa2dadf0eeab0a484688373cffe9f1939173084 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 18 Jan 2026 19:34:03 +0530
Subject: [PATCH 0375/1843] remove serialize from Expn Globals

---
 .../crates/proc-macro-api/src/bidirectional_protocol.rs   | 8 +-------
 .../proc-macro-api/src/bidirectional_protocol/msg.rs      | 2 --
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
index b5f43e1d3726..a13bff7d7d02 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol.rs
@@ -23,7 +23,6 @@ use crate::{
     },
     process::ProcMacroServerProcess,
     transport::codec::postcard::PostcardProtocol,
-    version,
 };
 
 pub mod msg;
@@ -159,12 +158,7 @@ pub(crate) fn expand(
             macro_name: proc_macro.name.to_string(),
             attributes: attr
                 .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)),
-            has_global_spans: ExpnGlobals {
-                serialize: version >= version::HAS_GLOBAL_SPANS,
-                def_site,
-                call_site,
-                mixed_site,
-            },
+            has_global_spans: ExpnGlobals { def_site, call_site, mixed_site },
             span_data_table: if process.rust_analyzer_spans() {
                 serialize_span_data_index_map(&span_data_table)
             } else {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
index 2644cd406b63..d030498e59c4 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/bidirectional_protocol/msg.rs
@@ -92,8 +92,6 @@ pub struct ExpandMacroData {
 
 #[derive(Clone, Copy, Default, Debug, Serialize, Deserialize)]
 pub struct ExpnGlobals {
-    #[serde(default)]
-    pub serialize: bool,
     pub def_site: usize,
     pub call_site: usize,
     pub mixed_site: usize,

From 8fd55694389332c648a76c0ad533ba9e01ecc1fe Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 18 Jan 2026 20:23:30 +0530
Subject: [PATCH 0376/1843] add bidirectional flow

---
 .../proc-macro-srv-cli/tests/common/utils.rs  | 570 +++++++++++-------
 1 file changed, 357 insertions(+), 213 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs
index 722e92eec7e5..63b3a74aa4e8 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs
@@ -1,213 +1,357 @@
-use std::{
-    collections::VecDeque,
-    io::{self, BufRead, Read, Write},
-    sync::{Arc, Condvar, Mutex},
-    thread,
-};
-
-use paths::Utf8PathBuf;
-use proc_macro_api::{
-    legacy_protocol::msg::{FlatTree, Message, Request, Response, SpanDataIndexMap},
-    transport::codec::json::JsonProtocol,
-};
-use span::{Edition, EditionedFileId, FileId, Span, SpanAnchor, SyntaxContext, TextRange};
-use tt::{Delimiter, DelimiterKind, TopSubtreeBuilder};
-
-/// Shared state for an in-memory byte channel.
-#[derive(Default)]
-struct ChannelState {
-    buffer: VecDeque,
-    closed: bool,
-}
-
-type InMemoryChannel = Arc<(Mutex, Condvar)>;
-
-/// Writer end of an in-memory channel.
-pub(crate) struct ChannelWriter {
-    state: InMemoryChannel,
-}
-
-impl Write for ChannelWriter {
-    fn write(&mut self, buf: &[u8]) -> io::Result {
-        let (lock, cvar) = &*self.state;
-        let mut state = lock.lock().unwrap();
-        if state.closed {
-            return Err(io::Error::new(io::ErrorKind::BrokenPipe, "channel closed"));
-        }
-        state.buffer.extend(buf);
-        cvar.notify_all();
-        Ok(buf.len())
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-impl Drop for ChannelWriter {
-    fn drop(&mut self) {
-        let (lock, cvar) = &*self.state;
-        let mut state = lock.lock().unwrap();
-        state.closed = true;
-        cvar.notify_all();
-    }
-}
-
-/// Reader end of an in-memory channel.
-pub(crate) struct ChannelReader {
-    state: InMemoryChannel,
-    internal_buf: Vec,
-}
-
-impl Read for ChannelReader {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result {
-        let (lock, cvar) = &*self.state;
-        let mut state = lock.lock().unwrap();
-
-        while state.buffer.is_empty() && !state.closed {
-            state = cvar.wait(state).unwrap();
-        }
-
-        if state.buffer.is_empty() && state.closed {
-            return Ok(0);
-        }
-
-        let to_read = buf.len().min(state.buffer.len());
-        for (dst, src) in buf.iter_mut().zip(state.buffer.drain(..to_read)) {
-            *dst = src;
-        }
-        Ok(to_read)
-    }
-}
-
-impl BufRead for ChannelReader {
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        let (lock, cvar) = &*self.state;
-        let mut state = lock.lock().unwrap();
-
-        while state.buffer.is_empty() && !state.closed {
-            state = cvar.wait(state).unwrap();
-        }
-
-        self.internal_buf.clear();
-        self.internal_buf.extend(&state.buffer);
-        Ok(&self.internal_buf)
-    }
-
-    fn consume(&mut self, amt: usize) {
-        let (lock, _) = &*self.state;
-        let mut state = lock.lock().unwrap();
-        let to_drain = amt.min(state.buffer.len());
-        drop(state.buffer.drain(..to_drain));
-    }
-}
-
-/// Creates a connected pair of channels for bidirectional communication.
-fn create_channel_pair() -> (ChannelWriter, ChannelReader, ChannelWriter, ChannelReader) {
-    // Channel for client -> server communication
-    let client_to_server = Arc::new((
-        Mutex::new(ChannelState { buffer: VecDeque::new(), closed: false }),
-        Condvar::new(),
-    ));
-    let client_writer = ChannelWriter { state: client_to_server.clone() };
-    let server_reader = ChannelReader { state: client_to_server, internal_buf: Vec::new() };
-
-    // Channel for server -> client communication
-    let server_to_client = Arc::new((
-        Mutex::new(ChannelState { buffer: VecDeque::new(), closed: false }),
-        Condvar::new(),
-    ));
-
-    let server_writer = ChannelWriter { state: server_to_client.clone() };
-    let client_reader = ChannelReader { state: server_to_client, internal_buf: Vec::new() };
-
-    (client_writer, client_reader, server_writer, server_reader)
-}
-
-pub(crate) fn proc_macro_test_dylib_path() -> Utf8PathBuf {
-    let path = proc_macro_test::PROC_MACRO_TEST_LOCATION;
-    if path.is_empty() {
-        panic!("proc-macro-test dylib not available (requires nightly toolchain)");
-    }
-    path.into()
-}
-
-/// Runs a test with the server in a background thread.
-pub(crate) fn with_server(test_fn: F) -> R
-where
-    F: FnOnce(&mut dyn Write, &mut dyn BufRead) -> R,
-{
-    let (mut client_writer, mut client_reader, mut server_writer, mut server_reader) =
-        create_channel_pair();
-
-    let server_handle = thread::spawn(move || {
-        proc_macro_srv_cli::main_loop::run(
-            &mut server_reader,
-            &mut server_writer,
-            proc_macro_api::ProtocolFormat::JsonLegacy,
-        )
-    });
-
-    let result = test_fn(&mut client_writer, &mut client_reader);
-
-    // Close the client writer to signal the server to stop
-    drop(client_writer);
-
-    // Wait for server to finish
-    match server_handle.join() {
-        Ok(Ok(())) => {}
-        Ok(Err(e)) => {
-            // IO error from server is expected when client disconnects
-            if matches!(
-                e.kind(),
-                io::ErrorKind::BrokenPipe
-                    | io::ErrorKind::UnexpectedEof
-                    | io::ErrorKind::InvalidData
-            ) {
-                panic!("Server error: {e}");
-            }
-        }
-        Err(e) => std::panic::resume_unwind(e),
-    }
-
-    result
-}
-
-/// Sends a request and reads the response using JSON protocol.
-pub(crate) fn request(
-    writer: &mut dyn Write,
-    reader: &mut dyn BufRead,
-    request: Request,
-) -> Response {
-    request.write::(writer).expect("failed to write request");
-
-    let mut buf = String::new();
-    Response::read::(reader, &mut buf)
-        .expect("failed to read response")
-        .expect("no response received")
-}
-
-/// Creates a simple empty token tree suitable for testing.
-pub(crate) fn create_empty_token_tree(
-    version: u32,
-    span_data_table: &mut SpanDataIndexMap,
-) -> FlatTree {
-    let anchor = SpanAnchor {
-        file_id: EditionedFileId::new(FileId::from_raw(0), Edition::CURRENT),
-        ast_id: span::ROOT_ERASED_FILE_AST_ID,
-    };
-    let span = Span {
-        range: TextRange::empty(0.into()),
-        anchor,
-        ctx: SyntaxContext::root(Edition::CURRENT),
-    };
-
-    let builder = TopSubtreeBuilder::new(Delimiter {
-        open: span,
-        close: span,
-        kind: DelimiterKind::Invisible,
-    });
-    let tt = builder.build();
-
-    FlatTree::from_subtree(tt.view(), version, span_data_table)
-}
+use std::{
+    collections::VecDeque,
+    io::{self, BufRead, Read, Write},
+    sync::{Arc, Condvar, Mutex},
+    thread,
+};
+
+use paths::Utf8PathBuf;
+use proc_macro_api::{
+    ServerError,
+    bidirectional_protocol::msg::{
+        BidirectionalMessage, Request as BiRequest, Response as BiResponse, SubRequest, SubResponse,
+    },
+    legacy_protocol::msg::{FlatTree, Message, Request, Response, SpanDataIndexMap},
+    transport::codec::{json::JsonProtocol, postcard::PostcardProtocol},
+};
+use span::{Edition, EditionedFileId, FileId, Span, SpanAnchor, SyntaxContext, TextRange};
+use tt::{Delimiter, DelimiterKind, TopSubtreeBuilder};
+
+/// Shared state for an in-memory byte channel.
+#[derive(Default)]
+struct ChannelState {
+    buffer: VecDeque,
+    closed: bool,
+}
+
+type InMemoryChannel = Arc<(Mutex, Condvar)>;
+
+/// Writer end of an in-memory channel.
+pub(crate) struct ChannelWriter {
+    state: InMemoryChannel,
+}
+
+impl Write for ChannelWriter {
+    fn write(&mut self, buf: &[u8]) -> io::Result {
+        let (lock, cvar) = &*self.state;
+        let mut state = lock.lock().unwrap();
+        if state.closed {
+            return Err(io::Error::new(io::ErrorKind::BrokenPipe, "channel closed"));
+        }
+        state.buffer.extend(buf);
+        cvar.notify_all();
+        Ok(buf.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+impl Drop for ChannelWriter {
+    fn drop(&mut self) {
+        let (lock, cvar) = &*self.state;
+        let mut state = lock.lock().unwrap();
+        state.closed = true;
+        cvar.notify_all();
+    }
+}
+
+/// Reader end of an in-memory channel.
+pub(crate) struct ChannelReader {
+    state: InMemoryChannel,
+    internal_buf: Vec,
+}
+
+impl Read for ChannelReader {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result {
+        let (lock, cvar) = &*self.state;
+        let mut state = lock.lock().unwrap();
+
+        while state.buffer.is_empty() && !state.closed {
+            state = cvar.wait(state).unwrap();
+        }
+
+        if state.buffer.is_empty() && state.closed {
+            return Ok(0);
+        }
+
+        let to_read = buf.len().min(state.buffer.len());
+        for (dst, src) in buf.iter_mut().zip(state.buffer.drain(..to_read)) {
+            *dst = src;
+        }
+        Ok(to_read)
+    }
+}
+
+impl BufRead for ChannelReader {
+    fn fill_buf(&mut self) -> io::Result<&[u8]> {
+        let (lock, cvar) = &*self.state;
+        let mut state = lock.lock().unwrap();
+
+        while state.buffer.is_empty() && !state.closed {
+            state = cvar.wait(state).unwrap();
+        }
+
+        self.internal_buf.clear();
+        self.internal_buf.extend(&state.buffer);
+        Ok(&self.internal_buf)
+    }
+
+    fn consume(&mut self, amt: usize) {
+        let (lock, _) = &*self.state;
+        let mut state = lock.lock().unwrap();
+        let to_drain = amt.min(state.buffer.len());
+        drop(state.buffer.drain(..to_drain));
+    }
+}
+
+/// Creates a connected pair of channels for bidirectional communication.
+fn create_channel_pair() -> (ChannelWriter, ChannelReader, ChannelWriter, ChannelReader) {
+    // Channel for client -> server communication
+    let client_to_server = Arc::new((
+        Mutex::new(ChannelState { buffer: VecDeque::new(), closed: false }),
+        Condvar::new(),
+    ));
+    let client_writer = ChannelWriter { state: client_to_server.clone() };
+    let server_reader = ChannelReader { state: client_to_server, internal_buf: Vec::new() };
+
+    // Channel for server -> client communication
+    let server_to_client = Arc::new((
+        Mutex::new(ChannelState { buffer: VecDeque::new(), closed: false }),
+        Condvar::new(),
+    ));
+
+    let server_writer = ChannelWriter { state: server_to_client.clone() };
+    let client_reader = ChannelReader { state: server_to_client, internal_buf: Vec::new() };
+
+    (client_writer, client_reader, server_writer, server_reader)
+}
+
+pub(crate) fn proc_macro_test_dylib_path() -> Utf8PathBuf {
+    let path = proc_macro_test::PROC_MACRO_TEST_LOCATION;
+    if path.is_empty() {
+        panic!("proc-macro-test dylib not available (requires nightly toolchain)");
+    }
+    path.into()
+}
+
+/// Creates a simple empty token tree suitable for testing.
+pub(crate) fn create_empty_token_tree(
+    version: u32,
+    span_data_table: &mut SpanDataIndexMap,
+) -> FlatTree {
+    let anchor = SpanAnchor {
+        file_id: EditionedFileId::new(FileId::from_raw(0), Edition::CURRENT),
+        ast_id: span::ROOT_ERASED_FILE_AST_ID,
+    };
+    let span = Span {
+        range: TextRange::empty(0.into()),
+        anchor,
+        ctx: SyntaxContext::root(Edition::CURRENT),
+    };
+
+    let builder = TopSubtreeBuilder::new(Delimiter {
+        open: span,
+        close: span,
+        kind: DelimiterKind::Invisible,
+    });
+    let tt = builder.build();
+
+    FlatTree::from_subtree(tt.view(), version, span_data_table)
+}
+
+pub(crate) fn with_server(format: proc_macro_api::ProtocolFormat, test_fn: F) -> R
+where
+    F: FnOnce(&mut dyn Write, &mut dyn BufRead) -> R,
+{
+    let (mut client_writer, mut client_reader, mut server_writer, mut server_reader) =
+        create_channel_pair();
+
+    let server_handle = thread::spawn(move || {
+        proc_macro_srv_cli::main_loop::run(&mut server_reader, &mut server_writer, format)
+    });
+
+    let result = test_fn(&mut client_writer, &mut client_reader);
+
+    drop(client_writer);
+
+    match server_handle.join() {
+        Ok(Ok(())) => {}
+        Ok(Err(e)) => {
+            if !matches!(
+                e.kind(),
+                io::ErrorKind::BrokenPipe
+                    | io::ErrorKind::UnexpectedEof
+                    | io::ErrorKind::InvalidData
+            ) {
+                panic!("Server error: {e}");
+            }
+        }
+        Err(e) => std::panic::resume_unwind(e),
+    }
+
+    result
+}
+
+trait TestProtocol {
+    type Request;
+    type Response;
+
+    fn send(&self, writer: &mut dyn Write, req: Self::Request);
+    fn drive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> Self::Response;
+}
+
+struct JsonLegacy;
+
+impl TestProtocol for JsonLegacy {
+    type Request = Request;
+    type Response = Response;
+
+    fn send(&self, writer: &mut dyn Write, req: Request) {
+        req.write::(writer).expect("failed to write request");
+    }
+
+    fn drive(&self, reader: &mut dyn BufRead, _writer: &mut dyn Write) -> Response {
+        let mut buf = String::new();
+        Response::read::(reader, &mut buf)
+            .expect("failed to read response")
+            .expect("no response received")
+    }
+}
+
+struct PostcardBidirectional
+where
+    F: Fn(SubRequest) -> Result,
+{
+    callback: F,
+}
+
+impl TestProtocol for PostcardBidirectional
+where
+    F: Fn(SubRequest) -> Result,
+{
+    type Request = BiRequest;
+    type Response = BiResponse;
+
+    fn send(&self, writer: &mut dyn Write, req: BiRequest) {
+        let msg = BidirectionalMessage::Request(req);
+        msg.write::(writer).expect("failed to write request");
+    }
+
+    fn drive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> BiResponse {
+        let mut buf = Vec::new();
+
+        loop {
+            let msg = BidirectionalMessage::read::(reader, &mut buf)
+                .expect("failed to read message")
+                .expect("no message received");
+
+            match msg {
+                BidirectionalMessage::Response(resp) => return resp,
+                BidirectionalMessage::SubRequest(sr) => {
+                    let reply = (self.callback)(sr).expect("subrequest callback failed");
+                    let msg = BidirectionalMessage::SubResponse(reply);
+                    msg.write::(writer).expect("failed to write subresponse");
+                }
+                other => panic!("unexpected message: {other:?}"),
+            }
+        }
+    }
+}
+
+pub(crate) fn request(
+    writer: &mut dyn Write,
+    reader: &mut dyn BufRead,
+    request: impl Into,
+    callback: Option<&dyn Fn(SubRequest) -> Result>,
+) -> AutoResponse {
+    let protocol = match callback {
+        None => AutoProtocol::Legacy(JsonLegacy),
+        Some(cb) => AutoProtocol::Bidirectional(PostcardBidirectional { callback: cb }),
+    };
+
+    protocol.send(writer, request.into());
+    protocol.drive(reader, writer)
+}
+
+enum AutoProtocol
+where
+    F: Fn(SubRequest) -> Result,
+{
+    Legacy(JsonLegacy),
+    Bidirectional(PostcardBidirectional),
+}
+
+impl TestProtocol for AutoProtocol
+where
+    F: Fn(SubRequest) -> Result,
+{
+    type Request = AutoRequest;
+    type Response = AutoResponse;
+
+    fn send(&self, writer: &mut dyn Write, req: AutoRequest) {
+        match (self, req) {
+            (AutoProtocol::Legacy(p), AutoRequest::Legacy(r)) => {
+                p.send(writer, r);
+            }
+            (AutoProtocol::Bidirectional(p), AutoRequest::Bidirectional(r)) => {
+                p.send(writer, r);
+            }
+            (AutoProtocol::Legacy(_), AutoRequest::Bidirectional(_)) => {
+                panic!("bidirectional request used with legacy protocol");
+            }
+            (AutoProtocol::Bidirectional(_), AutoRequest::Legacy(_)) => {
+                panic!("legacy request used with bidirectional protocol");
+            }
+        }
+    }
+
+    fn drive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> AutoResponse {
+        match self {
+            AutoProtocol::Legacy(p) => AutoResponse::Legacy(p.drive(reader, writer)),
+            AutoProtocol::Bidirectional(p) => AutoResponse::Bidirectional(p.drive(reader, writer)),
+        }
+    }
+}
+
+pub(crate) enum AutoRequest {
+    Legacy(Request),
+    Bidirectional(BiRequest),
+}
+
+#[derive(Debug)]
+pub(crate) enum AutoResponse {
+    Legacy(Response),
+    Bidirectional(BiResponse),
+}
+
+impl From for AutoRequest {
+    fn from(req: Request) -> AutoRequest {
+        AutoRequest::Legacy(req)
+    }
+}
+
+impl From for AutoRequest {
+    fn from(req: BiRequest) -> AutoRequest {
+        AutoRequest::Bidirectional(req)
+    }
+}
+
+impl From for Response {
+    fn from(res: AutoResponse) -> Response {
+        match res {
+            AutoResponse::Legacy(res) => res,
+            _ => panic!("Should be legacy response"),
+        }
+    }
+}
+
+impl From for BiResponse {
+    fn from(res: AutoResponse) -> BiResponse {
+        match res {
+            AutoResponse::Bidirectional(res) => res,
+            _ => panic!("Should be bidirectional response"),
+        }
+    }
+}

From ee35fd6cb206e00b0e43b7e676a1aed8eb013684 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 18 Jan 2026 20:23:40 +0530
Subject: [PATCH 0377/1843] add bidirectional test

---
 .../tests/bidirectional_postcard.rs           | 223 ++++++++++++++++++
 1 file changed, 223 insertions(+)
 create mode 100644 src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs

diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs
new file mode 100644
index 000000000000..08e44bad3723
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs
@@ -0,0 +1,223 @@
+#![cfg(feature = "sysroot-abi")]
+
+mod common {
+    pub(crate) mod utils;
+}
+
+use common::utils::{create_empty_token_tree, proc_macro_test_dylib_path, request, with_server};
+use expect_test::expect;
+use proc_macro_api::{
+    ProtocolFormat::BidirectionalPostcardPrototype,
+    bidirectional_protocol::{
+        msg::{ExpandMacro, ExpandMacroData, ExpnGlobals, Request, Response},
+        reject_subrequests,
+    },
+    legacy_protocol::msg::{PanicMessage, ServerConfig, SpanDataIndexMap, SpanMode},
+    version::CURRENT_API_VERSION,
+};
+
+#[test]
+fn test_bidi_version_check_bidirectional() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let response =
+            request(writer, reader, Request::ApiVersionCheck {}, Some(&reject_subrequests)).into();
+
+        match response {
+            Response::ApiVersionCheck(version) => {
+                assert_eq!(version, CURRENT_API_VERSION);
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_bidi_list_macros() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+        let response =
+            request(writer, reader, Request::ListMacros { dylib_path }, Some(&reject_subrequests))
+                .into();
+
+        let Response::ListMacros(Ok(macros)) = response else {
+            panic!("expected successful ListMacros response");
+        };
+
+        let mut macro_list: Vec<_> =
+            macros.iter().map(|(name, kind)| format!("{name} [{kind:?}]")).collect();
+        macro_list.sort();
+        let macro_list_str = macro_list.join("\n");
+
+        expect![[r#"
+            DeriveEmpty [CustomDerive]
+            DeriveError [CustomDerive]
+            DerivePanic [CustomDerive]
+            DeriveReemit [CustomDerive]
+            attr_error [Attr]
+            attr_noop [Attr]
+            attr_panic [Attr]
+            fn_like_clone_tokens [Bang]
+            fn_like_error [Bang]
+            fn_like_mk_idents [Bang]
+            fn_like_mk_literals [Bang]
+            fn_like_noop [Bang]
+            fn_like_panic [Bang]
+            fn_like_span_join [Bang]
+            fn_like_span_line_column [Bang]
+            fn_like_span_ops [Bang]"#]]
+        .assert_eq(¯o_list_str);
+    });
+}
+
+#[test]
+fn test_bidi_list_macros_invalid_path() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let response = request(
+            writer,
+            reader,
+            Request::ListMacros { dylib_path: "/nonexistent/path/to/dylib.so".into() },
+            Some(&reject_subrequests),
+        )
+        .into();
+
+        match response {
+            Response::ListMacros(Err(e)) => assert!(
+                e.starts_with("Cannot create expander for /nonexistent/path/to/dylib.so"),
+                "{e}"
+            ),
+            other => panic!("expected error response, got: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_bidi_set_config() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let config = ServerConfig { span_mode: SpanMode::Id };
+        let response =
+            request(writer, reader, Request::SetConfig(config), Some(&reject_subrequests)).into();
+
+        match response {
+            Response::SetConfig(returned_config) => {
+                assert_eq!(returned_config.span_mode, SpanMode::Id);
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_bidi_set_config_rust_analyzer_mode() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let config = ServerConfig { span_mode: SpanMode::RustAnalyzer };
+        let response =
+            request(writer, reader, Request::SetConfig(config), Some(&reject_subrequests)).into();
+
+        match response {
+            Response::SetConfig(returned_config) => {
+                assert_eq!(returned_config.span_mode, SpanMode::RustAnalyzer);
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_bidi_expand_macro_panic() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+
+        let mut span_data_table = SpanDataIndexMap::default();
+        let macro_body =
+            common::utils::create_empty_token_tree(CURRENT_API_VERSION, &mut span_data_table);
+
+        let request1 = Request::ExpandMacro(Box::new(ExpandMacro {
+            lib: dylib_path,
+            env: vec![],
+            current_dir: None,
+            data: ExpandMacroData {
+                macro_body,
+                macro_name: "fn_like_panic".to_owned(),
+                attributes: None,
+                has_global_spans: ExpnGlobals { def_site: 0, call_site: 0, mixed_site: 0 },
+                span_data_table: vec![],
+            },
+        }));
+
+        let response = request(writer, reader, request1, Some(&reject_subrequests)).into();
+
+        match response {
+            Response::ExpandMacro(Err(PanicMessage(msg))) => {
+                assert!(msg.contains("fn_like_panic"), "panic message should mention macro name");
+            }
+            other => panic!("expected panic response, got: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_bidi_basic_call_flow() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+
+        let response1 =
+            request(writer, reader, Request::ApiVersionCheck {}, Some(&reject_subrequests)).into();
+        assert!(matches!(response1, Response::ApiVersionCheck(_)));
+
+        let response2 = request(
+            writer,
+            reader,
+            Request::SetConfig(ServerConfig { span_mode: SpanMode::Id }),
+            Some(&reject_subrequests),
+        )
+        .into();
+        assert!(matches!(response2, Response::SetConfig(_)));
+
+        let response3 = request(
+            writer,
+            reader,
+            Request::ListMacros { dylib_path: dylib_path.clone() },
+            Some(&reject_subrequests),
+        )
+        .into();
+        assert!(matches!(response3, Response::ListMacros(Ok(_))));
+    });
+}
+
+#[test]
+fn test_bidi_expand_nonexistent_macro() {
+    with_server(BidirectionalPostcardPrototype, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+
+        let version_response =
+            request(writer, reader, Request::ApiVersionCheck {}, Some(&reject_subrequests)).into();
+        let Response::ApiVersionCheck(version) = version_response else {
+            panic!("expected version check response");
+        };
+
+        let mut span_data_table = SpanDataIndexMap::default();
+        let macro_body = create_empty_token_tree(version, &mut span_data_table);
+
+        let expand_request = Request::ExpandMacro(Box::new(ExpandMacro {
+            lib: dylib_path,
+            env: vec![],
+            current_dir: None,
+            data: ExpandMacroData {
+                macro_body,
+                macro_name: "NonexistentMacro".to_owned(),
+                attributes: None,
+                has_global_spans: ExpnGlobals { def_site: 0, call_site: 0, mixed_site: 0 },
+                span_data_table: vec![],
+            },
+        }));
+
+        let response = request(writer, reader, expand_request, Some(&reject_subrequests)).into();
+
+        match response {
+            Response::ExpandMacro(Err(PanicMessage(msg))) => {
+                expect!["proc-macro `NonexistentMacro` is missing"].assert_eq(&msg)
+            }
+            other => panic!("expected error for nonexistent macro, got: {other:?}"),
+        }
+    });
+}

From a151d7dc5617ecd8c0cf8fbb688654c66e9776b6 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 18 Jan 2026 20:23:49 +0530
Subject: [PATCH 0378/1843] adapt json test

---
 .../proc-macro-srv-cli/tests/legacy_json.rs   | 457 +++++++++---------
 1 file changed, 233 insertions(+), 224 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs
index 1fa886219a8a..8daee7b2bceb 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs
@@ -1,224 +1,233 @@
-//! Integration tests for the proc-macro-srv-cli main loop.
-//!
-//! These tests exercise the full client-server RPC procedure using in-memory
-//! channels without needing to spawn the actual server and client processes.
-
-#![cfg(feature = "sysroot-abi")]
-
-mod common {
-    pub(crate) mod utils;
-}
-
-use common::utils::{create_empty_token_tree, proc_macro_test_dylib_path, request, with_server};
-use expect_test::expect;
-use proc_macro_api::{
-    legacy_protocol::msg::{
-        ExpandMacro, ExpandMacroData, ExpnGlobals, PanicMessage, Request, Response, ServerConfig,
-        SpanDataIndexMap, SpanMode,
-    },
-    version::CURRENT_API_VERSION,
-};
-
-#[test]
-fn test_version_check() {
-    with_server(|writer, reader| {
-        let response = request(writer, reader, Request::ApiVersionCheck {});
-
-        match response {
-            Response::ApiVersionCheck(version) => {
-                assert_eq!(version, CURRENT_API_VERSION);
-            }
-            other => panic!("unexpected response: {other:?}"),
-        }
-    });
-}
-
-#[test]
-fn test_list_macros() {
-    with_server(|writer, reader| {
-        let dylib_path = proc_macro_test_dylib_path();
-        let response = request(writer, reader, Request::ListMacros { dylib_path });
-
-        let Response::ListMacros(Ok(macros)) = response else {
-            panic!("expected successful ListMacros response");
-        };
-
-        let mut macro_list: Vec<_> =
-            macros.iter().map(|(name, kind)| format!("{name} [{kind:?}]")).collect();
-        macro_list.sort();
-        let macro_list_str = macro_list.join("\n");
-
-        expect![[r#"
-            DeriveEmpty [CustomDerive]
-            DeriveError [CustomDerive]
-            DerivePanic [CustomDerive]
-            DeriveReemit [CustomDerive]
-            attr_error [Attr]
-            attr_noop [Attr]
-            attr_panic [Attr]
-            fn_like_clone_tokens [Bang]
-            fn_like_error [Bang]
-            fn_like_mk_idents [Bang]
-            fn_like_mk_literals [Bang]
-            fn_like_noop [Bang]
-            fn_like_panic [Bang]
-            fn_like_span_join [Bang]
-            fn_like_span_line_column [Bang]
-            fn_like_span_ops [Bang]"#]]
-        .assert_eq(¯o_list_str);
-    });
-}
-
-#[test]
-fn test_list_macros_invalid_path() {
-    with_server(|writer, reader| {
-        let response = request(
-            writer,
-            reader,
-            Request::ListMacros { dylib_path: "/nonexistent/path/to/dylib.so".into() },
-        );
-
-        match response {
-            Response::ListMacros(Err(e)) => assert!(
-                e.starts_with("Cannot create expander for /nonexistent/path/to/dylib.so"),
-                "{e}"
-            ),
-            other => panic!("expected error response, got: {other:?}"),
-        }
-    });
-}
-
-#[test]
-fn test_set_config() {
-    with_server(|writer, reader| {
-        let config = ServerConfig { span_mode: SpanMode::Id };
-        let response = request(writer, reader, Request::SetConfig(config));
-
-        match response {
-            Response::SetConfig(returned_config) => {
-                assert_eq!(returned_config.span_mode, SpanMode::Id);
-            }
-            other => panic!("unexpected response: {other:?}"),
-        }
-    });
-}
-
-#[test]
-fn test_set_config_rust_analyzer_mode() {
-    with_server(|writer, reader| {
-        let config = ServerConfig { span_mode: SpanMode::RustAnalyzer };
-        let response = request(writer, reader, Request::SetConfig(config));
-
-        match response {
-            Response::SetConfig(returned_config) => {
-                assert_eq!(returned_config.span_mode, SpanMode::RustAnalyzer);
-            }
-            other => panic!("unexpected response: {other:?}"),
-        }
-    });
-}
-
-#[test]
-fn test_expand_macro_panic() {
-    with_server(|writer, reader| {
-        let dylib_path = proc_macro_test_dylib_path();
-
-        let version_response = request(writer, reader, Request::ApiVersionCheck {});
-        let Response::ApiVersionCheck(version) = version_response else {
-            panic!("expected version check response");
-        };
-
-        let mut span_data_table = SpanDataIndexMap::default();
-        let macro_body = create_empty_token_tree(version, &mut span_data_table);
-
-        let expand_request = Request::ExpandMacro(Box::new(ExpandMacro {
-            lib: dylib_path,
-            env: vec![],
-            current_dir: None,
-            data: ExpandMacroData {
-                macro_body,
-                macro_name: "fn_like_panic".to_owned(),
-                attributes: None,
-                has_global_spans: ExpnGlobals {
-                    serialize: version >= 3,
-                    def_site: 0,
-                    call_site: 0,
-                    mixed_site: 0,
-                },
-                span_data_table: vec![],
-            },
-        }));
-
-        let response = request(writer, reader, expand_request);
-
-        match response {
-            Response::ExpandMacro(Err(PanicMessage(msg))) => {
-                assert!(msg.contains("fn_like_panic"), "panic message should mention the macro");
-            }
-            Response::ExpandMacro(Ok(_)) => {
-                panic!("expected panic, but macro succeeded");
-            }
-            other => panic!("unexpected response: {other:?}"),
-        }
-    });
-}
-
-#[test]
-fn test_basic_call_flow() {
-    with_server(|writer, reader| {
-        let dylib_path = proc_macro_test_dylib_path();
-
-        let response1 = request(writer, reader, Request::ApiVersionCheck {});
-        assert!(matches!(response1, Response::ApiVersionCheck(_)));
-
-        let response2 =
-            request(writer, reader, Request::SetConfig(ServerConfig { span_mode: SpanMode::Id }));
-        assert!(matches!(response2, Response::SetConfig(_)));
-
-        let response3 =
-            request(writer, reader, Request::ListMacros { dylib_path: dylib_path.clone() });
-        assert!(matches!(response3, Response::ListMacros(Ok(_))));
-    });
-}
-
-#[test]
-fn test_expand_nonexistent_macro() {
-    with_server(|writer, reader| {
-        let dylib_path = proc_macro_test_dylib_path();
-
-        let version_response = request(writer, reader, Request::ApiVersionCheck {});
-        let Response::ApiVersionCheck(version) = version_response else {
-            panic!("expected version check response");
-        };
-
-        let mut span_data_table = SpanDataIndexMap::default();
-        let macro_body = create_empty_token_tree(version, &mut span_data_table);
-
-        let expand_request = Request::ExpandMacro(Box::new(ExpandMacro {
-            lib: dylib_path,
-            env: vec![],
-            current_dir: None,
-            data: ExpandMacroData {
-                macro_body,
-                macro_name: "NonexistentMacro".to_owned(),
-                attributes: None,
-                has_global_spans: ExpnGlobals {
-                    serialize: version >= 3,
-                    def_site: 0,
-                    call_site: 0,
-                    mixed_site: 0,
-                },
-                span_data_table: vec![],
-            },
-        }));
-
-        let response = request(writer, reader, expand_request);
-
-        match response {
-            Response::ExpandMacro(Err(PanicMessage(msg))) => {
-                expect!["proc-macro `NonexistentMacro` is missing"].assert_eq(&msg)
-            }
-            other => panic!("expected error for nonexistent macro, got: {other:?}"),
-        }
-    });
-}
+//! Integration tests for the proc-macro-srv-cli main loop.
+//!
+//! These tests exercise the full client-server RPC procedure using in-memory
+//! channels without needing to spawn the actual server and client processes.
+
+#![cfg(feature = "sysroot-abi")]
+
+mod common {
+    pub(crate) mod utils;
+}
+
+use common::utils::{create_empty_token_tree, proc_macro_test_dylib_path, request, with_server};
+use expect_test::expect;
+use proc_macro_api::{
+    ProtocolFormat::JsonLegacy,
+    legacy_protocol::msg::{
+        ExpandMacro, ExpandMacroData, ExpnGlobals, PanicMessage, Request, Response, ServerConfig,
+        SpanDataIndexMap, SpanMode,
+    },
+    version::CURRENT_API_VERSION,
+};
+
+#[test]
+fn test_version_check() {
+    with_server(JsonLegacy, |writer, reader| {
+        let response = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+
+        match response {
+            Response::ApiVersionCheck(version) => {
+                assert_eq!(version, CURRENT_API_VERSION);
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_list_macros() {
+    with_server(JsonLegacy, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+        let response = request(writer, reader, Request::ListMacros { dylib_path }, None).into();
+
+        let Response::ListMacros(Ok(macros)) = response else {
+            panic!("expected successful ListMacros response");
+        };
+
+        let mut macro_list: Vec<_> =
+            macros.iter().map(|(name, kind)| format!("{name} [{kind:?}]")).collect();
+        macro_list.sort();
+        let macro_list_str = macro_list.join("\n");
+
+        expect![[r#"
+            DeriveEmpty [CustomDerive]
+            DeriveError [CustomDerive]
+            DerivePanic [CustomDerive]
+            DeriveReemit [CustomDerive]
+            attr_error [Attr]
+            attr_noop [Attr]
+            attr_panic [Attr]
+            fn_like_clone_tokens [Bang]
+            fn_like_error [Bang]
+            fn_like_mk_idents [Bang]
+            fn_like_mk_literals [Bang]
+            fn_like_noop [Bang]
+            fn_like_panic [Bang]
+            fn_like_span_join [Bang]
+            fn_like_span_line_column [Bang]
+            fn_like_span_ops [Bang]"#]]
+        .assert_eq(¯o_list_str);
+    });
+}
+
+#[test]
+fn test_list_macros_invalid_path() {
+    with_server(JsonLegacy, |writer, reader| {
+        let response = request(
+            writer,
+            reader,
+            Request::ListMacros { dylib_path: "/nonexistent/path/to/dylib.so".into() },
+            None,
+        )
+        .into();
+
+        match response {
+            Response::ListMacros(Err(e)) => assert!(
+                e.starts_with("Cannot create expander for /nonexistent/path/to/dylib.so"),
+                "{e}"
+            ),
+            other => panic!("expected error response, got: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_set_config() {
+    with_server(JsonLegacy, |writer, reader| {
+        let config = ServerConfig { span_mode: SpanMode::Id };
+        let response = request(writer, reader, Request::SetConfig(config), None).into();
+
+        match response {
+            Response::SetConfig(returned_config) => {
+                assert_eq!(returned_config.span_mode, SpanMode::Id);
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_set_config_rust_analyzer_mode() {
+    with_server(JsonLegacy, |writer, reader| {
+        let config = ServerConfig { span_mode: SpanMode::RustAnalyzer };
+        let response = request(writer, reader, Request::SetConfig(config), None).into();
+
+        match response {
+            Response::SetConfig(returned_config) => {
+                assert_eq!(returned_config.span_mode, SpanMode::RustAnalyzer);
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_expand_macro_panic() {
+    with_server(JsonLegacy, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+
+        let version_response = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        let Response::ApiVersionCheck(version) = version_response else {
+            panic!("expected version check response");
+        };
+
+        let mut span_data_table = SpanDataIndexMap::default();
+        let macro_body = create_empty_token_tree(version, &mut span_data_table);
+
+        let expand_request = Request::ExpandMacro(Box::new(ExpandMacro {
+            lib: dylib_path,
+            env: vec![],
+            current_dir: None,
+            data: ExpandMacroData {
+                macro_body,
+                macro_name: "fn_like_panic".to_owned(),
+                attributes: None,
+                has_global_spans: ExpnGlobals {
+                    serialize: version >= 3,
+                    def_site: 0,
+                    call_site: 0,
+                    mixed_site: 0,
+                },
+                span_data_table: vec![],
+            },
+        }));
+
+        let response = request(writer, reader, expand_request, None).into();
+
+        match response {
+            Response::ExpandMacro(Err(PanicMessage(msg))) => {
+                assert!(msg.contains("fn_like_panic"), "panic message should mention the macro");
+            }
+            Response::ExpandMacro(Ok(_)) => {
+                panic!("expected panic, but macro succeeded");
+            }
+            other => panic!("unexpected response: {other:?}"),
+        }
+    });
+}
+
+#[test]
+fn test_basic_call_flow() {
+    with_server(JsonLegacy, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+
+        let response1 = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        assert!(matches!(response1, Response::ApiVersionCheck(_)));
+
+        let response2 = request(
+            writer,
+            reader,
+            Request::SetConfig(ServerConfig { span_mode: SpanMode::Id }),
+            None,
+        )
+        .into();
+        assert!(matches!(response2, Response::SetConfig(_)));
+
+        let response3 =
+            request(writer, reader, Request::ListMacros { dylib_path: dylib_path.clone() }, None)
+                .into();
+        assert!(matches!(response3, Response::ListMacros(Ok(_))));
+    });
+}
+
+#[test]
+fn test_expand_nonexistent_macro() {
+    with_server(JsonLegacy, |writer, reader| {
+        let dylib_path = proc_macro_test_dylib_path();
+
+        let version_response = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        let Response::ApiVersionCheck(version) = version_response else {
+            panic!("expected version check response");
+        };
+
+        let mut span_data_table = SpanDataIndexMap::default();
+        let macro_body = create_empty_token_tree(version, &mut span_data_table);
+
+        let expand_request = Request::ExpandMacro(Box::new(ExpandMacro {
+            lib: dylib_path,
+            env: vec![],
+            current_dir: None,
+            data: ExpandMacroData {
+                macro_body,
+                macro_name: "NonexistentMacro".to_owned(),
+                attributes: None,
+                has_global_spans: ExpnGlobals {
+                    serialize: version >= 3,
+                    def_site: 0,
+                    call_site: 0,
+                    mixed_site: 0,
+                },
+                span_data_table: vec![],
+            },
+        }));
+
+        let response = request(writer, reader, expand_request, None).into();
+
+        match response {
+            Response::ExpandMacro(Err(PanicMessage(msg))) => {
+                expect!["proc-macro `NonexistentMacro` is missing"].assert_eq(&msg)
+            }
+            other => panic!("expected error for nonexistent macro, got: {other:?}"),
+        }
+    });
+}

From 095b0138028d51e45ca987e0b0b1dcf4e16a876d Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 18 Jan 2026 20:38:58 +0530
Subject: [PATCH 0379/1843] rename send and drive to request and receive and
 remove auto*

---
 .../tests/bidirectional_postcard.rs           |  48 +++----
 .../proc-macro-srv-cli/tests/common/utils.rs  | 118 ++++--------------
 .../proc-macro-srv-cli/tests/legacy_json.rs   |  37 +++---
 3 files changed, 67 insertions(+), 136 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs
index 08e44bad3723..33ca1d791de7 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs
@@ -4,7 +4,9 @@ mod common {
     pub(crate) mod utils;
 }
 
-use common::utils::{create_empty_token_tree, proc_macro_test_dylib_path, request, with_server};
+use common::utils::{
+    create_empty_token_tree, proc_macro_test_dylib_path, request_bidirectional, with_server,
+};
 use expect_test::expect;
 use proc_macro_api::{
     ProtocolFormat::BidirectionalPostcardPrototype,
@@ -20,7 +22,7 @@ use proc_macro_api::{
 fn test_bidi_version_check_bidirectional() {
     with_server(BidirectionalPostcardPrototype, |writer, reader| {
         let response =
-            request(writer, reader, Request::ApiVersionCheck {}, Some(&reject_subrequests)).into();
+            request_bidirectional(writer, reader, Request::ApiVersionCheck {}, reject_subrequests);
 
         match response {
             Response::ApiVersionCheck(version) => {
@@ -35,9 +37,12 @@ fn test_bidi_version_check_bidirectional() {
 fn test_bidi_list_macros() {
     with_server(BidirectionalPostcardPrototype, |writer, reader| {
         let dylib_path = proc_macro_test_dylib_path();
-        let response =
-            request(writer, reader, Request::ListMacros { dylib_path }, Some(&reject_subrequests))
-                .into();
+        let response = request_bidirectional(
+            writer,
+            reader,
+            Request::ListMacros { dylib_path },
+            &reject_subrequests,
+        );
 
         let Response::ListMacros(Ok(macros)) = response else {
             panic!("expected successful ListMacros response");
@@ -72,13 +77,12 @@ fn test_bidi_list_macros() {
 #[test]
 fn test_bidi_list_macros_invalid_path() {
     with_server(BidirectionalPostcardPrototype, |writer, reader| {
-        let response = request(
+        let response = request_bidirectional(
             writer,
             reader,
             Request::ListMacros { dylib_path: "/nonexistent/path/to/dylib.so".into() },
-            Some(&reject_subrequests),
-        )
-        .into();
+            reject_subrequests,
+        );
 
         match response {
             Response::ListMacros(Err(e)) => assert!(
@@ -95,7 +99,7 @@ fn test_bidi_set_config() {
     with_server(BidirectionalPostcardPrototype, |writer, reader| {
         let config = ServerConfig { span_mode: SpanMode::Id };
         let response =
-            request(writer, reader, Request::SetConfig(config), Some(&reject_subrequests)).into();
+            request_bidirectional(writer, reader, Request::SetConfig(config), reject_subrequests);
 
         match response {
             Response::SetConfig(returned_config) => {
@@ -111,7 +115,7 @@ fn test_bidi_set_config_rust_analyzer_mode() {
     with_server(BidirectionalPostcardPrototype, |writer, reader| {
         let config = ServerConfig { span_mode: SpanMode::RustAnalyzer };
         let response =
-            request(writer, reader, Request::SetConfig(config), Some(&reject_subrequests)).into();
+            request_bidirectional(writer, reader, Request::SetConfig(config), reject_subrequests);
 
         match response {
             Response::SetConfig(returned_config) => {
@@ -144,7 +148,7 @@ fn test_bidi_expand_macro_panic() {
             },
         }));
 
-        let response = request(writer, reader, request1, Some(&reject_subrequests)).into();
+        let response = request_bidirectional(writer, reader, request1, reject_subrequests);
 
         match response {
             Response::ExpandMacro(Err(PanicMessage(msg))) => {
@@ -161,25 +165,23 @@ fn test_bidi_basic_call_flow() {
         let dylib_path = proc_macro_test_dylib_path();
 
         let response1 =
-            request(writer, reader, Request::ApiVersionCheck {}, Some(&reject_subrequests)).into();
+            request_bidirectional(writer, reader, Request::ApiVersionCheck {}, reject_subrequests);
         assert!(matches!(response1, Response::ApiVersionCheck(_)));
 
-        let response2 = request(
+        let response2 = request_bidirectional(
             writer,
             reader,
             Request::SetConfig(ServerConfig { span_mode: SpanMode::Id }),
-            Some(&reject_subrequests),
-        )
-        .into();
+            reject_subrequests,
+        );
         assert!(matches!(response2, Response::SetConfig(_)));
 
-        let response3 = request(
+        let response3 = request_bidirectional(
             writer,
             reader,
             Request::ListMacros { dylib_path: dylib_path.clone() },
-            Some(&reject_subrequests),
-        )
-        .into();
+            reject_subrequests,
+        );
         assert!(matches!(response3, Response::ListMacros(Ok(_))));
     });
 }
@@ -190,7 +192,7 @@ fn test_bidi_expand_nonexistent_macro() {
         let dylib_path = proc_macro_test_dylib_path();
 
         let version_response =
-            request(writer, reader, Request::ApiVersionCheck {}, Some(&reject_subrequests)).into();
+            request_bidirectional(writer, reader, Request::ApiVersionCheck {}, reject_subrequests);
         let Response::ApiVersionCheck(version) = version_response else {
             panic!("expected version check response");
         };
@@ -211,7 +213,7 @@ fn test_bidi_expand_nonexistent_macro() {
             },
         }));
 
-        let response = request(writer, reader, expand_request, Some(&reject_subrequests)).into();
+        let response = request_bidirectional(writer, reader, expand_request, reject_subrequests);
 
         match response {
             Response::ExpandMacro(Err(PanicMessage(msg))) => {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs
index 63b3a74aa4e8..85c394734b33 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/common/utils.rs
@@ -198,21 +198,22 @@ trait TestProtocol {
     type Request;
     type Response;
 
-    fn send(&self, writer: &mut dyn Write, req: Self::Request);
-    fn drive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> Self::Response;
+    fn request(&self, writer: &mut dyn Write, req: Self::Request);
+    fn receive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> Self::Response;
 }
 
+#[allow(dead_code)]
 struct JsonLegacy;
 
 impl TestProtocol for JsonLegacy {
     type Request = Request;
     type Response = Response;
 
-    fn send(&self, writer: &mut dyn Write, req: Request) {
+    fn request(&self, writer: &mut dyn Write, req: Request) {
         req.write::(writer).expect("failed to write request");
     }
 
-    fn drive(&self, reader: &mut dyn BufRead, _writer: &mut dyn Write) -> Response {
+    fn receive(&self, reader: &mut dyn BufRead, _writer: &mut dyn Write) -> Response {
         let mut buf = String::new();
         Response::read::(reader, &mut buf)
             .expect("failed to read response")
@@ -220,6 +221,7 @@ impl TestProtocol for JsonLegacy {
     }
 }
 
+#[allow(dead_code)]
 struct PostcardBidirectional
 where
     F: Fn(SubRequest) -> Result,
@@ -234,12 +236,12 @@ where
     type Request = BiRequest;
     type Response = BiResponse;
 
-    fn send(&self, writer: &mut dyn Write, req: BiRequest) {
+    fn request(&self, writer: &mut dyn Write, req: BiRequest) {
         let msg = BidirectionalMessage::Request(req);
         msg.write::(writer).expect("failed to write request");
     }
 
-    fn drive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> BiResponse {
+    fn receive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> BiResponse {
         let mut buf = Vec::new();
 
         loop {
@@ -260,98 +262,28 @@ where
     }
 }
 
-pub(crate) fn request(
+#[allow(dead_code)]
+pub(crate) fn request_legacy(
     writer: &mut dyn Write,
     reader: &mut dyn BufRead,
-    request: impl Into,
-    callback: Option<&dyn Fn(SubRequest) -> Result>,
-) -> AutoResponse {
-    let protocol = match callback {
-        None => AutoProtocol::Legacy(JsonLegacy),
-        Some(cb) => AutoProtocol::Bidirectional(PostcardBidirectional { callback: cb }),
-    };
-
-    protocol.send(writer, request.into());
-    protocol.drive(reader, writer)
+    request: Request,
+) -> Response {
+    let protocol = JsonLegacy;
+    protocol.request(writer, request);
+    protocol.receive(reader, writer)
 }
 
-enum AutoProtocol
+#[allow(dead_code)]
+pub(crate) fn request_bidirectional(
+    writer: &mut dyn Write,
+    reader: &mut dyn BufRead,
+    request: BiRequest,
+    callback: F,
+) -> BiResponse
 where
     F: Fn(SubRequest) -> Result,
 {
-    Legacy(JsonLegacy),
-    Bidirectional(PostcardBidirectional),
-}
-
-impl TestProtocol for AutoProtocol
-where
-    F: Fn(SubRequest) -> Result,
-{
-    type Request = AutoRequest;
-    type Response = AutoResponse;
-
-    fn send(&self, writer: &mut dyn Write, req: AutoRequest) {
-        match (self, req) {
-            (AutoProtocol::Legacy(p), AutoRequest::Legacy(r)) => {
-                p.send(writer, r);
-            }
-            (AutoProtocol::Bidirectional(p), AutoRequest::Bidirectional(r)) => {
-                p.send(writer, r);
-            }
-            (AutoProtocol::Legacy(_), AutoRequest::Bidirectional(_)) => {
-                panic!("bidirectional request used with legacy protocol");
-            }
-            (AutoProtocol::Bidirectional(_), AutoRequest::Legacy(_)) => {
-                panic!("legacy request used with bidirectional protocol");
-            }
-        }
-    }
-
-    fn drive(&self, reader: &mut dyn BufRead, writer: &mut dyn Write) -> AutoResponse {
-        match self {
-            AutoProtocol::Legacy(p) => AutoResponse::Legacy(p.drive(reader, writer)),
-            AutoProtocol::Bidirectional(p) => AutoResponse::Bidirectional(p.drive(reader, writer)),
-        }
-    }
-}
-
-pub(crate) enum AutoRequest {
-    Legacy(Request),
-    Bidirectional(BiRequest),
-}
-
-#[derive(Debug)]
-pub(crate) enum AutoResponse {
-    Legacy(Response),
-    Bidirectional(BiResponse),
-}
-
-impl From for AutoRequest {
-    fn from(req: Request) -> AutoRequest {
-        AutoRequest::Legacy(req)
-    }
-}
-
-impl From for AutoRequest {
-    fn from(req: BiRequest) -> AutoRequest {
-        AutoRequest::Bidirectional(req)
-    }
-}
-
-impl From for Response {
-    fn from(res: AutoResponse) -> Response {
-        match res {
-            AutoResponse::Legacy(res) => res,
-            _ => panic!("Should be legacy response"),
-        }
-    }
-}
-
-impl From for BiResponse {
-    fn from(res: AutoResponse) -> BiResponse {
-        match res {
-            AutoResponse::Bidirectional(res) => res,
-            _ => panic!("Should be bidirectional response"),
-        }
-    }
+    let protocol = PostcardBidirectional { callback };
+    protocol.request(writer, request);
+    protocol.receive(reader, writer)
 }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs
index 8daee7b2bceb..c0dbfd1679f7 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/tests/legacy_json.rs
@@ -9,7 +9,9 @@ mod common {
     pub(crate) mod utils;
 }
 
-use common::utils::{create_empty_token_tree, proc_macro_test_dylib_path, request, with_server};
+use common::utils::{
+    create_empty_token_tree, proc_macro_test_dylib_path, request_legacy, with_server,
+};
 use expect_test::expect;
 use proc_macro_api::{
     ProtocolFormat::JsonLegacy,
@@ -23,7 +25,7 @@ use proc_macro_api::{
 #[test]
 fn test_version_check() {
     with_server(JsonLegacy, |writer, reader| {
-        let response = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        let response = request_legacy(writer, reader, Request::ApiVersionCheck {});
 
         match response {
             Response::ApiVersionCheck(version) => {
@@ -38,7 +40,7 @@ fn test_version_check() {
 fn test_list_macros() {
     with_server(JsonLegacy, |writer, reader| {
         let dylib_path = proc_macro_test_dylib_path();
-        let response = request(writer, reader, Request::ListMacros { dylib_path }, None).into();
+        let response = request_legacy(writer, reader, Request::ListMacros { dylib_path });
 
         let Response::ListMacros(Ok(macros)) = response else {
             panic!("expected successful ListMacros response");
@@ -73,13 +75,11 @@ fn test_list_macros() {
 #[test]
 fn test_list_macros_invalid_path() {
     with_server(JsonLegacy, |writer, reader| {
-        let response = request(
+        let response = request_legacy(
             writer,
             reader,
             Request::ListMacros { dylib_path: "/nonexistent/path/to/dylib.so".into() },
-            None,
-        )
-        .into();
+        );
 
         match response {
             Response::ListMacros(Err(e)) => assert!(
@@ -95,7 +95,7 @@ fn test_list_macros_invalid_path() {
 fn test_set_config() {
     with_server(JsonLegacy, |writer, reader| {
         let config = ServerConfig { span_mode: SpanMode::Id };
-        let response = request(writer, reader, Request::SetConfig(config), None).into();
+        let response = request_legacy(writer, reader, Request::SetConfig(config));
 
         match response {
             Response::SetConfig(returned_config) => {
@@ -110,7 +110,7 @@ fn test_set_config() {
 fn test_set_config_rust_analyzer_mode() {
     with_server(JsonLegacy, |writer, reader| {
         let config = ServerConfig { span_mode: SpanMode::RustAnalyzer };
-        let response = request(writer, reader, Request::SetConfig(config), None).into();
+        let response = request_legacy(writer, reader, Request::SetConfig(config));
 
         match response {
             Response::SetConfig(returned_config) => {
@@ -126,7 +126,7 @@ fn test_expand_macro_panic() {
     with_server(JsonLegacy, |writer, reader| {
         let dylib_path = proc_macro_test_dylib_path();
 
-        let version_response = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        let version_response = request_legacy(writer, reader, Request::ApiVersionCheck {});
         let Response::ApiVersionCheck(version) = version_response else {
             panic!("expected version check response");
         };
@@ -152,7 +152,7 @@ fn test_expand_macro_panic() {
             },
         }));
 
-        let response = request(writer, reader, expand_request, None).into();
+        let response = request_legacy(writer, reader, expand_request);
 
         match response {
             Response::ExpandMacro(Err(PanicMessage(msg))) => {
@@ -171,21 +171,18 @@ fn test_basic_call_flow() {
     with_server(JsonLegacy, |writer, reader| {
         let dylib_path = proc_macro_test_dylib_path();
 
-        let response1 = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        let response1 = request_legacy(writer, reader, Request::ApiVersionCheck {});
         assert!(matches!(response1, Response::ApiVersionCheck(_)));
 
-        let response2 = request(
+        let response2 = request_legacy(
             writer,
             reader,
             Request::SetConfig(ServerConfig { span_mode: SpanMode::Id }),
-            None,
-        )
-        .into();
+        );
         assert!(matches!(response2, Response::SetConfig(_)));
 
         let response3 =
-            request(writer, reader, Request::ListMacros { dylib_path: dylib_path.clone() }, None)
-                .into();
+            request_legacy(writer, reader, Request::ListMacros { dylib_path: dylib_path.clone() });
         assert!(matches!(response3, Response::ListMacros(Ok(_))));
     });
 }
@@ -195,7 +192,7 @@ fn test_expand_nonexistent_macro() {
     with_server(JsonLegacy, |writer, reader| {
         let dylib_path = proc_macro_test_dylib_path();
 
-        let version_response = request(writer, reader, Request::ApiVersionCheck {}, None).into();
+        let version_response = request_legacy(writer, reader, Request::ApiVersionCheck {});
         let Response::ApiVersionCheck(version) = version_response else {
             panic!("expected version check response");
         };
@@ -221,7 +218,7 @@ fn test_expand_nonexistent_macro() {
             },
         }));
 
-        let response = request(writer, reader, expand_request, None).into();
+        let response = request_legacy(writer, reader, expand_request);
 
         match response {
             Response::ExpandMacro(Err(PanicMessage(msg))) => {

From 7ec34defe9e62a1a6946d3e700b5903d8dc89ece Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Mon, 19 Jan 2026 14:46:38 +1100
Subject: [PATCH 0380/1843] Temporarily re-export `assert_matches!` to reduce
 stabilization churn

---
 compiler/rustc_abi/src/extern_abi/tests.rs                   | 3 ++-
 compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs    | 3 +--
 compiler/rustc_borrowck/src/type_check/input_output.rs       | 3 +--
 compiler/rustc_builtin_macros/src/test.rs                    | 2 +-
 compiler/rustc_codegen_llvm/src/asm.rs                       | 3 +--
 compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs       | 2 +-
 compiler/rustc_codegen_llvm/src/intrinsic.rs                 | 2 +-
 compiler/rustc_codegen_ssa/src/back/write.rs                 | 2 +-
 compiler/rustc_codegen_ssa/src/traits/builder.rs             | 2 +-
 compiler/rustc_const_eval/src/check_consts/check.rs          | 2 +-
 compiler/rustc_const_eval/src/interpret/call.rs              | 2 +-
 compiler/rustc_const_eval/src/interpret/cast.rs              | 3 +--
 compiler/rustc_const_eval/src/interpret/eval_context.rs      | 3 +--
 compiler/rustc_const_eval/src/interpret/intrinsics.rs        | 3 +--
 compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs   | 3 ++-
 compiler/rustc_const_eval/src/interpret/memory.rs            | 2 +-
 compiler/rustc_const_eval/src/interpret/operand.rs           | 3 +--
 compiler/rustc_const_eval/src/interpret/place.rs             | 3 +--
 compiler/rustc_data_structures/src/graph/scc/mod.rs          | 2 +-
 compiler/rustc_data_structures/src/lib.rs                    | 5 +++++
 compiler/rustc_errors/src/lib.rs                             | 3 +--
 compiler/rustc_hir_analysis/src/collect.rs                   | 2 +-
 compiler/rustc_hir_analysis/src/collect/generics_of.rs       | 2 +-
 compiler/rustc_hir_analysis/src/collect/predicates_of.rs     | 3 +--
 compiler/rustc_hir_analysis/src/delegation.rs                | 3 +--
 compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs       | 2 +-
 compiler/rustc_hir_analysis/src/impl_wf_check.rs             | 3 +--
 compiler/rustc_hir_typeck/src/method/probe.rs                | 2 +-
 compiler/rustc_infer/src/infer/outlives/verify.rs            | 3 +--
 compiler/rustc_infer/src/infer/snapshot/undo_log.rs          | 3 +--
 compiler/rustc_lint/src/impl_trait_overcaptures.rs           | 2 +-
 compiler/rustc_middle/src/ty/consts/kind.rs                  | 3 +--
 compiler/rustc_middle/src/ty/context.rs                      | 3 +--
 compiler/rustc_middle/src/ty/instance.rs                     | 2 +-
 compiler/rustc_middle/src/ty/mod.rs                          | 2 +-
 compiler/rustc_middle/src/ty/sty.rs                          | 2 +-
 compiler/rustc_mir_build/src/builder/coverageinfo.rs         | 2 +-
 compiler/rustc_mir_build/src/builder/expr/as_place.rs        | 2 +-
 compiler/rustc_mir_build/src/builder/matches/mod.rs          | 2 +-
 compiler/rustc_mir_build/src/builder/matches/user_ty.rs      | 2 +-
 compiler/rustc_mir_build/src/thir/pattern/mod.rs             | 2 +-
 compiler/rustc_mir_build/src/thir/util.rs                    | 3 +--
 compiler/rustc_mir_dataflow/src/impls/initialized.rs         | 3 +--
 compiler/rustc_mir_dataflow/src/value_analysis.rs            | 2 +-
 compiler/rustc_mir_transform/src/dataflow_const_prop.rs      | 2 +-
 compiler/rustc_mir_transform/src/inline.rs                   | 2 +-
 compiler/rustc_mir_transform/src/promote_consts.rs           | 2 +-
 compiler/rustc_mir_transform/src/shim.rs                     | 2 +-
 compiler/rustc_parse/src/parser/mod.rs                       | 2 +-
 compiler/rustc_parse/src/parser/tests.rs                     | 2 +-
 compiler/rustc_query_system/src/dep_graph/graph.rs           | 3 +--
 compiler/rustc_resolve/src/late.rs                           | 2 +-
 compiler/rustc_symbol_mangling/src/export.rs                 | 3 +--
 .../src/error_reporting/traits/suggestions.rs                | 2 +-
 compiler/rustc_trait_selection/src/solve/inspect/analyse.rs  | 3 +--
 compiler/rustc_trait_selection/src/traits/select/mod.rs      | 2 +-
 compiler/rustc_ty_utils/src/abi.rs                           | 2 +-
 compiler/rustc_ty_utils/src/layout/invariant.rs              | 3 +--
 58 files changed, 64 insertions(+), 79 deletions(-)

diff --git a/compiler/rustc_abi/src/extern_abi/tests.rs b/compiler/rustc_abi/src/extern_abi/tests.rs
index fc546a6570f0..8b9353ccae97 100644
--- a/compiler/rustc_abi/src/extern_abi/tests.rs
+++ b/compiler/rustc_abi/src/extern_abi/tests.rs
@@ -1,6 +1,7 @@
-use std::assert_matches::assert_matches;
 use std::str::FromStr;
 
+use rustc_data_structures::assert_matches;
+
 use super::*;
 
 #[allow(non_snake_case)]
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 743a28822eb9..8f643d8f460f 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -3,8 +3,7 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
 
-use std::assert_matches::assert_matches;
-
+use rustc_data_structures::assert_matches;
 use rustc_errors::{Applicability, Diag, EmissionGuarantee};
 use rustc_hir as hir;
 use rustc_hir::intravisit::Visitor;
diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs
index eb31b5de05d2..f3b9dcc90a84 100644
--- a/compiler/rustc_borrowck/src/type_check/input_output.rs
+++ b/compiler/rustc_borrowck/src/type_check/input_output.rs
@@ -7,9 +7,8 @@
 //! `RETURN_PLACE` the MIR arguments) are always fully normalized (and
 //! contain revealed `impl Trait` values).
 
-use std::assert_matches::assert_matches;
-
 use itertools::Itertools;
+use rustc_data_structures::assert_matches;
 use rustc_hir as hir;
 use rustc_infer::infer::{BoundRegionConversionTime, RegionVariableOrigin};
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index 6f14385d5d30..d5f774865a9e 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -1,12 +1,12 @@
 //! The expansion from a test function to the appropriate test struct for libtest
 //! Ideally, this code would be in libtest but for efficiency and error messages it lives here.
 
-use std::assert_matches::assert_matches;
 use std::iter;
 
 use rustc_ast::{self as ast, GenericParamKind, HasNodeId, attr, join_path_idents};
 use rustc_ast_pretty::pprust;
 use rustc_attr_parsing::AttributeParser;
+use rustc_data_structures::assert_matches;
 use rustc_errors::{Applicability, Diag, Level};
 use rustc_expand::base::*;
 use rustc_hir::Attribute;
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index ee1b6d45e149..7f02518d6c0d 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -1,9 +1,8 @@
-use std::assert_matches::assert_matches;
-
 use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_codegen_ssa::mir::operand::OperandValue;
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::Instance;
 use rustc_middle::ty::layout::TyAndLayout;
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index a728f3ea1e66..b27b1a88f133 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -1,9 +1,9 @@
-use std::assert_matches::assert_matches;
 use std::sync::Arc;
 
 use itertools::Itertools;
 use rustc_abi::Align;
 use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods};
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_index::IndexVec;
 use rustc_middle::ty::TyCtxt;
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index c6aae89f1e51..565db7d298bc 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -1,4 +1,3 @@
-use std::assert_matches::assert_matches;
 use std::cmp::Ordering;
 use std::ffi::c_uint;
 use std::ptr;
@@ -13,6 +12,7 @@ use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphizati
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::assert_matches;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_hir::{self as hir};
 use rustc_middle::mir::BinOp;
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 3e36bd8552b1..53121fc6275b 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -1,4 +1,3 @@
-use std::assert_matches::assert_matches;
 use std::marker::PhantomData;
 use std::panic::AssertUnwindSafe;
 use std::path::{Path, PathBuf};
@@ -8,6 +7,7 @@ use std::{fs, io, mem, str, thread};
 
 use rustc_abi::Size;
 use rustc_ast::attr;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::jobserver::{self, Acquired};
 use rustc_data_structures::memmap::Mmap;
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 4f45c614e003..ba36188f05d1 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -1,7 +1,7 @@
-use std::assert_matches::assert_matches;
 use std::ops::Deref;
 
 use rustc_abi::{Align, Scalar, Size, WrappingRange};
+use rustc_data_structures::assert_matches;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::mir;
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs
index 3a85ca3760d2..95dbf42d4d44 100644
--- a/compiler/rustc_const_eval/src/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/check_consts/check.rs
@@ -1,11 +1,11 @@
 //! The `Visitor` responsible for actually checking a `mir::Body` for invalid operations.
 
-use std::assert_matches::assert_matches;
 use std::borrow::Cow;
 use std::mem;
 use std::num::NonZero;
 use std::ops::Deref;
 
+use rustc_data_structures::assert_matches;
 use rustc_errors::{Diag, ErrorGuaranteed};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs
index 94c6fd1b3238..9c8ca44c5e8f 100644
--- a/compiler/rustc_const_eval/src/interpret/call.rs
+++ b/compiler/rustc_const_eval/src/interpret/call.rs
@@ -1,10 +1,10 @@
 //! Manages calling a concrete function (with known MIR body) with argument passing,
 //! and returning the return value to the caller.
-use std::assert_matches::assert_matches;
 use std::borrow::Cow;
 
 use either::{Left, Right};
 use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx};
+use rustc_data_structures::assert_matches;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::layout::{IntegerExt, TyAndLayout};
 use rustc_middle::ty::{self, AdtDef, Instance, Ty, VariantDef};
diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index 3485a5c625ba..43de2e7f078a 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -1,8 +1,7 @@
-use std::assert_matches::assert_matches;
-
 use rustc_abi::{FieldIdx, Integer};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_apfloat::{Float, FloatConvert};
+use rustc_data_structures::assert_matches;
 use rustc_middle::mir::CastKind;
 use rustc_middle::mir::interpret::{InterpResult, PointerArithmetic, Scalar};
 use rustc_middle::ty::adjustment::PointerCoercion;
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index d23369caffa4..aeadf0257ea8 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -1,7 +1,6 @@
-use std::assert_matches::debug_assert_matches;
-
 use either::{Left, Right};
 use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout};
+use rustc_data_structures::debug_assert_matches;
 use rustc_errors::DiagCtxtHandle;
 use rustc_hir::def_id::DefId;
 use rustc_hir::limit::Limit;
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index fe1dd1b6eb35..e526f6120689 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -4,10 +4,9 @@
 
 mod simd;
 
-use std::assert_matches::assert_matches;
-
 use rustc_abi::{FIRST_VARIANT, FieldIdx, HasDataLayout, Size, VariantIdx};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
+use rustc_data_structures::assert_matches;
 use rustc_hir::def_id::CRATE_DEF_ID;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs
index 33a115384a88..d7fe7801fb08 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs
@@ -2,6 +2,7 @@ use either::Either;
 use rustc_abi::{BackendRepr, Endian};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_apfloat::{Float, Round};
+use rustc_data_structures::assert_matches;
 use rustc_middle::mir::interpret::{InterpErrorKind, Pointer, UndefinedBehaviorInfo};
 use rustc_middle::ty::{FloatTy, ScalarInt, SimdAlign};
 use rustc_middle::{bug, err_ub_format, mir, span_bug, throw_unsup_format, ty};
@@ -10,7 +11,7 @@ use tracing::trace;
 
 use super::{
     ImmTy, InterpCx, InterpResult, Machine, MinMax, MulAddType, OpTy, PlaceTy, Provenance, Scalar,
-    Size, TyAndLayout, assert_matches, interp_ok, throw_ub_format,
+    Size, TyAndLayout, interp_ok, throw_ub_format,
 };
 use crate::interpret::Writeable;
 
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index 862fe4779080..a6c8b28cce9f 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -6,7 +6,6 @@
 //! integer. It is crucial that these operations call `check_align` *before*
 //! short-circuiting the empty case!
 
-use std::assert_matches::assert_matches;
 use std::borrow::{Borrow, Cow};
 use std::cell::Cell;
 use std::collections::VecDeque;
@@ -14,6 +13,7 @@ use std::{fmt, ptr};
 
 use rustc_abi::{Align, HasDataLayout, Size};
 use rustc_ast::Mutability;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_middle::mir::display_allocation;
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 9a956259ba57..e8e77de8eb3e 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -1,11 +1,10 @@
 //! Functions concerning immediate values and operands, and reading from operands.
 //! All high-level functions to read from memory work on operands as sources.
 
-use std::assert_matches::assert_matches;
-
 use either::{Either, Left, Right};
 use rustc_abi as abi;
 use rustc_abi::{BackendRepr, HasDataLayout, Size};
+use rustc_data_structures::assert_matches;
 use rustc_hir::def::Namespace;
 use rustc_middle::mir::interpret::ScalarSizeMismatch;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, TyAndLayout};
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index a409c7fad417..d472c14253b5 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -2,10 +2,9 @@
 //! into a place.
 //! All high-level functions to write to memory work on places as destinations.
 
-use std::assert_matches::assert_matches;
-
 use either::{Either, Left, Right};
 use rustc_abi::{BackendRepr, HasDataLayout, Size};
+use rustc_data_structures::assert_matches;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::{bug, mir, span_bug};
diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs
index 1882e6e835a4..91cbe3c533bb 100644
--- a/compiler/rustc_data_structures/src/graph/scc/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs
@@ -8,7 +8,6 @@
 //! Typical examples would include: minimum element in SCC, maximum element
 //! reachable from it, etc.
 
-use std::assert_matches::debug_assert_matches;
 use std::fmt::Debug;
 use std::marker::PhantomData;
 use std::ops::Range;
@@ -16,6 +15,7 @@ use std::ops::Range;
 use rustc_index::{Idx, IndexSlice, IndexVec};
 use tracing::{debug, instrument, trace};
 
+use crate::debug_assert_matches;
 use crate::fx::FxHashSet;
 use crate::graph::vec_graph::VecGraph;
 use crate::graph::{DirectedGraph, NumEdges, Successors};
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index ff1dd41c82cc..8377213850b8 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -38,6 +38,11 @@
 #![feature(unwrap_infallible)]
 // tidy-alphabetical-end
 
+// Temporarily re-export `assert_matches!`, so that the rest of the compiler doesn't
+// have to worry about it being moved to a different module in std during stabilization.
+// FIXME(#151359): Remove this when `feature(assert_matches)` is stable in stage0.
+// (This doesn't necessarily need to be fixed during the beta bump itself.)
+pub use std::assert_matches::{assert_matches, debug_assert_matches};
 use std::fmt;
 
 pub use atomic_ref::AtomicRef;
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 148368045f4f..ce40f3ae8bf8 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -23,7 +23,6 @@
 
 extern crate self as rustc_errors;
 
-use std::assert_matches::assert_matches;
 use std::backtrace::{Backtrace, BacktraceStatus};
 use std::borrow::Cow;
 use std::cell::Cell;
@@ -55,10 +54,10 @@ pub use diagnostic_impls::{
 };
 pub use emitter::ColorConfig;
 use emitter::{ConfusionType, DynEmitter, Emitter, detect_confusion_type, is_different};
-use rustc_data_structures::AtomicRef;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::sync::{DynSend, Lock};
+use rustc_data_structures::{AtomicRef, assert_matches};
 pub use rustc_error_messages::{
     DiagArg, DiagArgFromDisplay, DiagArgName, DiagArgValue, DiagMessage, FluentBundle, IntoDiagArg,
     LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, SubdiagMessage,
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index 2c0253409387..9c0b638c1482 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -14,13 +14,13 @@
 //! At present, however, we do run collection across all items in the
 //! crate as a kind of pass. This should eventually be factored away.
 
-use std::assert_matches::assert_matches;
 use std::cell::Cell;
 use std::iter;
 use std::ops::Bound;
 
 use rustc_abi::{ExternAbi, Size};
 use rustc_ast::Recovered;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_errors::{
     Applicability, Diag, DiagCtxtHandle, E0228, ErrorGuaranteed, StashKey, struct_span_code_err,
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index 3d2f0466cad0..511d513216eb 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -1,6 +1,6 @@
-use std::assert_matches::assert_matches;
 use std::ops::ControlFlow;
 
+use rustc_data_structures::assert_matches;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::{self, Visitor, VisitorExt};
diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
index 178c47b09c84..a2236b426305 100644
--- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
@@ -1,6 +1,5 @@
-use std::assert_matches::assert_matches;
-
 use hir::Node;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir as hir;
 use rustc_hir::attrs::AttributeKind;
diff --git a/compiler/rustc_hir_analysis/src/delegation.rs b/compiler/rustc_hir_analysis/src/delegation.rs
index cf0533c39e73..f64341d755f8 100644
--- a/compiler/rustc_hir_analysis/src/delegation.rs
+++ b/compiler/rustc_hir_analysis/src/delegation.rs
@@ -2,8 +2,7 @@
 //!
 //! For more information about delegation design, see the tracking issue #118212.
 
-use std::assert_matches::debug_assert_matches;
-
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index 6e1e6c157a91..924967b65c19 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -19,10 +19,10 @@ mod dyn_trait;
 pub mod errors;
 pub mod generics;
 
-use std::assert_matches::assert_matches;
 use std::slice;
 
 use rustc_ast::LitKind;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_errors::codes::*;
 use rustc_errors::{
diff --git a/compiler/rustc_hir_analysis/src/impl_wf_check.rs b/compiler/rustc_hir_analysis/src/impl_wf_check.rs
index cadbc54c3410..f5c77c680000 100644
--- a/compiler/rustc_hir_analysis/src/impl_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/impl_wf_check.rs
@@ -8,9 +8,8 @@
 //! specialization errors. These things can (and probably should) be
 //! fixed, but for the moment it's easier to do these checks early.
 
-use std::assert_matches::debug_assert_matches;
-
 use min_specialization::check_min_specialization;
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::Applicability;
 use rustc_errors::codes::*;
diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs
index beb0337d8c59..ed71ad2119c9 100644
--- a/compiler/rustc_hir_typeck/src/method/probe.rs
+++ b/compiler/rustc_hir_typeck/src/method/probe.rs
@@ -1,8 +1,8 @@
-use std::assert_matches::debug_assert_matches;
 use std::cell::{Cell, RefCell};
 use std::cmp::max;
 use std::ops::Deref;
 
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::sso::SsoHashSet;
 use rustc_errors::Applicability;
diff --git a/compiler/rustc_infer/src/infer/outlives/verify.rs b/compiler/rustc_infer/src/infer/outlives/verify.rs
index f67b99cb3f84..affeb01e6d05 100644
--- a/compiler/rustc_infer/src/infer/outlives/verify.rs
+++ b/compiler/rustc_infer/src/infer/outlives/verify.rs
@@ -1,5 +1,4 @@
-use std::assert_matches::assert_matches;
-
+use rustc_data_structures::assert_matches;
 use rustc_middle::ty::outlives::{Component, compute_alias_components_recursive};
 use rustc_middle::ty::{self, OutlivesPredicate, Ty, TyCtxt};
 use smallvec::smallvec;
diff --git a/compiler/rustc_infer/src/infer/snapshot/undo_log.rs b/compiler/rustc_infer/src/infer/snapshot/undo_log.rs
index c859d64133c4..a6f324b70471 100644
--- a/compiler/rustc_infer/src/infer/snapshot/undo_log.rs
+++ b/compiler/rustc_infer/src/infer/snapshot/undo_log.rs
@@ -1,8 +1,7 @@
-use std::assert_matches::assert_matches;
 use std::marker::PhantomData;
 
 use rustc_data_structures::undo_log::{Rollback, UndoLogs};
-use rustc_data_structures::{snapshot_vec as sv, unify as ut};
+use rustc_data_structures::{assert_matches, snapshot_vec as sv, unify as ut};
 use rustc_middle::ty::{self, OpaqueTypeKey, ProvisionalHiddenType};
 use tracing::debug;
 
diff --git a/compiler/rustc_lint/src/impl_trait_overcaptures.rs b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
index 2fc9d562dc56..f6c2e5946079 100644
--- a/compiler/rustc_lint/src/impl_trait_overcaptures.rs
+++ b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
@@ -1,6 +1,6 @@
-use std::assert_matches::debug_assert_matches;
 use std::cell::LazyCell;
 
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{LintDiagnostic, Subdiagnostic};
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index b3436550e8e0..c9ccb9bd0b3e 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -1,5 +1,4 @@
-use std::assert_matches::assert_matches;
-
+use rustc_data_structures::assert_matches;
 use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
 
 use super::Const;
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index e9fa3f14358e..a5aead9829f1 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -4,7 +4,6 @@
 
 pub mod tls;
 
-use std::assert_matches::debug_assert_matches;
 use std::borrow::{Borrow, Cow};
 use std::cmp::Ordering;
 use std::env::VarError;
@@ -17,7 +16,6 @@ use std::{fmt, iter, mem};
 
 use rustc_abi::{ExternAbi, FieldIdx, Layout, LayoutData, TargetDataLayout, VariantIdx};
 use rustc_ast as ast;
-use rustc_data_structures::defer;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::intern::Interned;
@@ -29,6 +27,7 @@ use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{
     self, DynSend, DynSync, FreezeReadGuard, Lock, RwLock, WorkerLocal,
 };
+use rustc_data_structures::{debug_assert_matches, defer};
 use rustc_errors::{
     Applicability, Diag, DiagCtxtHandle, ErrorGuaranteed, LintDiagnostic, MultiSpan,
 };
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index c27d47fcc0d8..0e9dd7dd169c 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -1,6 +1,6 @@
-use std::assert_matches::assert_matches;
 use std::fmt;
 
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 8eee114ead02..ce713dcf42f5 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -11,7 +11,6 @@
 
 #![allow(rustc::usage_of_ty_tykind)]
 
-use std::assert_matches::assert_matches;
 use std::fmt::Debug;
 use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
@@ -31,6 +30,7 @@ use rustc_ast::AttrVec;
 use rustc_ast::expand::typetree::{FncTree, Kind, Type, TypeTree};
 use rustc_ast::node_id::NodeMap;
 pub use rustc_ast_ir::{Movability, Mutability, try_visit};
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index c282f2211f65..34aca8adb4a0 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -2,12 +2,12 @@
 
 #![allow(rustc::usage_of_ty_tykind)]
 
-use std::assert_matches::debug_assert_matches;
 use std::borrow::Cow;
 use std::ops::{ControlFlow, Range};
 
 use hir::def::{CtorKind, DefKind};
 use rustc_abi::{FIRST_VARIANT, FieldIdx, ScalableElt, VariantIdx};
+use rustc_data_structures::debug_assert_matches;
 use rustc_errors::{ErrorGuaranteed, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::LangItem;
diff --git a/compiler/rustc_mir_build/src/builder/coverageinfo.rs b/compiler/rustc_mir_build/src/builder/coverageinfo.rs
index 091b9dad5bc1..ae36b2fb7f38 100644
--- a/compiler/rustc_mir_build/src/builder/coverageinfo.rs
+++ b/compiler/rustc_mir_build/src/builder/coverageinfo.rs
@@ -1,6 +1,6 @@
-use std::assert_matches::assert_matches;
 use std::collections::hash_map::Entry;
 
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, CoverageInfoHi, CoverageKind};
 use rustc_middle::mir::{self, BasicBlock, SourceInfo, UnOp};
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_place.rs b/compiler/rustc_mir_build/src/builder/expr/as_place.rs
index 139c6da29d44..172dbf7c31b5 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_place.rs
@@ -1,9 +1,9 @@
 //! See docs in build/expr/mod.rs
 
-use std::assert_matches::assert_matches;
 use std::iter;
 
 use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
+use rustc_data_structures::assert_matches;
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::hir::place::{Projection as HirProjection, ProjectionKind as HirProjectionKind};
 use rustc_middle::mir::AssertKind::BoundsCheck;
diff --git a/compiler/rustc_mir_build/src/builder/matches/mod.rs b/compiler/rustc_mir_build/src/builder/matches/mod.rs
index 11a181cfa8ce..2f9486c2d552 100644
--- a/compiler/rustc_mir_build/src/builder/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/mod.rs
@@ -5,13 +5,13 @@
 //! This also includes code for pattern bindings in `let` statements and
 //! function parameters.
 
-use std::assert_matches::debug_assert_matches;
 use std::borrow::Borrow;
 use std::mem;
 use std::sync::Arc;
 
 use itertools::{Itertools, Position};
 use rustc_abi::{FIRST_VARIANT, VariantIdx};
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::{BindingMode, ByRef, LangItem, LetStmt, LocalSource, Node};
diff --git a/compiler/rustc_mir_build/src/builder/matches/user_ty.rs b/compiler/rustc_mir_build/src/builder/matches/user_ty.rs
index 2dcfd3772902..6ba5e360ef82 100644
--- a/compiler/rustc_mir_build/src/builder/matches/user_ty.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/user_ty.rs
@@ -4,10 +4,10 @@
 //! This avoids having to repeatedly clone a partly-built [`UserTypeProjections`]
 //! at every step of the traversal, which is what the previous code was doing.
 
-use std::assert_matches::assert_matches;
 use std::iter;
 
 use rustc_abi::{FieldIdx, VariantIdx};
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::smallvec::SmallVec;
 use rustc_middle::mir::{ProjectionElem, UserTypeProjection, UserTypeProjections};
 use rustc_middle::ty::{AdtDef, UserTypeAnnotationIndex};
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index 650650cbaac9..d0abb6396145 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -4,11 +4,11 @@ mod check_match;
 mod const_to_pat;
 mod migration;
 
-use std::assert_matches::assert_matches;
 use std::cmp::Ordering;
 use std::sync::Arc;
 
 use rustc_abi::{FieldIdx, Integer};
+use rustc_data_structures::assert_matches;
 use rustc_errors::codes::*;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::pat_util::EnumerateAndAdjustIterator;
diff --git a/compiler/rustc_mir_build/src/thir/util.rs b/compiler/rustc_mir_build/src/thir/util.rs
index 457957f5fce9..0093bb762110 100644
--- a/compiler/rustc_mir_build/src/thir/util.rs
+++ b/compiler/rustc_mir_build/src/thir/util.rs
@@ -1,5 +1,4 @@
-use std::assert_matches::assert_matches;
-
+use rustc_data_structures::assert_matches;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_middle::bug;
diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized.rs b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
index 9216106b6edd..6a0881ec2bcb 100644
--- a/compiler/rustc_mir_dataflow/src/impls/initialized.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
@@ -1,6 +1,5 @@
-use std::assert_matches::assert_matches;
-
 use rustc_abi::VariantIdx;
+use rustc_data_structures::assert_matches;
 use rustc_index::Idx;
 use rustc_index::bit_set::{DenseBitSet, MixedBitSet};
 use rustc_middle::bug;
diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs
index ac94ee5c8104..f102b7bb50f5 100644
--- a/compiler/rustc_mir_dataflow/src/value_analysis.rs
+++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs
@@ -1,8 +1,8 @@
-use std::assert_matches::debug_assert_matches;
 use std::fmt::{Debug, Formatter};
 use std::ops::Range;
 
 use rustc_abi::{FieldIdx, VariantIdx};
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet, StdEntry};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_index::IndexVec;
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index 5254f60a1503..604f1da1a3ab 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -2,7 +2,6 @@
 //!
 //! Currently, this pass only propagates scalar values.
 
-use std::assert_matches::assert_matches;
 use std::cell::RefCell;
 use std::fmt::Formatter;
 
@@ -11,6 +10,7 @@ use rustc_const_eval::const_eval::{DummyMachine, throw_machine_stop_str};
 use rustc_const_eval::interpret::{
     ImmTy, Immediate, InterpCx, OpTy, PlaceTy, Projectable, interp_ok,
 };
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def::DefKind;
 use rustc_middle::bug;
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 1e9665f4337d..179ada36be75 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -1,10 +1,10 @@
 //! Inlining pass for MIR functions.
 
-use std::assert_matches::debug_assert_matches;
 use std::iter;
 use std::ops::{Range, RangeFrom};
 
 use rustc_abi::{ExternAbi, FieldIdx};
+use rustc_data_structures::debug_assert_matches;
 use rustc_hir::attrs::{InlineAttr, OptimizeAttr};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs
index 11266ccc2832..2cc8e9d6739c 100644
--- a/compiler/rustc_mir_transform/src/promote_consts.rs
+++ b/compiler/rustc_mir_transform/src/promote_consts.rs
@@ -10,12 +10,12 @@
 //! otherwise silence errors, if move analysis runs after promotion on broken
 //! MIR.
 
-use std::assert_matches::assert_matches;
 use std::cell::Cell;
 use std::{cmp, iter, mem};
 
 use either::{Left, Right};
 use rustc_const_eval::check_consts::{ConstCx, qualifs};
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir as hir;
 use rustc_index::{IndexSlice, IndexVec};
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 85e340c0a02a..cc1cb3d4f3ff 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -1,7 +1,7 @@
-use std::assert_matches::assert_matches;
 use std::{fmt, iter};
 
 use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx, VariantIdx};
+use rustc_data_structures::assert_matches;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItem;
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index d6e99bc540f7..c1ab8d76380a 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -16,7 +16,6 @@ mod ty;
 pub mod asm;
 pub mod cfg_select;
 
-use std::assert_matches::debug_assert_matches;
 use std::{fmt, mem, slice};
 
 use attr_wrapper::{AttrWrapper, UsePreAttrPos};
@@ -40,6 +39,7 @@ use rustc_ast::{
     Mutability, Recovered, Safety, StrLit, Visibility, VisibilityKind,
 };
 use rustc_ast_pretty::pprust;
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Applicability, Diag, FatalError, MultiSpan, PResult};
 use rustc_index::interval::IntervalSet;
diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs
index 62e97c0c308c..2e1acca9e9af 100644
--- a/compiler/rustc_parse/src/parser/tests.rs
+++ b/compiler/rustc_parse/src/parser/tests.rs
@@ -1,5 +1,4 @@
 #![allow(rustc::symbol_intern_string_literal)]
-use std::assert_matches::assert_matches;
 use std::io::prelude::*;
 use std::iter::Peekable;
 use std::path::PathBuf;
@@ -11,6 +10,7 @@ use rustc_ast::token::{self, Delimiter, Token};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_ast::{self as ast, PatKind, visit};
 use rustc_ast_pretty::pprust::item_to_string;
+use rustc_data_structures::assert_matches;
 use rustc_errors::annotate_snippet_emitter_writer::AnnotateSnippetEmitter;
 use rustc_errors::emitter::{HumanEmitter, OutputTheme};
 use rustc_errors::translation::Translator;
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 0b50d376b552..f32f3f78c04b 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -1,4 +1,3 @@
-use std::assert_matches::assert_matches;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
@@ -7,12 +6,12 @@ use std::sync::atomic::{AtomicU32, Ordering};
 
 use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::outline;
 use rustc_data_structures::profiling::QueryInvocationId;
 use rustc_data_structures::sharded::{self, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{AtomicU64, Lock};
 use rustc_data_structures::unord::UnordMap;
+use rustc_data_structures::{assert_matches, outline};
 use rustc_errors::DiagInner;
 use rustc_index::IndexVec;
 use rustc_macros::{Decodable, Encodable};
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 6557e1dea1a1..963bee369f6b 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -6,7 +6,6 @@
 //! If you wonder why there's no `early.rs`, that's because it's split into three files -
 //! `build_reduced_graph.rs`, `macros.rs` and `imports.rs`.
 
-use std::assert_matches::debug_assert_matches;
 use std::borrow::Cow;
 use std::collections::hash_map::Entry;
 use std::mem::{replace, swap, take};
@@ -16,6 +15,7 @@ use rustc_ast::visit::{
     AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor, try_visit, visit_opt, walk_list,
 };
 use rustc_ast::*;
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_data_structures::unord::{UnordMap, UnordSet};
 use rustc_errors::codes::*;
diff --git a/compiler/rustc_symbol_mangling/src/export.rs b/compiler/rustc_symbol_mangling/src/export.rs
index 3896e06a627b..c99ba1d58f31 100644
--- a/compiler/rustc_symbol_mangling/src/export.rs
+++ b/compiler/rustc_symbol_mangling/src/export.rs
@@ -1,6 +1,5 @@
-use std::assert_matches::debug_assert_matches;
-
 use rustc_abi::IntegerType;
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_hashes::Hash128;
 use rustc_hir::def::DefKind;
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs
index ef50dafec950..1c08b5e33142 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs
@@ -1,12 +1,12 @@
 // ignore-tidy-filelength
 
-use std::assert_matches::debug_assert_matches;
 use std::borrow::Cow;
 use std::iter;
 use std::path::PathBuf;
 
 use itertools::{EitherOrBoth, Itertools};
 use rustc_abi::ExternAbi;
+use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::codes::*;
diff --git a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
index 944d57bf95d1..ea8360c10b6f 100644
--- a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
+++ b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
@@ -9,8 +9,7 @@
 //! coherence right now and was annoying to implement, so I am leaving it
 //! as is until we start using it for something else.
 
-use std::assert_matches::assert_matches;
-
+use rustc_data_structures::assert_matches;
 use rustc_infer::infer::InferCtxt;
 use rustc_infer::traits::Obligation;
 use rustc_macros::extension;
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index d6c9adfb2817..787dd4ea6254 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -2,13 +2,13 @@
 //!
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/resolution.html#selection
 
-use std::assert_matches::assert_matches;
 use std::cell::{Cell, RefCell};
 use std::cmp;
 use std::fmt::{self, Display};
 use std::ops::ControlFlow;
 
 use hir::def::DefKind;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{Diag, EmissionGuarantee};
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index ad621c67772c..34c080c4938f 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -1,8 +1,8 @@
-use std::assert_matches::assert_matches;
 use std::iter;
 
 use rustc_abi::Primitive::Pointer;
 use rustc_abi::{BackendRepr, ExternAbi, PointerKind, Scalar, Size};
+use rustc_data_structures::assert_matches;
 use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::bug;
diff --git a/compiler/rustc_ty_utils/src/layout/invariant.rs b/compiler/rustc_ty_utils/src/layout/invariant.rs
index b013902f3fe3..97c98d0c2403 100644
--- a/compiler/rustc_ty_utils/src/layout/invariant.rs
+++ b/compiler/rustc_ty_utils/src/layout/invariant.rs
@@ -1,6 +1,5 @@
-use std::assert_matches::assert_matches;
-
 use rustc_abi::{BackendRepr, FieldsShape, Scalar, Size, TagEncoding, Variants};
+use rustc_data_structures::assert_matches;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutCx, TyAndLayout};
 

From 29aad0493b082c5d36a006fa39656d61b2c3ddc3 Mon Sep 17 00:00:00 2001
From: Chayim Refael Friedman 
Date: Mon, 19 Jan 2026 09:58:15 +0200
Subject: [PATCH 0381/1843] Cache `Clauses::empty()`

---
 .../rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs
index 5758e2dc7e93..6f4fae707317 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs
@@ -273,9 +273,8 @@ impl<'db> std::fmt::Debug for Clauses<'db> {
 
 impl<'db> Clauses<'db> {
     #[inline]
-    pub fn empty(_interner: DbInterner<'db>) -> Self {
-        // FIXME: Get from a static.
-        Self::new_from_slice(&[])
+    pub fn empty(interner: DbInterner<'db>) -> Self {
+        interner.default_types().empty.clauses
     }
 
     #[inline]

From 03b8b68073bd9273b9614830aaefe5b7f5d603c3 Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 20:30:48 +0100
Subject: [PATCH 0382/1843] Port `#[rustc_nounwind]` to attr parser

---
 .../src/attributes/rustc_internal.rs              | 15 +++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs        |  5 +++--
 compiler/rustc_codegen_ssa/src/codegen_attrs.rs   |  4 +++-
 compiler/rustc_hir/src/attrs/data_structures.rs   |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_passes/src/check_attr.rs           |  2 +-
 6 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 834b1d988cb4..f8ee011f559b 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -320,3 +320,18 @@ impl NoArgsAttributeParser for RustcHasIncoherentInherentImplsParse
     ]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcHasIncoherentInherentImpls;
 }
+
+pub(crate) struct RustcNounwindParser;
+
+impl NoArgsAttributeParser for RustcNounwindParser {
+    const PATH: &[Symbol] = &[sym::rustc_nounwind];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::ForeignFn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNounwind;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 1a06f4edeaa6..e62dbc124eab 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -78,8 +78,8 @@ use crate::attributes::rustc_internal::{
     RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser, RustcLintOptTyParser,
     RustcLintQueryInstabilityParser, RustcLintUntrackedQueryInformationParser, RustcMainParser,
     RustcMustImplementOneOfParser, RustcNeverReturnsNullPointerParser,
-    RustcNoImplicitAutorefsParser, RustcObjectLifetimeDefaultParser, RustcScalableVectorParser,
-    RustcSimdMonomorphizeLaneLimitParser,
+    RustcNoImplicitAutorefsParser, RustcNounwindParser, RustcObjectLifetimeDefaultParser,
+    RustcScalableVectorParser, RustcSimdMonomorphizeLaneLimitParser,
 };
 use crate::attributes::semantics::MayDangleParser;
 use crate::attributes::stability::{
@@ -296,6 +296,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 1b0427e7e676..e29ba9c8725f 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -347,6 +347,9 @@ fn process_builtin_attrs(
                 AttributeKind::RustcAllocatorZeroed => {
                     codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
                 }
+                AttributeKind::RustcNounwind => {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND
+                }
                 _ => {}
             }
         }
@@ -356,7 +359,6 @@ fn process_builtin_attrs(
         };
 
         match name {
-            sym::rustc_nounwind => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND,
             sym::patchable_function_entry => {
                 codegen_fn_attrs.patchable_function_entry =
                     parse_patchable_function_entry(tcx, attr);
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 5cd508acc567..379ebf648f47 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -981,6 +981,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_no_implicit_autorefs]`
     RustcNoImplicitAutorefs,
 
+    /// Represents `#[rustc_nounwind]`
+    RustcNounwind,
+
     /// Represents `#[rustc_object_lifetime_default]`.
     RustcObjectLifetimeDefault,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 19243c7d8468..44f040e5fdc8 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -122,6 +122,7 @@ impl AttributeKind {
             RustcMustImplementOneOf { .. } => No,
             RustcNeverReturnsNullPointer => Yes,
             RustcNoImplicitAutorefs => Yes,
+            RustcNounwind => No,
             RustcObjectLifetimeDefault => No,
             RustcPassIndirectlyInNonRusticAbis(..) => No,
             RustcReallocator => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 3a086ba5ff78..74edff7badba 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -322,6 +322,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcAllocatorZeroedVariant { .. }
                     | AttributeKind::RustcDeallocator
                     | AttributeKind::RustcReallocator
+                    | AttributeKind::RustcNounwind
                 ) => { /* do nothing  */ }
                 Attribute::Unparsed(attr_item) => {
                     style = Some(attr_item.style);
@@ -390,7 +391,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_partition_reused
                             | sym::rustc_partition_codegened
                             | sym::rustc_expected_cgu_reuse
-                            | sym::rustc_nounwind
                             // crate-level attrs, are checked below
                             | sym::feature
                             | sym::register_tool

From 3e731f7e84301a898a36e46ee5e4845ff9bda98a Mon Sep 17 00:00:00 2001
From: Edvin Bryntesson 
Date: Sun, 18 Jan 2026 20:41:29 +0100
Subject: [PATCH 0383/1843] Port `#[rustc_offload_kernel]` to attr parser

---
 .../rustc_attr_parsing/src/attributes/rustc_internal.rs  | 9 +++++++++
 compiler/rustc_attr_parsing/src/context.rs               | 3 ++-
 compiler/rustc_codegen_ssa/src/codegen_attrs.rs          | 6 +++---
 compiler/rustc_hir/src/attrs/data_structures.rs          | 3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs       | 1 +
 compiler/rustc_passes/src/check_attr.rs                  | 1 +
 6 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index f8ee011f559b..51792b7aff72 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -335,3 +335,12 @@ impl NoArgsAttributeParser for RustcNounwindParser {
     ]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNounwind;
 }
+
+pub(crate) struct RustcOffloadKernelParser;
+
+impl NoArgsAttributeParser for RustcOffloadKernelParser {
+    const PATH: &[Symbol] = &[sym::rustc_offload_kernel];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcOffloadKernel;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index e62dbc124eab..0c882fee01c8 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -79,7 +79,7 @@ use crate::attributes::rustc_internal::{
     RustcLintQueryInstabilityParser, RustcLintUntrackedQueryInformationParser, RustcMainParser,
     RustcMustImplementOneOfParser, RustcNeverReturnsNullPointerParser,
     RustcNoImplicitAutorefsParser, RustcNounwindParser, RustcObjectLifetimeDefaultParser,
-    RustcScalableVectorParser, RustcSimdMonomorphizeLaneLimitParser,
+    RustcOffloadKernelParser, RustcScalableVectorParser, RustcSimdMonomorphizeLaneLimitParser,
 };
 use crate::attributes::semantics::MayDangleParser;
 use crate::attributes::stability::{
@@ -297,6 +297,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index e29ba9c8725f..e35d884b6711 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -350,6 +350,9 @@ fn process_builtin_attrs(
                 AttributeKind::RustcNounwind => {
                     codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND
                 }
+                AttributeKind::RustcOffloadKernel => {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::OFFLOAD_KERNEL
+                }
                 _ => {}
             }
         }
@@ -363,9 +366,6 @@ fn process_builtin_attrs(
                 codegen_fn_attrs.patchable_function_entry =
                     parse_patchable_function_entry(tcx, attr);
             }
-            sym::rustc_offload_kernel => {
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::OFFLOAD_KERNEL
-            }
             _ => {}
         }
     }
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 379ebf648f47..314f36d6132d 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -987,6 +987,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_object_lifetime_default]`.
     RustcObjectLifetimeDefault,
 
+    /// Represents `#[rustc_offload_kernel]`
+    RustcOffloadKernel,
+
     /// Represents `#[rustc_pass_indirectly_in_non_rustic_abis]`
     RustcPassIndirectlyInNonRusticAbis(Span),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 44f040e5fdc8..b55a5d0e29e1 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -124,6 +124,7 @@ impl AttributeKind {
             RustcNoImplicitAutorefs => Yes,
             RustcNounwind => No,
             RustcObjectLifetimeDefault => No,
+            RustcOffloadKernel => Yes,
             RustcPassIndirectlyInNonRusticAbis(..) => No,
             RustcReallocator => No,
             RustcScalableVector { .. } => Yes,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 74edff7badba..8074ae429892 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -323,6 +323,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDeallocator
                     | AttributeKind::RustcReallocator
                     | AttributeKind::RustcNounwind
+                    | AttributeKind::RustcOffloadKernel
                 ) => { /* do nothing  */ }
                 Attribute::Unparsed(attr_item) => {
                     style = Some(attr_item.style);

From eeed3376e25905b6b1b42ee1fa1af2e75f4482f0 Mon Sep 17 00:00:00 2001
From: KaiTomotake 
Date: Mon, 19 Jan 2026 17:06:07 +0900
Subject: [PATCH 0384/1843] add test program

A test for the issue where the variable meta is mistakenly treated as a reserved keyword.
---
 tests/ui/reserved/meta-is-not-reserved.rs     | 7 +++++++
 tests/ui/reserved/meta-is-not-reserved.stderr | 9 +++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 tests/ui/reserved/meta-is-not-reserved.rs
 create mode 100644 tests/ui/reserved/meta-is-not-reserved.stderr

diff --git a/tests/ui/reserved/meta-is-not-reserved.rs b/tests/ui/reserved/meta-is-not-reserved.rs
new file mode 100644
index 000000000000..ceefe345ff0c
--- /dev/null
+++ b/tests/ui/reserved/meta-is-not-reserved.rs
@@ -0,0 +1,7 @@
+// Regression test for 
+// A test for the issue where the variable meta is mistakenly treated as a reserved keyword.
+
+fn main() {
+    let xyz = meta;
+    //~^ ERROR cannot find value `meta` in this scope [E0425]
+}
diff --git a/tests/ui/reserved/meta-is-not-reserved.stderr b/tests/ui/reserved/meta-is-not-reserved.stderr
new file mode 100644
index 000000000000..33f8fd82df85
--- /dev/null
+++ b/tests/ui/reserved/meta-is-not-reserved.stderr
@@ -0,0 +1,9 @@
+error[E0425]: cannot find value `meta` in this scope
+  --> $DIR/meta-is-not-reserved.rs:5:15
+   |
+LL |     let xyz = meta;
+   |               ^^^^ not found in this scope
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0425`.

From 345e9a089bde20917fb7cf1f9db5092d69d70b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Fri, 16 Jan 2026 22:04:53 +0100
Subject: [PATCH 0385/1843] Document `-Zcache-proc-macros`

---
 .../src/compiler-flags/cache-proc-macros.md              | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 src/doc/unstable-book/src/compiler-flags/cache-proc-macros.md

diff --git a/src/doc/unstable-book/src/compiler-flags/cache-proc-macros.md b/src/doc/unstable-book/src/compiler-flags/cache-proc-macros.md
new file mode 100644
index 000000000000..eefaf9ca2f1b
--- /dev/null
+++ b/src/doc/unstable-book/src/compiler-flags/cache-proc-macros.md
@@ -0,0 +1,9 @@
+## `cache-proc-macros`
+
+The tracking issue for this feature is: [#151364]
+
+[#151364]: https://github.com/rust-lang/rust/issues/151364
+
+------------------------
+
+This option instructs `rustc` to cache (derive) proc-macro invocations using the incremental system. Note that the compiler does not currently check whether the proc-macro is actually "cacheable" or not. If you use this flag when compiling a crate that uses non-pure proc-macros, it can result in stale expansions being compiled.

From 67c45b739af089b9c8d96f0706c22b899a2ac324 Mon Sep 17 00:00:00 2001
From: JayanAXHF 
Date: Sat, 10 Jan 2026 01:37:51 +0530
Subject: [PATCH 0386/1843] feat: added syntax highlighting for code blocks in
 `rustc --explain`

This commit adds a heuristics-based syntax highlighter for the `rustc
--explain` command. It uses `rsutc_lexer`'s lexer to parse input in
tokens, and matches on them to determine their color.
---
 Cargo.lock                                    |   2 +
 compiler/rustc_driver_impl/Cargo.toml         |   2 +
 compiler/rustc_driver_impl/src/highlighter.rs | 159 ++++++++++++++++++
 compiler/rustc_driver_impl/src/lib.rs         |   7 +-
 compiler/rustc_errors/src/markdown/mod.rs     |  11 +-
 compiler/rustc_errors/src/markdown/term.rs    |  41 +++--
 .../src/markdown/tests/output.stdout          |  10 +-
 .../rustc_errors/src/markdown/tests/term.rs   |   2 +-
 tests/ui-fulldeps/explain_highlighter.rs      |  29 ++++
 .../explain_highlighter.run.stdout            |  12 ++
 10 files changed, 249 insertions(+), 26 deletions(-)
 create mode 100644 compiler/rustc_driver_impl/src/highlighter.rs
 create mode 100644 tests/ui-fulldeps/explain_highlighter.rs
 create mode 100644 tests/ui-fulldeps/explain_highlighter.run.stdout

diff --git a/Cargo.lock b/Cargo.lock
index 470b38d5a91c..2d23541b4314 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3809,6 +3809,7 @@ dependencies = [
 name = "rustc_driver_impl"
 version = "0.0.0"
 dependencies = [
+ "anstyle",
  "ctrlc",
  "jiff",
  "libc",
@@ -3834,6 +3835,7 @@ dependencies = [
  "rustc_index",
  "rustc_infer",
  "rustc_interface",
+ "rustc_lexer",
  "rustc_lint",
  "rustc_log",
  "rustc_macros",
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index 531b9e0c8ff7..c160240a18a7 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2024"
 
 [dependencies]
 # tidy-alphabetical-start
+anstyle = "1.0.13"
 jiff = { version = "0.2.5", default-features = false, features = ["std"] }
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
@@ -28,6 +29,7 @@ rustc_incremental = { path = "../rustc_incremental" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_interface = { path = "../rustc_interface" }
+rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint = { path = "../rustc_lint" }
 rustc_log = { path = "../rustc_log" }
 rustc_macros = { path = "../rustc_macros" }
diff --git a/compiler/rustc_driver_impl/src/highlighter.rs b/compiler/rustc_driver_impl/src/highlighter.rs
new file mode 100644
index 000000000000..70b73cc29b72
--- /dev/null
+++ b/compiler/rustc_driver_impl/src/highlighter.rs
@@ -0,0 +1,159 @@
+//! This module provides a syntax highlighter for Rust code.
+//! It is used by the `rustc --explain` command.
+//!
+//! The syntax highlighter uses `rustc_lexer`'s `tokenize`
+//! function to parse the Rust code into a `Vec` of tokens.
+//! The highlighter then highlights the tokens in the `Vec`,
+//! and writes the highlighted output to the buffer.
+use std::io::{self, Write};
+
+use anstyle::{AnsiColor, Color, Effects, Style};
+use rustc_lexer::{LiteralKind, strip_shebang, tokenize};
+
+const PRIMITIVE_TYPES: &'static [&str] = &[
+    "i8", "i16", "i32", "i64", "i128", "isize", // signed integers
+    "u8", "u16", "u32", "u64", "u128", "usize", // unsigned integers
+    "f32", "f64", // floating point
+    "char", "bool", // others
+];
+
+const KEYWORDS: &'static [&str] = &[
+    "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while", "as",
+    "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern",
+    "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
+    "ref",
+];
+
+const STR_LITERAL_COLOR: AnsiColor = AnsiColor::Green;
+const OTHER_LITERAL_COLOR: AnsiColor = AnsiColor::BrightRed;
+const DERIVE_COLOR: AnsiColor = AnsiColor::BrightRed;
+const KEYWORD_COLOR: AnsiColor = AnsiColor::BrightMagenta;
+const TYPE_COLOR: AnsiColor = AnsiColor::Yellow;
+const FUNCTION_COLOR: AnsiColor = AnsiColor::Blue;
+const USE_COLOR: AnsiColor = AnsiColor::BrightMagenta;
+const PRIMITIVE_TYPE_COLOR: AnsiColor = AnsiColor::Cyan;
+
+/// Highlight a Rust code string and write the highlighted
+/// output to the buffer. It serves as a wrapper around
+/// `Highlighter::highlight_rustc_lexer`. It is passed to
+/// `write_anstream_buf` in the `lib.rs` file.
+pub fn highlight(code: &str, buf: &mut Vec) -> io::Result<()> {
+    let mut highlighter = Highlighter::default();
+    highlighter.highlight_rustc_lexer(code, buf)
+}
+
+/// A syntax highlighter for Rust code
+/// It is used by the `rustc --explain` command.
+#[derive(Default)]
+pub struct Highlighter {
+    /// Used to track if the previous token was a token
+    /// that warrants the next token to be colored differently
+    ///
+    /// For example, the keyword `fn` requires the next token
+    /// (the function name) to be colored differently.
+    prev_was_special: bool,
+    /// Used to track the length of tokens that have been
+    /// written so far. This is used to find the original
+    /// lexeme for a token from the code string.
+    len_accum: usize,
+}
+
+impl Highlighter {
+    /// Create a new highlighter
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Highlight a Rust code string and write the highlighted
+    /// output to the buffer.
+    pub fn highlight_rustc_lexer(&mut self, code: &str, buf: &mut Vec) -> io::Result<()> {
+        use rustc_lexer::TokenKind;
+
+        // Remove shebang from code string
+        let stripped_idx = strip_shebang(code).unwrap_or(0);
+        let stripped_code = &code[stripped_idx..];
+        self.len_accum = stripped_idx;
+        let len_accum = &mut self.len_accum;
+        let tokens = tokenize(stripped_code, rustc_lexer::FrontmatterAllowed::No);
+        for token in tokens {
+            let len = token.len as usize;
+            // If the previous token was a special token, and this token is
+            // not a whitespace token, then it should be colored differently
+            let token_str = &code[*len_accum..*len_accum + len];
+            if self.prev_was_special {
+                if token_str != " " {
+                    self.prev_was_special = false;
+                }
+                let style = Style::new().fg_color(Some(Color::Ansi(AnsiColor::Blue)));
+                write!(buf, "{style}{token_str}{style:#}")?;
+                *len_accum += len;
+                continue;
+            }
+            match token.kind {
+                TokenKind::Ident => {
+                    let mut style = Style::new();
+                    // Match if an identifier is a (well-known) keyword
+                    if KEYWORDS.contains(&token_str) {
+                        if token_str == "fn" {
+                            self.prev_was_special = true;
+                        }
+                        style = style.fg_color(Some(Color::Ansi(KEYWORD_COLOR)));
+                    }
+                    // The `use` keyword is colored differently
+                    if matches!(token_str, "use") {
+                        style = style.fg_color(Some(Color::Ansi(USE_COLOR)));
+                    }
+                    // This heuristic test is to detect if the identifier is
+                    // a function call. If it is, then the function identifier is
+                    // colored differently.
+                    if code[*len_accum..*len_accum + len + 1].ends_with('(') {
+                        style = style.fg_color(Some(Color::Ansi(FUNCTION_COLOR)));
+                    }
+                    // The `derive` keyword is colored differently.
+                    if token_str == "derive" {
+                        style = style.fg_color(Some(Color::Ansi(DERIVE_COLOR)));
+                    }
+                    // This heuristic test is to detect if the identifier is
+                    // a type. If it is, then the identifier is colored differently.
+                    if matches!(token_str.chars().next().map(|c| c.is_uppercase()), Some(true)) {
+                        style = style.fg_color(Some(Color::Ansi(TYPE_COLOR)));
+                    }
+                    // This if statement is to detect if the identifier is a primitive type.
+                    if PRIMITIVE_TYPES.contains(&token_str) {
+                        style = style.fg_color(Some(Color::Ansi(PRIMITIVE_TYPE_COLOR)));
+                    }
+                    write!(buf, "{style}{token_str}{style:#}")?;
+                }
+
+                // Color literals
+                TokenKind::Literal { kind, suffix_start: _ } => {
+                    // Strings -> Green
+                    // Chars -> Green
+                    // Raw strings -> Green
+                    // C strings -> Green
+                    // Byte Strings -> Green
+                    // Other literals -> Bright Red (Orage-esque)
+                    let style = match kind {
+                        LiteralKind::Str { terminated: _ }
+                        | LiteralKind::Char { terminated: _ }
+                        | LiteralKind::RawStr { n_hashes: _ }
+                        | LiteralKind::CStr { terminated: _ } => {
+                            Style::new().fg_color(Some(Color::Ansi(STR_LITERAL_COLOR)))
+                        }
+                        _ => Style::new().fg_color(Some(Color::Ansi(OTHER_LITERAL_COLOR))),
+                    };
+                    write!(buf, "{style}{token_str}{style:#}")?;
+                }
+                _ => {
+                    // All other tokens are dimmed
+                    let style = Style::new()
+                        .fg_color(Some(Color::Ansi(AnsiColor::BrightWhite)))
+                        .effects(Effects::DIMMED);
+                    write!(buf, "{style}{token_str}{style:#}")?;
+                }
+            }
+            *len_accum += len;
+        }
+        Ok(())
+    }
+}
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 7820198f2dcf..045292338e58 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -86,6 +86,7 @@ pub mod args;
 pub mod pretty;
 #[macro_use]
 mod print;
+pub mod highlighter;
 mod session_diagnostics;
 
 // Keep the OS parts of this `cfg` in sync with the `cfg` on the `libc`
@@ -521,7 +522,11 @@ fn show_md_content_with_pager(content: &str, color: ColorConfig) {
         let mdstream = markdown::MdStream::parse_str(content);
         let bufwtr = markdown::create_stdout_bufwtr();
         let mut mdbuf = Vec::new();
-        if mdstream.write_anstream_buf(&mut mdbuf).is_ok() { Some((bufwtr, mdbuf)) } else { None }
+        if mdstream.write_anstream_buf(&mut mdbuf, Some(&highlighter::highlight)).is_ok() {
+            Some((bufwtr, mdbuf))
+        } else {
+            None
+        }
     };
 
     // Try to print via the pager, pretty output if possible.
diff --git a/compiler/rustc_errors/src/markdown/mod.rs b/compiler/rustc_errors/src/markdown/mod.rs
index 4f5e2328234d..9993407b05c0 100644
--- a/compiler/rustc_errors/src/markdown/mod.rs
+++ b/compiler/rustc_errors/src/markdown/mod.rs
@@ -18,9 +18,14 @@ impl<'a> MdStream<'a> {
         parse::entrypoint(s)
     }
 
-    /// Write formatted output to an anstream buffer
-    pub fn write_anstream_buf(&self, buf: &mut Vec) -> io::Result<()> {
-        term::entrypoint(self, buf)
+    /// Write formatted output to a stdout buffer, optionally with
+    /// a formatter for code blocks
+    pub fn write_anstream_buf(
+        &self,
+        buf: &mut Vec,
+        formatter: Option<&(dyn Fn(&str, &mut Vec) -> io::Result<()> + 'static)>,
+    ) -> io::Result<()> {
+        term::entrypoint(self, buf, formatter)
     }
 }
 
diff --git a/compiler/rustc_errors/src/markdown/term.rs b/compiler/rustc_errors/src/markdown/term.rs
index b0ce01548f00..b94cd06b30ef 100644
--- a/compiler/rustc_errors/src/markdown/term.rs
+++ b/compiler/rustc_errors/src/markdown/term.rs
@@ -12,29 +12,33 @@ thread_local! {
     static CURSOR: Cell = const { Cell::new(0) };
     /// Width of the terminal
     static WIDTH: Cell = const { Cell::new(DEFAULT_COLUMN_WIDTH) };
+
 }
 
-/// Print to terminal output to a buffer
-pub(crate) fn entrypoint(stream: &MdStream<'_>, buf: &mut Vec) -> io::Result<()> {
-    #[cfg(not(test))]
-    if let Some((w, _)) = termize::dimensions() {
-        WIDTH.set(std::cmp::min(w, DEFAULT_COLUMN_WIDTH));
-    }
-    write_stream(stream, buf, None, 0)?;
+/// Print to the terminal output to a buffer
+/// optionally with a formatter for code blocks
+pub(crate) fn entrypoint(
+    stream: &MdStream<'_>,
+    buf: &mut Vec,
+    formatter: Option<&(dyn Fn(&str, &mut Vec) -> io::Result<()> + 'static)>,
+) -> io::Result<()> {
+    write_stream(stream, buf, None, 0, formatter)?;
     buf.write_all(b"\n")
 }
-/// Write the buffer, reset to the default style after each
+
+/// Write the buffer, reset to the default style after each,
+/// optionally with a formatter for code blocks
 fn write_stream(
     MdStream(stream): &MdStream<'_>,
     buf: &mut Vec,
+
     default: Option
 
fn main() {
-    let hello = "hello";
-    let x = hello.to_string();
-    let y = hello.to_string();
+    let hello = "hello";
+    let x = hello.to_string();
+    let y = hello.to_string();
 
-    let x = "other color please!";
-    let y = x.to_string();
+    let x = "other color please!";
+    let y = x.to_string();
 }
 
 fn bar() {
-    let mut hello = "hello";
+    let mut hello = "hello";
 }

\ No newline at end of file

From 9a114c686f9e618a217f54920530bcbe66cb8a04 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 17:13:30 +0100
Subject: [PATCH 1389/1843] Convert to inline diagnostics in `rustc_parse`

---
 Cargo.lock                                    |    1 -
 compiler/rustc_driver_impl/src/lib.rs         |    1 -
 compiler/rustc_interface/src/interface.rs     |   14 +-
 .../rustc_macros/src/diagnostics/message.rs   |    3 +-
 compiler/rustc_parse/Cargo.toml               |    1 -
 compiler/rustc_parse/messages.ftl             | 1043 ---------
 compiler/rustc_parse/src/errors.rs            | 2012 +++++++++++------
 compiler/rustc_parse/src/lib.rs               |    2 -
 compiler/rustc_parse/src/parser/attr.rs       |   54 +-
 .../rustc_parse/src/parser/diagnostics.rs     |    6 +-
 compiler/rustc_parse/src/parser/item.rs       |    6 +-
 compiler/rustc_parse/src/parser/tests.rs      |   24 +-
 compiler/rustc_session/src/parse.rs           |    4 +-
 .../ui-fulldeps/mod_dir_path_canonicalized.rs |    2 +-
 tests/ui-fulldeps/pprust-expr-roundtrip.rs    |    2 +-
 .../pprust-parenthesis-insertion.rs           |    2 +-
 16 files changed, 1401 insertions(+), 1776 deletions(-)
 delete mode 100644 compiler/rustc_parse/messages.ftl

diff --git a/Cargo.lock b/Cargo.lock
index 1d2289357300..54822628ca45 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4381,7 +4381,6 @@ dependencies = [
  "rustc_data_structures",
  "rustc_errors",
  "rustc_feature",
- "rustc_fluent_macro",
  "rustc_index",
  "rustc_lexer",
  "rustc_macros",
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 3685d4ee7ebb..315b11056085 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -114,7 +114,6 @@ pub fn default_translator() -> Translator {
 pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
     // tidy-alphabetical-start
     rustc_lint::DEFAULT_LOCALE_RESOURCE,
-    rustc_parse::DEFAULT_LOCALE_RESOURCE,
     // tidy-alphabetical-end
 ];
 
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 3dbe1057c19d..722e9a420540 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -53,10 +53,9 @@ pub struct Compiler {
 pub(crate) fn parse_cfg(dcx: DiagCtxtHandle<'_>, cfgs: Vec) -> Cfg {
     cfgs.into_iter()
         .map(|s| {
-            let psess = ParseSess::emitter_with_note(
-                vec![rustc_parse::DEFAULT_LOCALE_RESOURCE],
-                format!("this occurred on the command line: `--cfg={s}`"),
-            );
+            let psess = ParseSess::emitter_with_note(format!(
+                "this occurred on the command line: `--cfg={s}`"
+            ));
             let filename = FileName::cfg_spec_source_code(&s);
 
             macro_rules! error {
@@ -125,10 +124,9 @@ pub(crate) fn parse_check_cfg(dcx: DiagCtxtHandle<'_>, specs: Vec) -> Ch
     let mut check_cfg = CheckCfg { exhaustive_names, exhaustive_values, ..CheckCfg::default() };
 
     for s in specs {
-        let psess = ParseSess::emitter_with_note(
-            vec![rustc_parse::DEFAULT_LOCALE_RESOURCE],
-            format!("this occurred on the command line: `--check-cfg={s}`"),
-        );
+        let psess = ParseSess::emitter_with_note(format!(
+            "this occurred on the command line: `--check-cfg={s}`"
+        ));
         let filename = FileName::cfg_spec_source_code(&s);
 
         const VISIT: &str =
diff --git a/compiler/rustc_macros/src/diagnostics/message.rs b/compiler/rustc_macros/src/diagnostics/message.rs
index 948cc9e93064..ad3ca4d1f8ea 100644
--- a/compiler/rustc_macros/src/diagnostics/message.rs
+++ b/compiler/rustc_macros/src/diagnostics/message.rs
@@ -3,6 +3,7 @@ use fluent_syntax::ast::{Expression, InlineExpression, Pattern, PatternElement};
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
 use syn::Path;
+use syn::ext::IdentExt;
 use synstructure::{Structure, VariantInfo};
 
 use crate::diagnostics::error::span_err;
@@ -100,7 +101,7 @@ fn verify_fluent_message(msg_span: Span, message_str: &str, variant: Option<&Var
             .bindings()
             .iter()
             .flat_map(|b| b.ast().ident.as_ref())
-            .map(|id| id.to_string())
+            .map(|id| id.unraw().to_string())
             .collect();
         for variable in variable_references(&message) {
             if !fields.iter().any(|f| f == variable) {
diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml
index f0c84e07a56f..28a67ae12126 100644
--- a/compiler/rustc_parse/Cargo.toml
+++ b/compiler/rustc_parse/Cargo.toml
@@ -12,7 +12,6 @@ rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_index = { path = "../rustc_index" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
deleted file mode 100644
index 3f3300117460..000000000000
--- a/compiler/rustc_parse/messages.ftl
+++ /dev/null
@@ -1,1043 +0,0 @@
-parse_add_paren = try adding parentheses
-
-parse_ambiguous_range_pattern = the range pattern here has ambiguous interpretation
-parse_ambiguous_range_pattern_suggestion = add parentheses to clarify the precedence
-
-parse_array_brackets_instead_of_braces = this is a block expression, not an array
-    .suggestion = to make an array, use square brackets instead of curly braces
-
-parse_array_index_offset_of = array indexing not supported in offset_of
-
-parse_asm_expected_comma = expected token: `,`
-    .label = expected `,`
-
-parse_asm_expected_other = expected operand, {$is_inline_asm ->
-    [false] options
-    *[true] clobber_abi, options
-    }, or additional template string
-
-parse_asm_expected_register_class_or_explicit_register = expected register class or explicit register
-
-parse_asm_expected_string_literal = expected string literal
-    .label = not a string literal
-
-parse_asm_non_abi = at least one abi must be provided as an argument to `clobber_abi`
-
-parse_asm_requires_template = requires at least a template string argument
-
-parse_asm_sym_no_path = expected a path for argument to `sym`
-
-parse_asm_underscore_input = _ cannot be used for input operands
-
-parse_asm_unsupported_operand = the `{$symbol}` operand cannot be used with `{$macro_name}!`
-    .label = the `{$symbol}` operand is not meaningful for global-scoped inline assembly, remove it
-
-parse_assignment_else_not_allowed =  ... else {"{"} ... {"}"} is not allowed
-
-parse_associated_static_item_not_allowed = associated `static` items are not allowed
-
-parse_async_block_in_2015 = `async` blocks are only allowed in Rust 2018 or later
-
-parse_async_bound_modifier_in_2015 = `async` trait bounds are only allowed in Rust 2018 or later
-
-parse_async_fn_in_2015 = `async fn` is not permitted in Rust 2015
-    .label = to use `async fn`, switch to Rust 2018 or later
-
-parse_async_impl = `async` trait implementations are unsupported
-
-parse_async_move_block_in_2015 = `async move` blocks are only allowed in Rust 2018 or later
-
-parse_async_move_order_incorrect = the order of `move` and `async` is incorrect
-    .suggestion = try switching the order
-
-parse_async_use_block_in_2015 = `async use` blocks are only allowed in Rust 2018 or later
-
-parse_async_use_order_incorrect = the order of `use` and `async` is incorrect
-    .suggestion = try switching the order
-
-parse_at_dot_dot_in_struct_pattern = `@ ..` is not supported in struct patterns
-    .suggestion = bind to each field separately or, if you don't need them, just remove `{$ident} @`
-
-parse_at_in_struct_pattern = unexpected `@` in struct pattern
-    .note = struct patterns use `field: pattern` syntax to bind to fields
-    .help = consider replacing `new_name @ field_name` with `field_name: new_name` if that is what you intended
-
-parse_attr_after_generic = trailing attribute after generic parameter
-    .label = attributes must go before parameters
-
-parse_attr_without_generics = attribute without generic parameters
-    .label = attributes are only permitted when preceding parameters
-
-parse_attribute_on_empty_type = attributes cannot be applied here
-    .label = attributes are not allowed here
-
-parse_attribute_on_generic_arg = attributes cannot be applied to generic arguments
-    .label = attributes are not allowed here
-    .suggestion = remove attribute from here
-
-parse_attribute_on_param_type = attributes cannot be applied to a function parameter's type
-    .label = attributes are not allowed here
-
-parse_attribute_on_type = attributes cannot be applied to types
-    .label = attributes are not allowed here
-    .suggestion = remove attribute from here
-
-parse_bad_assoc_type_bounds = bounds on associated types do not belong here
-    .label = belongs in `where` clause
-
-parse_bad_item_kind = {$descr} is not supported in {$ctx}
-    .help = consider moving the {$descr} out to a nearby module scope
-
-parse_bad_return_type_notation_output =
-    return type not allowed with return type notation
-    .suggestion = remove the return type
-
-parse_bare_cr = {$double_quotes ->
-    [true] bare CR not allowed in string, use `\r` instead
-    *[false] character constant must be escaped: `\r`
-    }
-    .escape = escape the character
-
-parse_bare_cr_in_frontmatter = bare CR not allowed in frontmatter
-
-parse_bare_cr_in_raw_string = bare CR not allowed in raw string
-
-parse_binder_and_polarity = `for<...>` binder not allowed with `{$polarity}` trait polarity modifier
-    .label = there is not a well-defined meaning for a higher-ranked `{$polarity}` trait
-
-parse_binder_before_modifiers = `for<...>` binder should be placed before trait bound modifiers
-    .label = place the `for<...>` binder before any modifiers
-
-parse_bounds_not_allowed_on_trait_aliases = bounds are not allowed on trait aliases
-
-parse_box_not_pat = expected pattern, found {$descr}
-    .note = `box` is a reserved keyword
-    .suggestion = escape `box` to use it as an identifier
-
-parse_box_syntax_removed = `box_syntax` has been removed
-parse_box_syntax_removed_suggestion = use `Box::new()` instead
-
-parse_cannot_be_raw_ident = `{$ident}` cannot be a raw identifier
-
-parse_cannot_be_raw_lifetime = `{$ident}` cannot be a raw lifetime
-
-parse_catch_after_try = keyword `catch` cannot follow a `try` block
-    .help = try using `match` on the result of the `try` block instead
-
-parse_colon_as_semi = statements are terminated with a semicolon
-    .suggestion = use a semicolon instead
-
-parse_comma_after_base_struct = cannot use a comma after the base struct
-    .note = the base struct must always be the last field
-    .suggestion = remove this comma
-
-parse_comparison_interpreted_as_generic =
-    `<` is interpreted as a start of generic arguments for `{$type}`, not a comparison
-    .label_args = interpreted as generic arguments
-    .label_comparison = not interpreted as comparison
-    .suggestion = try comparing the cast value
-
-parse_comparison_operators_cannot_be_chained = comparison operators cannot be chained
-    .sugg_parentheses_for_function_args = or use `(...)` if you meant to specify fn arguments
-    .sugg_split_comparison = split the comparison into two
-    .sugg_parenthesize = parenthesize the comparison
-parse_compound_assignment_expression_in_let = can't reassign to an uninitialized variable
-    .suggestion = initialize the variable
-    .help = if you meant to overwrite, remove the `let` binding
-
-parse_const_generic_without_braces = expressions must be enclosed in braces to be used as const generic arguments
-    .suggestion = enclose the `const` expression in braces
-
-parse_const_global_cannot_be_mutable = const globals cannot be mutable
-    .label = cannot be mutable
-    .suggestion = you might want to declare a static instead
-
-parse_const_let_mutually_exclusive = `const` and `let` are mutually exclusive
-    .suggestion = remove `let`
-
-parse_cr_doc_comment = bare CR not allowed in {$block ->
-    [true] block doc-comment
-    *[false] doc-comment
-}
-
-parse_default_not_followed_by_item = `default` is not followed by an item
-    .label = the `default` qualifier
-    .note = only `fn`, `const`, `type`, or `impl` items may be prefixed by `default`
-
-parse_delegation_non_trait_impl_reuse = only trait impls can be reused
-
-parse_do_catch_syntax_removed = found removed `do catch` syntax
-    .note = following RFC #2388, the new non-placeholder syntax is `try`
-    .suggestion = replace with the new syntax
-
-parse_doc_comment_does_not_document_anything = found a documentation comment that doesn't document anything
-    .help = doc comments must come before what they document, if a comment was intended use `//`
-    .suggestion = missing comma here
-
-parse_doc_comment_on_param_type = documentation comments cannot be applied to a function parameter's type
-    .label = doc comments are not allowed here
-
-parse_dot_dot_dot_for_remaining_fields = expected field pattern, found `{$token_str}`
-    .suggestion = to omit remaining fields, use `..`
-
-parse_dot_dot_dot_range_to_pattern_not_allowed = range-to patterns with `...` are not allowed
-    .suggestion = use `..=` instead
-
-parse_dot_dot_range_attribute = attributes are not allowed on range expressions starting with `..`
-
-parse_dotdotdot = unexpected token: `...`
-    .suggest_exclusive_range = use `..` for an exclusive range
-    .suggest_inclusive_range = or `..=` for an inclusive range
-
-parse_dotdotdot_rest_pattern = unexpected `...`
-    .label = not a valid pattern
-    .suggestion = for a rest pattern, use `..` instead of `...`
-    .note = only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list
-
-parse_dotdotdot_rest_type = unexpected `...`
-    .note = only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list
-
-parse_double_colon_in_bound = expected `:` followed by trait or lifetime
-    .suggestion = use single colon
-
-parse_dyn_after_mut = `mut` must precede `dyn`
-    .suggestion = place `mut` before `dyn`
-
-parse_empty_exponent_float = expected at least one digit in exponent
-
-parse_empty_unicode_escape = empty unicode escape
-    .label = this escape must have at least 1 hex digit
-
-parse_enum_pattern_instead_of_identifier = expected identifier, found enum pattern
-
-parse_enum_struct_mutually_exclusive = `enum` and `struct` are mutually exclusive
-    .suggestion = replace `enum struct` with
-
-parse_eq_field_init = expected `:`, found `=`
-    .suggestion = replace equals symbol with a colon
-
-parse_escape_only_char = {$byte ->
-    [true] byte
-    *[false] character
-    } constant must be escaped: `{$escaped_msg}`
-    .escape = escape the character
-
-parse_expect_dotdot_not_dotdotdot = expected `..`, found `...`
-    .suggestion = use `..` to fill in the rest of the fields
-
-parse_expect_eq_instead_of_eqeq = expected `=`, found `==`
-    .suggestion = consider using `=` here
-
-parse_expect_label_found_ident = expected a label, found an identifier
-    .suggestion = labels start with a tick
-
-parse_expect_path = expected a path
-
-parse_expected_binding_left_of_at = left-hand side of `@` must be a binding
-    .label_lhs = interpreted as a pattern, not a binding
-    .label_rhs = also a pattern
-    .note = bindings are `x`, `mut x`, `ref x`, and `ref mut x`
-
-parse_expected_builtin_ident = expected identifier after `builtin #`
-
-parse_expected_comma_after_pattern_field = expected `,`
-
-parse_expected_else_block = expected `{"{"}`, found {$first_tok}
-    .label = expected an `if` or a block after this `else`
-    .suggestion = add an `if` if this is the condition of a chained `else if` statement
-
-parse_expected_expression_found_let = expected expression, found `let` statement
-    .note = only supported directly in conditions of `if` and `while` expressions
-    .not_supported_or = `||` operators are not supported in let chain expressions
-    .not_supported_parentheses = `let`s wrapped in parentheses are not supported in a context with let chains
-
-parse_expected_fn_path_found_fn_keyword = expected identifier, found keyword `fn`
-    .suggestion = use `Fn` to refer to the trait
-
-parse_expected_identifier = expected identifier
-
-parse_expected_identifier_found_doc_comment = expected identifier, found doc comment
-parse_expected_identifier_found_doc_comment_str = expected identifier, found doc comment `{$token}`
-parse_expected_identifier_found_keyword = expected identifier, found keyword
-parse_expected_identifier_found_keyword_str = expected identifier, found keyword `{$token}`
-parse_expected_identifier_found_metavar = expected identifier, found metavariable
-# This one deliberately doesn't print a token.
-parse_expected_identifier_found_metavar_str = expected identifier, found metavariable
-parse_expected_identifier_found_reserved_identifier = expected identifier, found reserved identifier
-parse_expected_identifier_found_reserved_identifier_str = expected identifier, found reserved identifier `{$token}`
-parse_expected_identifier_found_reserved_keyword = expected identifier, found reserved keyword
-parse_expected_identifier_found_reserved_keyword_str = expected identifier, found reserved keyword `{$token}`
-parse_expected_identifier_found_str = expected identifier, found `{$token}`
-
-parse_expected_mut_or_const_in_raw_pointer_type = expected `mut` or `const` keyword in raw pointer type
-    .suggestion = add `mut` or `const` here
-
-parse_expected_semi_found_doc_comment_str = expected `;`, found doc comment `{$token}`
-parse_expected_semi_found_keyword_str = expected `;`, found keyword `{$token}`
-# This one deliberately doesn't print a token.
-parse_expected_semi_found_metavar_str = expected `;`, found metavariable
-parse_expected_semi_found_reserved_identifier_str = expected `;`, found reserved identifier `{$token}`
-parse_expected_semi_found_reserved_keyword_str = expected `;`, found reserved keyword `{$token}`
-parse_expected_semi_found_str = expected `;`, found `{$token}`
-
-parse_expected_statement_after_outer_attr = expected statement after outer attribute
-
-parse_expected_struct_field = expected one of `,`, `:`, or `{"}"}`, found `{$token}`
-    .label = expected one of `,`, `:`, or `{"}"}`
-    .ident_label = while parsing this struct field
-
-parse_expected_trait_in_trait_impl_found_type = expected a trait, found type
-
-parse_expr_rarrow_call = `->` is not valid syntax for field accesses and method calls
-    .suggestion = try using `.` instead
-    .help = the `.` operator will automatically dereference the value, except if the value is a raw pointer
-
-parse_extern_crate_name_with_dashes = crate name using dashes are not valid in `extern crate` statements
-    .label = dash-separated idents are not valid
-    .suggestion = if the original crate name uses dashes you need to use underscores in the code
-
-parse_extern_item_cannot_be_const = extern items cannot be `const`
-    .suggestion = try using a static value
-    .note = for more information, visit https://doc.rust-lang.org/std/keyword.extern.html
-
-parse_extra_if_in_let_else = remove the `if` if you meant to write a `let...else` statement
-
-parse_extra_impl_keyword_in_trait_impl = unexpected `impl` keyword
-    .suggestion = remove the extra `impl`
-    .note = this is parsed as an `impl Trait` type, but a trait is expected at this position
-
-
-parse_field_expression_with_generic = field expressions cannot have generic arguments
-
-parse_float_literal_requires_integer_part = float literals must have an integer part
-    .suggestion = must have an integer part
-
-parse_float_literal_unsupported_base = {$base} float literal is not supported
-
-parse_fn_pointer_cannot_be_async = an `fn` pointer type cannot be `async`
-    .label = `async` because of this
-    .suggestion = remove the `async` qualifier
-    .note = allowed qualifiers are: `unsafe` and `extern`
-
-parse_fn_pointer_cannot_be_const = an `fn` pointer type cannot be `const`
-    .label = `const` because of this
-    .suggestion = remove the `const` qualifier
-    .note = allowed qualifiers are: `unsafe` and `extern`
-
-parse_fn_ptr_with_generics = function pointer types may not have generic parameters
-    .suggestion = consider moving the lifetime {$arity ->
-        [one] parameter
-        *[other] parameters
-    } to {$for_param_list_exists ->
-        [true] the
-        *[false] a
-    } `for` parameter list
-
-parse_fn_trait_missing_paren = `Fn` bounds require arguments in parentheses
-    .add_paren = add the missing parentheses
-
-parse_forgot_paren = perhaps you forgot parentheses?
-
-parse_found_expr_would_be_stmt = expected expression, found `{$token}`
-    .label = expected expression
-
-parse_frontmatter_extra_characters_after_close = extra characters after frontmatter close are not allowed
-parse_frontmatter_invalid_close_preceding_whitespace = invalid preceding whitespace for frontmatter close
-    .note = frontmatter close should not be preceded by whitespace
-parse_frontmatter_invalid_infostring = invalid infostring for frontmatter
-    .note = frontmatter infostrings must be a single identifier immediately following the opening
-parse_frontmatter_invalid_opening_preceding_whitespace = invalid preceding whitespace for frontmatter opening
-    .note = frontmatter opening should not be preceded by whitespace
-parse_frontmatter_length_mismatch = frontmatter close does not match the opening
-    .label_opening = the opening here has {$len_opening} dashes...
-    .label_close = ...while the close has {$len_close} dashes
-parse_frontmatter_too_many_dashes = too many `-` symbols: frontmatter openings may be delimited by up to 255 `-` symbols, but found {$len_opening}
-parse_frontmatter_unclosed = unclosed frontmatter
-    .note = frontmatter opening here was not closed
-parse_function_body_equals_expr = function body cannot be `= expression;`
-    .suggestion = surround the expression with `{"{"}` and `{"}"}` instead of `=` and `;`
-
-parse_generic_args_in_pat_require_turbofish_syntax = generic args in patterns require the turbofish syntax
-
-parse_generic_parameters_without_angle_brackets = generic parameters without surrounding angle brackets
-    .suggestion = surround the type parameters with angle brackets
-
-parse_generics_in_path = unexpected generic arguments in path
-
-parse_help_set_edition_cargo = set `edition = "{$edition}"` in `Cargo.toml`
-parse_help_set_edition_standalone = pass `--edition {$edition}` to `rustc`
-
-parse_hidden_unicode_codepoints = unicode codepoint changing visible direction of text present in {$label}
-    .label = this {$label} contains {$count ->
-        [one] an invisible
-        *[other] invisible
-    } unicode text flow control {$count ->
-        [one] codepoint
-        *[other] codepoints
-    }
-    .note = these kind of unicode codepoints change the way text flows on applications that support them, but can cause confusion because they change the order of characters on the screen
-    .suggestion_remove = if their presence wasn't intentional, you can remove them
-    .suggestion_escape = if you want to keep them but make them visible in your source code, you can escape them
-    .no_suggestion_note_escape = if you want to keep them but make them visible in your source code, you can escape them: {$escaped}
-
-parse_if_expression_missing_condition = missing condition for `if` expression
-    .condition_label = expected condition here
-    .block_label = if this block is the condition of the `if` expression, then it must be followed by another block
-
-parse_if_expression_missing_then_block = this `if` expression is missing a block after the condition
-    .add_then_block = add a block here
-    .condition_possibly_unfinished = this binary operation is possibly unfinished
-
-parse_in_in_typo =
-    expected iterable, found keyword `in`
-    .suggestion = remove the duplicated `in`
-
-parse_inappropriate_default = {$article} {$descr} cannot be `default`
-    .label = `default` because of this
-    .note = only associated `fn`, `const`, and `type` items can be `default`
-
-parse_inclusive_range_extra_equals = unexpected `=` after inclusive range
-    .suggestion_remove_eq = use `..=` instead
-    .note = inclusive ranges end with a single equals sign (`..=`)
-
-parse_inclusive_range_match_arrow = unexpected `>` after inclusive range
-    .label = this is parsed as an inclusive range `..=`
-    .suggestion = add a space between the pattern and `=>`
-
-parse_inclusive_range_no_end = inclusive range with no end
-    .suggestion_open_range = use `..` instead
-    .note = inclusive ranges must be bounded at the end (`..=b` or `a..=b`)
-
-parse_incorrect_parens_trait_bounds = incorrect parentheses around trait bounds
-parse_incorrect_parens_trait_bounds_sugg = fix the parentheses
-
-parse_incorrect_semicolon =
-    expected item, found `;`
-    .suggestion = remove this semicolon
-    .help = {$name} declarations are not followed by a semicolon
-
-parse_incorrect_type_on_self = type not allowed for shorthand `self` parameter
-    .suggestion = move the modifiers on `self` to the type
-
-parse_incorrect_use_of_await = incorrect use of `await`
-    .parentheses_suggestion = `await` is not a method call, remove the parentheses
-
-parse_incorrect_use_of_await_postfix_suggestion = `await` is a postfix operation
-
-parse_incorrect_use_of_use = incorrect use of `use`
-    .parentheses_suggestion = `use` is not a method call, try removing the parentheses
-
-parse_incorrect_visibility_restriction = incorrect visibility restriction
-    .help = some possible visibility restrictions are:
-            `pub(crate)`: visible only on the current crate
-            `pub(super)`: visible only in the current module's parent
-            `pub(in path::to::module)`: visible only on the specified path
-    .suggestion = make this visible only to module `{$inner_str}` with `in`
-
-parse_inner_attr_explanation = inner attributes, like `#![no_std]`, annotate the item enclosing them, and are usually found at the beginning of source files
-parse_inner_attr_not_permitted = an inner attribute is not permitted in this context
-    .label_does_not_annotate_this = {parse_label_inner_attr_does_not_annotate_this}
-    .sugg_change_inner_to_outer = {parse_sugg_change_inner_attr_to_outer}
-
-parse_inner_attr_not_permitted_after_outer_attr = an inner attribute is not permitted following an outer attribute
-    .label_attr = not permitted following an outer attribute
-    .label_prev_attr = previous outer attribute
-    .label_does_not_annotate_this = {parse_label_inner_attr_does_not_annotate_this}
-    .sugg_change_inner_to_outer = {parse_sugg_change_inner_attr_to_outer}
-
-parse_inner_attr_not_permitted_after_outer_doc_comment = an inner attribute is not permitted following an outer doc comment
-    .label_attr = not permitted following an outer doc comment
-    .label_prev_doc_comment = previous doc comment
-    .label_does_not_annotate_this = {parse_label_inner_attr_does_not_annotate_this}
-    .sugg_change_inner_to_outer = {parse_sugg_change_inner_attr_to_outer}
-
-parse_inner_doc_comment_not_permitted = expected outer doc comment
-    .note = inner doc comments like this (starting with `//!` or `/*!`) can only appear before items
-    .suggestion = you might have meant to write a regular comment
-    .label_does_not_annotate_this = the inner doc comment doesn't annotate this {$item}
-    .sugg_change_inner_to_outer = to annotate the {$item}, change the doc comment from inner to outer style
-
-parse_invalid_block_macro_segment = cannot use a `block` macro fragment here
-    .label = the `block` fragment is within this context
-    .suggestion = wrap this in another block
-
-parse_invalid_char_in_escape = {parse_invalid_char_in_escape_msg}: `{$ch}`
-    .label = {parse_invalid_char_in_escape_msg}
-
-parse_invalid_char_in_escape_msg = invalid character in {$is_hex ->
-    [true] numeric character
-    *[false] unicode
-    } escape
-
-
-parse_invalid_comparison_operator = invalid comparison operator `{$invalid}`
-    .use_instead = `{$invalid}` is not a valid comparison operator, use `{$correct}`
-    .spaceship_operator_invalid = `<=>` is not a valid comparison operator, use `std::cmp::Ordering`
-
-parse_invalid_curly_in_let_else = right curly brace `{"}"}` before `else` in a `let...else` statement not allowed
-parse_invalid_digit_literal = invalid digit for a base {$base} literal
-
-parse_invalid_dyn_keyword = invalid `dyn` keyword
-    .help = `dyn` is only needed at the start of a trait `+`-separated list
-    .suggestion = remove this keyword
-
-parse_invalid_expression_in_let_else = a `{$operator}` expression cannot be directly assigned in `let...else`
-parse_invalid_identifier_with_leading_number = identifiers cannot start with a number
-
-parse_invalid_literal_suffix_on_tuple_index = suffixes on a tuple index are invalid
-    .label = invalid suffix `{$suffix}`
-
-parse_invalid_logical_operator = `{$incorrect}` is not a logical operator
-    .note = unlike in e.g., Python and PHP, `&&` and `||` are used for logical operators
-    .use_amp_amp_for_conjunction = use `&&` to perform logical conjunction
-    .use_pipe_pipe_for_disjunction = use `||` to perform logical disjunction
-
-parse_invalid_meta_item = expected unsuffixed literal, found {$descr}
-    .quote_ident_sugg = surround the identifier with quotation marks to make it into a string literal
-
-parse_invalid_offset_of = offset_of expects dot-separated field and variant names
-
-parse_invalid_path_sep_in_fn_definition = invalid path separator in function definition
-    .suggestion = remove invalid path separator
-
-parse_invalid_unicode_escape = invalid unicode character escape
-    .label = invalid escape
-    .help = unicode escape must {$surrogate ->
-    [true] not be a surrogate
-    *[false] be at most 10FFFF
-    }
-
-parse_invalid_variable_declaration =
-    invalid variable declaration
-
-parse_keyword_label = labels cannot use keyword names
-
-parse_keyword_lifetime =
-    lifetimes cannot use keyword names
-
-parse_kw_bad_case = keyword `{$kw}` is written in the wrong case
-    .suggestion = write it in {$case}
-
-parse_label_inner_attr_does_not_annotate_this = the inner attribute doesn't annotate this {$item}
-parse_label_unexpected_token = unexpected token
-
-parse_label_while_parsing_or_pattern_here = while parsing this or-pattern starting here
-
-parse_labeled_loop_in_break = parentheses are required around this expression to avoid confusion with a labeled break expression
-
-parse_leading_plus_not_supported = leading `+` is not supported
-    .label = unexpected `+`
-    .suggestion_remove_plus = try removing the `+`
-
-parse_leading_underscore_unicode_escape = {parse_leading_underscore_unicode_escape_label}: `_`
-parse_leading_underscore_unicode_escape_label = invalid start of unicode escape
-
-parse_left_arrow_operator = unexpected token: `<-`
-    .suggestion = if you meant to write a comparison against a negative value, add a space in between `<` and `-`
-
-parse_let_chain_pre_2024 = let chains are only allowed in Rust 2024 or later
-
-parse_lifetime_after_mut = lifetime must precede `mut`
-    .suggestion = place the lifetime before `mut`
-
-parse_lifetime_in_borrow_expression = borrow expressions cannot be annotated with lifetimes
-    .suggestion = remove the lifetime annotation
-    .label = annotated with lifetime here
-
-parse_lifetime_in_eq_constraint = lifetimes are not permitted in this context
-    .label = lifetime is not allowed here
-    .context_label = this introduces an associated item binding
-    .help = if you meant to specify a trait object, write `dyn /* Trait */ + {$lifetime}`
-    .colon_sugg = you might have meant to write a bound here
-
-parse_lone_slash = invalid trailing slash in literal
-    .label = {parse_lone_slash}
-
-parse_loop_else = `{$loop_kind}...else` loops are not supported
-    .note = consider moving this `else` clause to a separate `if` statement and use a `bool` variable to control if it should run
-    .loop_keyword = `else` is attached to this loop
-
-parse_macro_expands_to_adt_field = macros cannot expand to {$adt_ty} fields
-
-parse_macro_expands_to_enum_variant = macros cannot expand to enum variants
-
-parse_macro_invocation_visibility = can't qualify macro invocation with `pub`
-    .suggestion = remove the visibility
-    .help = try adjusting the macro to put `{$vis}` inside the invocation
-
-parse_macro_invocation_with_qualified_path = macros cannot use qualified paths
-
-parse_macro_name_remove_bang = macro names aren't followed by a `!`
-    .suggestion = remove the `!`
-
-parse_macro_rules_missing_bang = expected `!` after `macro_rules`
-    .suggestion = add a `!`
-
-parse_macro_rules_visibility = can't qualify macro_rules invocation with `{$vis}`
-    .suggestion = try exporting the macro
-
-parse_malformed_loop_label = malformed loop label
-    .suggestion = use the correct loop label format
-
-parse_match_arm_body_without_braces = `match` arm body without braces
-    .label_statements = {$num_statements ->
-            [one] this statement is not surrounded by a body
-           *[other] these statements are not surrounded by a body
-        }
-    .label_arrow = while parsing the `match` arm starting here
-    .suggestion_add_braces = surround the {$num_statements ->
-            [one] statement
-           *[other] statements
-        } with a body
-    .suggestion_use_comma_not_semicolon = replace `;` with `,` to end a `match` arm expression
-
-parse_maybe_comparison = you might have meant to compare for equality
-
-parse_maybe_fn_typo_with_impl = you might have meant to write `impl` instead of `fn`
-    .suggestion = replace `fn` with `impl` here
-
-parse_maybe_missing_let = you might have meant to continue the let-chain
-
-parse_maybe_recover_from_bad_qpath_stage_2 =
-    missing angle brackets in associated item path
-    .suggestion = types that don't start with an identifier need to be surrounded with angle brackets in qualified paths
-
-parse_maybe_recover_from_bad_type_plus =
-    expected a path on the left-hand side of `+`
-
-parse_maybe_report_ambiguous_plus =
-    ambiguous `+` in a type
-    .suggestion = use parentheses to disambiguate
-
-parse_mismatched_closing_delimiter = mismatched closing delimiter: `{$delimiter}`
-    .label_unmatched = mismatched closing delimiter
-    .label_opening_candidate = closing delimiter possibly meant for this
-    .label_unclosed = unclosed delimiter
-
-parse_misplaced_return_type = place the return type after the function parameters
-
-parse_missing_comma_after_match_arm = expected `,` following `match` arm
-    .suggestion = missing a comma here to end this `match` arm
-
-parse_missing_const_type = missing type for `{$kind}` item
-    .suggestion = provide a type for the item
-
-parse_missing_enum_for_enum_definition = missing `enum` for enum definition
-    .suggestion = add `enum` here to parse `{$ident}` as an enum
-
-parse_missing_enum_or_struct_for_item_definition = missing `enum` or `struct` for enum or struct definition
-
-parse_missing_expression_in_for_loop = missing expression to iterate on in `for` loop
-    .suggestion = try adding an expression to the `for` loop
-
-parse_missing_fn_for_function_definition = missing `fn` for function definition
-    .suggestion = add `fn` here to parse `{$ident}` as a function
-
-parse_missing_fn_for_method_definition = missing `fn` for method definition
-    .suggestion = add `fn` here to parse `{$ident}` as a method
-
-parse_missing_fn_or_struct_for_item_definition = missing `fn` or `struct` for function or struct definition
-    .suggestion = if you meant to call a macro, try
-    .help = if you meant to call a macro, remove the `pub` and add a trailing `!` after the identifier
-
-parse_missing_fn_params = missing parameters for function definition
-    .suggestion = add a parameter list
-
-parse_missing_for_in_trait_impl = missing `for` in a trait impl
-    .suggestion = add `for` here
-
-parse_missing_in_in_for_loop = missing `in` in `for` loop
-    .use_in = try using `in` here instead
-    .add_in = try adding `in` here
-
-parse_missing_let_before_mut = missing keyword
-parse_missing_plus_in_bounds = expected `+` between lifetime and {$sym}
-    .suggestion = add `+`
-
-parse_missing_semicolon_before_array = expected `;`, found `[`
-    .suggestion = consider adding `;` here
-
-parse_missing_struct_for_struct_definition = missing `struct` for struct definition
-    .suggestion = add `struct` here to parse `{$ident}` as a struct
-
-parse_missing_trait_in_trait_impl = missing trait in a trait impl
-    .suggestion_add_trait = add a trait here
-    .suggestion_remove_for = for an inherent impl, drop this `for`
-
-parse_misspelled_kw = {$is_incorrect_case ->
-                    [true] write keyword `{$similar_kw}` in lowercase
-                    *[false] there is a keyword `{$similar_kw}` with a similar name
-}
-
-parse_modifier_lifetime = `{$modifier}` may only modify trait bounds, not lifetime bounds
-    .suggestion = remove the `{$modifier}`
-
-parse_modifiers_and_polarity = `{$modifiers_concatenated}` trait not allowed with `{$polarity}` trait polarity modifier
-    .label = there is not a well-defined meaning for a `{$modifiers_concatenated} {$polarity}` trait
-
-parse_more_than_one_char = character literal may only contain one codepoint
-    .followed_by = this `{$chr}` is followed by the combining {$len ->
-        [one] mark
-        *[other] marks
-        } `{$escaped_marks}`
-    .non_printing = there are non-printing characters, the full sequence is `{$escaped}`
-    .consider_normalized = consider using the normalized form `{$ch}` of this character
-    .remove_non = consider removing the non-printing characters
-    .use_double_quotes = if you meant to write a {$is_byte ->
-        [true] byte string
-        *[false] string
-        } literal, use double quotes
-
-parse_multiple_skipped_lines = multiple lines skipped by escaped newline
-    .label = skipping everything up to and including this point
-
-parse_multiple_where_clauses = cannot define duplicate `where` clauses on an item
-    .label = previous `where` clause starts here
-    .suggestion = consider joining the two `where` clauses into one
-
-parse_mut_on_nested_ident_pattern = `mut` must be attached to each individual binding
-    .suggestion = add `mut` to each binding
-parse_mut_on_non_ident_pattern = `mut` must be followed by a named binding
-    .suggestion = remove the `mut` prefix
-
-parse_need_plus_after_trait_object_lifetime = lifetimes must be followed by `+` to form a trait object type
-    .suggestion = consider adding a trait bound after the potential lifetime bound
-
-parse_nested_adt = `{$kw_str}` definition cannot be nested inside `{$keyword}`
-    .suggestion = consider creating a new `{$kw_str}` definition instead of nesting
-
-parse_nested_c_variadic_type = C-variadic type `...` may not be nested inside another type
-
-parse_no_brace_unicode_escape = incorrect unicode escape sequence
-    .label = {parse_no_brace_unicode_escape}
-    .use_braces = format of unicode escape sequences uses braces
-    .format_of_unicode = format of unicode escape sequences is `\u{"{...}"}`
-
-parse_no_digits_literal = no valid digits found for number
-
-parse_non_string_abi_literal = non-string ABI literal
-    .suggestion = specify the ABI with a string literal
-
-parse_nonterminal_expected_ident = expected ident, found `{$token}`
-parse_nonterminal_expected_item_keyword = expected an item keyword
-parse_nonterminal_expected_lifetime = expected a lifetime, found `{$token}`
-
-parse_nonterminal_expected_statement = expected a statement
-
-parse_note_edition_guide = for more on editions, read https://doc.rust-lang.org/edition-guide
-
-parse_note_mut_pattern_usage = `mut` may be followed by `variable` and `variable @ pattern`
-
-parse_note_pattern_alternatives_use_single_vert = alternatives in or-patterns are separated with `|`, not `||`
-
-parse_nul_in_c_str = null characters in C string literals are not supported
-
-parse_or_in_let_chain = `||` operators are not supported in let chain conditions
-
-parse_or_pattern_not_allowed_in_fn_parameters = function parameters require top-level or-patterns in parentheses
-parse_or_pattern_not_allowed_in_let_binding = `let` bindings require top-level or-patterns in parentheses
-
-parse_outer_attr_explanation = outer attributes, like `#[test]`, annotate the item following them
-
-parse_outer_attribute_not_allowed_on_if_else = outer attributes are not allowed on `if` and `else` branches
-    .branch_label = the attributes are attached to this branch
-    .ctx_label = the branch belongs to this `{$ctx}`
-    .suggestion = remove the attributes
-
-parse_overlong_unicode_escape = overlong unicode escape
-    .label = must have at most 6 hex digits
-
-parse_parentheses_with_struct_fields = invalid `struct` delimiters or `fn` call arguments
-    .suggestion_braces_for_struct = if `{$type}` is a struct, use braces as delimiters
-    .suggestion_no_fields_for_fn = if `{$type}` is a function, use the arguments directly
-
-parse_path_double_colon = path separator must be a double colon
-    .suggestion = use a double colon instead
-
-
-parse_path_found_attribute_in_params = `Trait(...)` syntax does not support attributes in parameters
-    .suggestion = remove the attributes
-
-parse_path_found_c_variadic_params = `Trait(...)` syntax does not support c_variadic parameters
-    .suggestion = remove the `...`
-
-parse_path_found_named_params = `Trait(...)` syntax does not support named parameters
-    .suggestion = remove the parameter name
-
-parse_pattern_method_param_without_body = patterns aren't allowed in methods without bodies
-    .suggestion = give this argument a name or use an underscore to ignore it
-
-parse_pattern_on_wrong_side_of_at = pattern on wrong side of `@`
-    .label_pattern = pattern on the left, should be on the right
-    .label_binding = binding on the right, should be on the left
-    .suggestion = switch the order
-
-parse_question_mark_in_type = invalid `?` in type
-    .label = `?` is only allowed on expressions, not types
-    .suggestion = if you meant to express that the type might not contain a value, use the `Option` wrapper type
-
-parse_recover_import_as_use = expected item, found {$token_name}
-    .suggestion = items are imported using the `use` keyword
-
-parse_remove_let = expected pattern, found `let`
-    .suggestion = remove the unnecessary `let` keyword
-
-parse_repeated_mut_in_pattern = `mut` on a binding may not be repeated
-    .suggestion = remove the additional `mut`s
-
-parse_require_colon_after_labeled_expression = labeled expression must be followed by `:`
-    .note = labels are used before loops and blocks, allowing e.g., `break 'label` to them
-    .label = the label
-    .suggestion = add `:` after the label
-
-parse_reserved_multihash = reserved multi-hash token is forbidden
-    .note = sequences of two or more # are reserved for future use since Rust 2024
-    .suggestion_whitespace = consider inserting whitespace here
-
-parse_reserved_string = invalid string literal
-    .note = unprefixed guarded string literals are reserved for future use since Rust 2024
-    .suggestion_whitespace = consider inserting whitespace here
-
-parse_return_types_use_thin_arrow = return types are denoted using `->`
-    .suggestion = use `->` instead
-
-parse_self_argument_pointer = cannot pass `self` by raw pointer
-    .label = cannot pass `self` by raw pointer
-
-parse_self_param_not_first = unexpected `self` parameter in function
-    .label = must be the first parameter of an associated function
-
-parse_shift_interpreted_as_generic =
-    `<<` is interpreted as a start of generic arguments for `{$type}`, not a shift
-    .label_args = interpreted as generic arguments
-    .label_comparison = not interpreted as shift
-    .suggestion = try shifting the cast value
-
-parse_single_colon_import_path = expected `::`, found `:`
-    .suggestion = use double colon
-    .note = import paths are delimited using `::`
-
-parse_static_with_generics = static items may not have generic parameters
-
-parse_struct_literal_body_without_path =
-    struct literal body without path
-    .suggestion = you might have forgotten to add the struct literal inside the block
-
-parse_struct_literal_body_without_path_late =
-    struct literal body without path
-    .label = struct name missing for struct literal
-    .suggestion = add the correct type
-
-parse_struct_literal_not_allowed_here = struct literals are not allowed here
-    .suggestion = surround the struct literal with parentheses
-
-parse_struct_literal_placeholder_path =
-    placeholder `_` is not allowed for the path in struct literals
-    .label = not allowed in struct literals
-    .suggestion = replace it with the correct type
-
-parse_suffixed_literal_in_attribute = suffixed literals are not allowed in attributes
-    .help = instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), use an unsuffixed version (`1`, `1.0`, etc.)
-
-parse_sugg_add_let_for_stmt = you might have meant to introduce a new binding
-
-parse_sugg_add_semi = add `;` here
-parse_sugg_change_inner_attr_to_outer = to annotate the {$item}, change the attribute from inner to outer style
-
-parse_sugg_change_this_to_semi = change this to `;`
-parse_sugg_escape_identifier = escape `{$ident_name}` to use it as an identifier
-
-parse_sugg_remove_comma = remove this comma
-parse_sugg_remove_leading_vert_in_pattern = remove the `|`
-parse_sugg_turbofish_syntax = use `::<...>` instead of `<...>` to specify lifetime, type, or const arguments
-
-parse_sugg_wrap_expression_in_parentheses = wrap the expression in parentheses
-
-parse_sugg_wrap_macro_in_parentheses = use parentheses instead of braces for this macro
-
-parse_sugg_wrap_pattern_in_parens = wrap the pattern in parentheses
-
-parse_switch_mut_let_order =
-    switch the order of `mut` and `let`
-
-parse_switch_ref_box_order = switch the order of `ref` and `box`
-    .suggestion = swap them
-
-parse_ternary_operator = Rust has no ternary operator
-
-parse_tilde_is_not_unary_operator = `~` cannot be used as a unary operator
-    .suggestion = use `!` to perform bitwise not
-
-parse_too_many_hashes = too many `#` symbols: raw strings may be delimited by up to 255 `#` symbols, but found {$num}
-
-parse_too_short_hex_escape = numeric character escape is too short
-
-parse_trailing_vert_not_allowed = a trailing `{$token}` is not allowed in an or-pattern
-
-parse_trait_alias_cannot_be_auto = trait aliases cannot be `auto`
-parse_trait_alias_cannot_be_unsafe = trait aliases cannot be `unsafe`
-
-parse_trait_impl_modifier_in_inherent_impl = inherent impls cannot be {$modifier_name}
-    .because = {$modifier_name} because of this
-    .type = inherent impl for this type
-    .note = only trait implementations may be annotated with `{$modifier}`
-
-parse_transpose_dyn_or_impl = `for<...>` expected after `{$kw}`, not before
-    .suggestion = move `{$kw}` before the `for<...>`
-
-parse_unclosed_unicode_escape = unterminated unicode escape
-    .label = missing a closing `{"}"}`
-    .terminate = terminate the unicode escape
-
-parse_underscore_literal_suffix = underscore literal suffix is not allowed
-
-parse_unexpected_const_in_generic_param = expected lifetime, type, or constant, found keyword `const`
-    .suggestion = the `const` keyword is only needed in the definition of the type
-
-parse_unexpected_const_param_declaration = unexpected `const` parameter declaration
-    .label = expected a `const` expression, not a parameter declaration
-    .suggestion = `const` parameters must be declared for the `impl`
-
-parse_unexpected_default_value_for_lifetime_in_generic_parameters = unexpected default lifetime parameter
-    .label = lifetime parameters cannot have default values
-
-parse_unexpected_expr_in_pat =
-    expected {$is_bound ->
-        [true] a pattern range bound
-       *[false] a pattern
-    }, found an expression
-
-    .label = not a pattern
-    .note = arbitrary expressions are not allowed in patterns: 
-
-parse_unexpected_expr_in_pat_const_sugg = consider extracting the expression into a `const`
-
-parse_unexpected_expr_in_pat_create_guard_sugg = consider moving the expression to a match arm guard
-
-parse_unexpected_expr_in_pat_update_guard_sugg = consider moving the expression to the match arm guard
-
-parse_unexpected_if_with_if = unexpected `if` in the condition expression
-    .suggestion = remove the `if`
-
-parse_unexpected_lifetime_in_pattern = unexpected lifetime `{$symbol}` in pattern
-    .suggestion = remove the lifetime
-
-parse_unexpected_paren_in_range_pat = range pattern bounds cannot have parentheses
-parse_unexpected_paren_in_range_pat_sugg = remove these parentheses
-
-parse_unexpected_parentheses_in_for_head = unexpected parentheses surrounding `for` loop head
-    .suggestion = remove parentheses in `for` loop
-
-parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surrounding `match` arm pattern
-    .suggestion = remove parentheses surrounding the pattern
-
-parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
-    .note = you cannot use `Self` as a generic parameter because it is reserved for associated items
-
-parse_unexpected_token_after_dot = unexpected token: {$actual}
-
-parse_unexpected_token_after_label = expected `while`, `for`, `loop` or `{"{"}` after a label
-    .suggestion_remove_label = consider removing the label
-    .suggestion_enclose_in_block = consider enclosing expression in a block
-
-parse_unexpected_token_after_not = unexpected {$negated_desc} after identifier
-parse_unexpected_token_after_not_bitwise = use `!` to perform bitwise not
-parse_unexpected_token_after_not_default = use `!` to perform logical negation or bitwise not
-
-parse_unexpected_token_after_not_logical = use `!` to perform logical negation
-parse_unexpected_token_after_struct_name = expected `where`, `{"{"}`, `(`, or `;` after struct name
-parse_unexpected_token_after_struct_name_found_doc_comment = expected `where`, `{"{"}`, `(`, or `;` after struct name, found doc comment `{$token}`
-parse_unexpected_token_after_struct_name_found_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found keyword `{$token}`
-# This one deliberately doesn't print a token.
-parse_unexpected_token_after_struct_name_found_metavar = expected `where`, `{"{"}`, `(`, or `;` after struct name, found metavar
-parse_unexpected_token_after_struct_name_found_other = expected `where`, `{"{"}`, `(`, or `;` after struct name, found `{$token}`
-
-parse_unexpected_token_after_struct_name_found_reserved_identifier = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved identifier `{$token}`
-parse_unexpected_token_after_struct_name_found_reserved_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved keyword `{$token}`
-parse_unexpected_vert_vert_before_function_parameter = unexpected `||` before function parameter
-    .suggestion = remove the `||`
-
-parse_unexpected_vert_vert_in_pattern = unexpected token `||` in pattern
-    .suggestion = use a single `|` to separate multiple alternative patterns
-
-parse_unicode_escape_in_byte = unicode escape in byte string
-    .label = {parse_unicode_escape_in_byte}
-    .help = unicode escape sequences cannot be used as a byte or in a byte string
-
-parse_unknown_builtin_construct = unknown `builtin #` construct `{$name}`
-
-parse_unknown_prefix = prefix `{$prefix}` is unknown
-    .label = unknown prefix
-    .note =  prefixed identifiers and literals are reserved since Rust 2021
-    .suggestion_br = use `br` for a raw byte string
-    .suggestion_cr = use `cr` for a raw C-string
-    .suggestion_str = if you meant to write a string literal, use double quotes
-    .suggestion_whitespace = consider inserting whitespace here
-
-parse_unknown_start_of_token = unknown start of token: {$escaped}
-    .sugg_quotes = Unicode characters '“' (Left Double Quotation Mark) and '”' (Right Double Quotation Mark) look like '{$ascii_str}' ({$ascii_name}), but are not
-    .sugg_other = Unicode character '{$ch}' ({$u_name}) looks like '{$ascii_str}' ({$ascii_name}), but it is not
-    .help_null = source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used
-    .help_invisible_char = invisible characters like '{$escaped}' are not usually visible in text editors
-    .note_repeats = character appears {$repeats ->
-        [one] once more
-        *[other] {$repeats} more times
-    }
-
-parse_unmatched_angle = unmatched angle {$plural ->
-    [true] brackets
-    *[false] bracket
-    }
-    .suggestion = remove extra angle {$plural ->
-    [true] brackets
-    *[false] bracket
-    }
-
-parse_unmatched_angle_brackets = {$num_extra_brackets ->
-        [one] unmatched angle bracket
-       *[other] unmatched angle brackets
-    }
-    .suggestion = {$num_extra_brackets ->
-            [one] remove extra angle bracket
-           *[other] remove extra angle brackets
-        }
-
-
-parse_unskipped_whitespace = whitespace symbol '{$ch}' is not skipped
-    .label = {parse_unskipped_whitespace}
-
-parse_use_empty_block_not_semi = expected { "`{}`" }, found `;`
-    .suggestion = try using { "`{}`" } instead
-
-parse_use_eq_instead = unexpected `==`
-    .suggestion = try using `=` instead
-
-parse_use_if_else = use an `if-else` expression instead
-
-parse_use_let_not_auto = write `let` instead of `auto` to introduce a new variable
-parse_use_let_not_var = write `let` instead of `var` to introduce a new variable
-
-parse_varargs_without_pattern = missing pattern for `...` argument
-    .suggestion = name the argument, or use `_` to continue ignoring it
-
-parse_visibility_not_followed_by_item = visibility `{$vis}` is not followed by an item
-    .label = the visibility
-    .help = you likely meant to define an item, e.g., `{$vis} fn foo() {"{}"}`
-
-parse_where_clause_before_const_body = where clauses are not allowed before const item bodies
-    .label = unexpected where clause
-    .name_label = while parsing this const item
-    .body_label = the item body
-    .suggestion = move the body before the where clause
-
-parse_where_clause_before_tuple_struct_body = where clauses are not allowed before tuple struct bodies
-    .label = unexpected where clause
-    .name_label = while parsing this tuple struct
-    .body_label = the struct body
-    .suggestion = move the body before the where clause
-
-parse_where_generics = generic parameters on `where` clauses are reserved for future use
-    .label = currently unsupported
-
-parse_zero_chars = empty character literal
-    .label = {parse_zero_chars}
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 8991e3b9d502..4a7031257700 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -9,17 +9,15 @@ use rustc_ast::{Path, Visibility};
 use rustc_errors::codes::*;
 use rustc_errors::{
     Applicability, Diag, DiagArgValue, DiagCtxtHandle, Diagnostic, EmissionGuarantee, IntoDiagArg,
-    Level, Subdiagnostic, SuggestionStyle,
+    Level, Subdiagnostic, SuggestionStyle, inline_fluent,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::edition::{Edition, LATEST_STABLE_EDITION};
 use rustc_span::{Ident, Span, Symbol};
 
-use crate::fluent_generated as fluent;
-
 #[derive(Diagnostic)]
-#[diag(parse_maybe_report_ambiguous_plus)]
+#[diag("ambiguous `+` in a type")]
 pub(crate) struct AmbiguousPlus {
     #[primary_span]
     pub span: Span,
@@ -28,7 +26,7 @@ pub(crate) struct AmbiguousPlus {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_maybe_recover_from_bad_type_plus, code = E0178)]
+#[diag("expected a path on the left-hand side of `+`", code = E0178)]
 pub(crate) struct BadTypePlus {
     #[primary_span]
     pub span: Span,
@@ -37,7 +35,7 @@ pub(crate) struct BadTypePlus {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_add_paren, applicability = "machine-applicable")]
+#[multipart_suggestion("try adding parentheses", applicability = "machine-applicable")]
 pub(crate) struct AddParen {
     #[suggestion_part(code = "(")]
     pub lo: Span,
@@ -51,12 +49,12 @@ pub(crate) enum BadTypePlusSub {
         #[subdiagnostic]
         suggestion: AddParen,
     },
-    #[label(parse_forgot_paren)]
+    #[label("perhaps you forgot parentheses?")]
     ForgotParen {
         #[primary_span]
         span: Span,
     },
-    #[label(parse_expect_path)]
+    #[label("expected a path")]
     ExpectPath {
         #[primary_span]
         span: Span,
@@ -64,7 +62,7 @@ pub(crate) enum BadTypePlusSub {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_maybe_recover_from_bad_qpath_stage_2)]
+#[diag("missing angle brackets in associated item path")]
 pub(crate) struct BadQPathStage2 {
     #[primary_span]
     pub span: Span,
@@ -73,21 +71,24 @@ pub(crate) struct BadQPathStage2 {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_trait_impl_modifier_in_inherent_impl)]
-#[note]
+#[diag("inherent impls cannot be {$modifier_name}")]
+#[note("only trait implementations may be annotated with `{$modifier}`")]
 pub(crate) struct TraitImplModifierInInherentImpl {
     #[primary_span]
     pub span: Span,
     pub modifier: &'static str,
     pub modifier_name: &'static str,
-    #[label(parse_because)]
+    #[label("{$modifier_name} because of this")]
     pub modifier_span: Span,
-    #[label(parse_type)]
+    #[label("inherent impl for this type")]
     pub self_ty: Span,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "types that don't start with an identifier need to be surrounded with angle brackets in qualified paths",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct WrapType {
     #[suggestion_part(code = "<")]
     pub lo: Span,
@@ -96,22 +97,27 @@ pub(crate) struct WrapType {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_semicolon)]
+#[diag("expected item, found `;`")]
 pub(crate) struct IncorrectSemicolon<'a> {
     #[primary_span]
-    #[suggestion(style = "verbose", code = "", applicability = "machine-applicable")]
+    #[suggestion(
+        "remove this semicolon",
+        style = "verbose",
+        code = "",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
-    #[help]
+    #[help("{$name} declarations are not followed by a semicolon")]
     pub show_help: bool,
     pub name: &'a str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_use_of_await)]
+#[diag("incorrect use of `await`")]
 pub(crate) struct IncorrectUseOfAwait {
     #[primary_span]
     #[suggestion(
-        parse_parentheses_suggestion,
+        "`await` is not a method call, remove the parentheses",
         style = "verbose",
         code = "",
         applicability = "machine-applicable"
@@ -120,11 +126,11 @@ pub(crate) struct IncorrectUseOfAwait {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_use_of_use)]
+#[diag("incorrect use of `use`")]
 pub(crate) struct IncorrectUseOfUse {
     #[primary_span]
     #[suggestion(
-        parse_parentheses_suggestion,
+        "`use` is not a method call, try removing the parentheses",
         style = "verbose",
         code = "",
         applicability = "machine-applicable"
@@ -133,10 +139,7 @@ pub(crate) struct IncorrectUseOfUse {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(
-    parse_incorrect_use_of_await_postfix_suggestion,
-    applicability = "machine-applicable"
-)]
+#[multipart_suggestion("`await` is a postfix operation", applicability = "machine-applicable")]
 pub(crate) struct AwaitSuggestion {
     #[suggestion_part(code = "")]
     pub removal: Span,
@@ -146,7 +149,7 @@ pub(crate) struct AwaitSuggestion {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_use_of_await)]
+#[diag("incorrect use of `await`")]
 pub(crate) struct IncorrectAwait {
     #[primary_span]
     pub span: Span,
@@ -155,16 +158,21 @@ pub(crate) struct IncorrectAwait {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_in_in_typo)]
+#[diag("expected iterable, found keyword `in`")]
 pub(crate) struct InInTypo {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "", style = "verbose", applicability = "machine-applicable")]
+    #[suggestion(
+        "remove the duplicated `in`",
+        code = "",
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
     pub sugg_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_variable_declaration)]
+#[diag("invalid variable declaration")]
 pub(crate) struct InvalidVariableDeclaration {
     #[primary_span]
     pub span: Span,
@@ -175,28 +183,28 @@ pub(crate) struct InvalidVariableDeclaration {
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidVariableDeclarationSub {
     #[suggestion(
-        parse_switch_mut_let_order,
+        "switch the order of `mut` and `let`",
         style = "verbose",
         applicability = "maybe-incorrect",
         code = "let mut"
     )]
     SwitchMutLetOrder(#[primary_span] Span),
     #[suggestion(
-        parse_missing_let_before_mut,
+        "missing keyword",
         applicability = "machine-applicable",
         style = "verbose",
         code = "let mut"
     )]
     MissingLet(#[primary_span] Span),
     #[suggestion(
-        parse_use_let_not_auto,
+        "write `let` instead of `auto` to introduce a new variable",
         style = "verbose",
         applicability = "machine-applicable",
         code = "let"
     )]
     UseLetNotAuto(#[primary_span] Span),
     #[suggestion(
-        parse_use_let_not_var,
+        "write `let` instead of `var` to introduce a new variable",
         style = "verbose",
         applicability = "machine-applicable",
         code = "let"
@@ -205,15 +213,20 @@ pub(crate) enum InvalidVariableDeclarationSub {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_switch_ref_box_order)]
+#[diag("switch the order of `ref` and `box`")]
 pub(crate) struct SwitchRefBoxOrder {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", style = "verbose", code = "box ref")]
+    #[suggestion(
+        "swap them",
+        applicability = "machine-applicable",
+        style = "verbose",
+        code = "box ref"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_comparison_operator)]
+#[diag("invalid comparison operator `{$invalid}`")]
 pub(crate) struct InvalidComparisonOperator {
     #[primary_span]
     pub span: Span,
@@ -225,7 +238,7 @@ pub(crate) struct InvalidComparisonOperator {
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidComparisonOperatorSub {
     #[suggestion(
-        parse_use_instead,
+        "`{$invalid}` is not a valid comparison operator, use `{$correct}`",
         style = "verbose",
         applicability = "machine-applicable",
         code = "{correct}"
@@ -236,13 +249,13 @@ pub(crate) enum InvalidComparisonOperatorSub {
         invalid: String,
         correct: String,
     },
-    #[label(parse_spaceship_operator_invalid)]
+    #[label("`<=>` is not a valid comparison operator, use `std::cmp::Ordering`")]
     Spaceship(#[primary_span] Span),
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_logical_operator)]
-#[note]
+#[diag("`{$incorrect}` is not a logical operator")]
+#[note("unlike in e.g., Python and PHP, `&&` and `||` are used for logical operators")]
 pub(crate) struct InvalidLogicalOperator {
     #[primary_span]
     pub span: Span,
@@ -254,14 +267,14 @@ pub(crate) struct InvalidLogicalOperator {
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidLogicalOperatorSub {
     #[suggestion(
-        parse_use_amp_amp_for_conjunction,
+        "use `&&` to perform logical conjunction",
         style = "verbose",
         applicability = "machine-applicable",
         code = "&&"
     )]
     Conjunction(#[primary_span] Span),
     #[suggestion(
-        parse_use_pipe_pipe_for_disjunction,
+        "use `||` to perform logical disjunction",
         style = "verbose",
         applicability = "machine-applicable",
         code = "||"
@@ -270,15 +283,20 @@ pub(crate) enum InvalidLogicalOperatorSub {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_tilde_is_not_unary_operator)]
+#[diag("`~` cannot be used as a unary operator")]
 pub(crate) struct TildeAsUnaryOperator(
     #[primary_span]
-    #[suggestion(style = "verbose", applicability = "machine-applicable", code = "!")]
+    #[suggestion(
+        "use `!` to perform bitwise not",
+        style = "verbose",
+        applicability = "machine-applicable",
+        code = "!"
+    )]
     pub Span,
 );
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_token_after_not)]
+#[diag("unexpected {$negated_desc} after identifier")]
 pub(crate) struct NotAsNegationOperator {
     #[primary_span]
     pub negated: Span,
@@ -290,7 +308,7 @@ pub(crate) struct NotAsNegationOperator {
 #[derive(Subdiagnostic)]
 pub(crate) enum NotAsNegationOperatorSub {
     #[suggestion(
-        parse_unexpected_token_after_not_default,
+        "use `!` to perform logical negation or bitwise not",
         style = "verbose",
         applicability = "machine-applicable",
         code = "!"
@@ -298,7 +316,7 @@ pub(crate) enum NotAsNegationOperatorSub {
     SuggestNotDefault(#[primary_span] Span),
 
     #[suggestion(
-        parse_unexpected_token_after_not_bitwise,
+        "use `!` to perform bitwise not",
         style = "verbose",
         applicability = "machine-applicable",
         code = "!"
@@ -306,7 +324,7 @@ pub(crate) enum NotAsNegationOperatorSub {
     SuggestNotBitwise(#[primary_span] Span),
 
     #[suggestion(
-        parse_unexpected_token_after_not_logical,
+        "use `!` to perform logical negation",
         style = "verbose",
         applicability = "machine-applicable",
         code = "!"
@@ -315,46 +333,59 @@ pub(crate) enum NotAsNegationOperatorSub {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_malformed_loop_label)]
+#[diag("malformed loop label")]
 pub(crate) struct MalformedLoopLabel {
     #[primary_span]
     pub span: Span,
-    #[suggestion(applicability = "machine-applicable", code = "'", style = "verbose")]
+    #[suggestion(
+        "use the correct loop label format",
+        applicability = "machine-applicable",
+        code = "'",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_lifetime_in_borrow_expression)]
+#[diag("borrow expressions cannot be annotated with lifetimes")]
 pub(crate) struct LifetimeInBorrowExpression {
     #[primary_span]
     pub span: Span,
-    #[suggestion(applicability = "machine-applicable", code = "", style = "verbose")]
-    #[label]
+    #[suggestion(
+        "remove the lifetime annotation",
+        applicability = "machine-applicable",
+        code = "",
+        style = "verbose"
+    )]
+    #[label("annotated with lifetime here")]
     pub lifetime_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_field_expression_with_generic)]
+#[diag("field expressions cannot have generic arguments")]
 pub(crate) struct FieldExpressionWithGeneric(#[primary_span] pub Span);
 
 #[derive(Diagnostic)]
-#[diag(parse_macro_invocation_with_qualified_path)]
+#[diag("macros cannot use qualified paths")]
 pub(crate) struct MacroInvocationWithQualifiedPath(#[primary_span] pub Span);
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_token_after_label)]
+#[diag("expected `while`, `for`, `loop` or `{\"{\"}` after a label")]
 pub(crate) struct UnexpectedTokenAfterLabel {
     #[primary_span]
-    #[label(parse_unexpected_token_after_label)]
+    #[label("expected `while`, `for`, `loop` or `{\"{\"}` after a label")]
     pub span: Span,
-    #[suggestion(parse_suggestion_remove_label, style = "verbose", code = "")]
+    #[suggestion("consider removing the label", style = "verbose", code = "")]
     pub remove_label: Option,
     #[subdiagnostic]
     pub enclose_in_block: Option,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion_enclose_in_block, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "consider enclosing expression in a block",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct UnexpectedTokenAfterLabelSugg {
     #[suggestion_part(code = "{{ ")]
     pub left: Span,
@@ -363,66 +394,91 @@ pub(crate) struct UnexpectedTokenAfterLabelSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_require_colon_after_labeled_expression)]
-#[note]
+#[diag("labeled expression must be followed by `:`")]
+#[note("labels are used before loops and blocks, allowing e.g., `break 'label` to them")]
 pub(crate) struct RequireColonAfterLabeledExpression {
     #[primary_span]
     pub span: Span,
-    #[label]
+    #[label("the label")]
     pub label: Span,
-    #[suggestion(style = "verbose", applicability = "machine-applicable", code = ": ")]
+    #[suggestion(
+        "add `:` after the label",
+        style = "verbose",
+        applicability = "machine-applicable",
+        code = ": "
+    )]
     pub label_end: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_do_catch_syntax_removed)]
-#[note]
+#[diag("found removed `do catch` syntax")]
+#[note("following RFC #2388, the new non-placeholder syntax is `try`")]
 pub(crate) struct DoCatchSyntaxRemoved {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = "try", style = "verbose")]
+    #[suggestion(
+        "replace with the new syntax",
+        applicability = "machine-applicable",
+        code = "try",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_float_literal_requires_integer_part)]
+#[diag("float literals must have an integer part")]
 pub(crate) struct FloatLiteralRequiresIntegerPart {
     #[primary_span]
     pub span: Span,
-    #[suggestion(applicability = "machine-applicable", code = "0", style = "verbose")]
+    #[suggestion(
+        "must have an integer part",
+        applicability = "machine-applicable",
+        code = "0",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_semicolon_before_array)]
+#[diag("expected `;`, found `[`")]
 pub(crate) struct MissingSemicolonBeforeArray {
     #[primary_span]
     pub open_delim: Span,
-    #[suggestion(style = "verbose", applicability = "maybe-incorrect", code = ";")]
+    #[suggestion(
+        "consider adding `;` here",
+        style = "verbose",
+        applicability = "maybe-incorrect",
+        code = ";"
+    )]
     pub semicolon: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expect_dotdot_not_dotdotdot)]
+#[diag("expected `..`, found `...`")]
 pub(crate) struct MissingDotDot {
     #[primary_span]
     pub token_span: Span,
-    #[suggestion(applicability = "maybe-incorrect", code = "..", style = "verbose")]
+    #[suggestion(
+        "use `..` to fill in the rest of the fields",
+        applicability = "maybe-incorrect",
+        code = "..",
+        style = "verbose"
+    )]
     pub sugg_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_block_macro_segment)]
+#[diag("cannot use a `block` macro fragment here")]
 pub(crate) struct InvalidBlockMacroSegment {
     #[primary_span]
     pub span: Span,
-    #[label]
+    #[label("the `block` fragment is within this context")]
     pub context: Span,
     #[subdiagnostic]
     pub wrap: WrapInExplicitBlock,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("wrap this in another block", applicability = "machine-applicable")]
 pub(crate) struct WrapInExplicitBlock {
     #[suggestion_part(code = "{{ ")]
     pub lo: Span,
@@ -431,7 +487,7 @@ pub(crate) struct WrapInExplicitBlock {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_if_expression_missing_then_block)]
+#[diag("this `if` expression is missing a block after the condition")]
 pub(crate) struct IfExpressionMissingThenBlock {
     #[primary_span]
     pub if_span: Span,
@@ -443,14 +499,14 @@ pub(crate) struct IfExpressionMissingThenBlock {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum IfExpressionMissingThenBlockSub {
-    #[help(parse_condition_possibly_unfinished)]
+    #[help("this binary operation is possibly unfinished")]
     UnfinishedCondition(#[primary_span] Span),
-    #[help(parse_add_then_block)]
+    #[help("add a block here")]
     AddThenBlock(#[primary_span] Span),
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_ternary_operator)]
+#[diag("Rust has no ternary operator")]
 pub(crate) struct TernaryOperator {
     #[primary_span]
     pub span: Span,
@@ -458,12 +514,16 @@ pub(crate) struct TernaryOperator {
     #[subdiagnostic]
     pub sugg: Option,
     /// Otherwise, just print the suggestion message
-    #[help(parse_use_if_else)]
+    #[help("use an `if-else` expression instead")]
     pub no_sugg: bool,
 }
 
 #[derive(Subdiagnostic, Copy, Clone)]
-#[multipart_suggestion(parse_use_if_else, applicability = "maybe-incorrect", style = "verbose")]
+#[multipart_suggestion(
+    "use an `if-else` expression instead",
+    applicability = "maybe-incorrect",
+    style = "verbose"
+)]
 pub(crate) struct TernaryOperatorSuggestion {
     #[suggestion_part(code = "if ")]
     pub before_cond: Span,
@@ -477,7 +537,7 @@ pub(crate) struct TernaryOperatorSuggestion {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_extra_if_in_let_else,
+    "remove the `if` if you meant to write a `let...else` statement",
     applicability = "maybe-incorrect",
     code = "",
     style = "verbose"
@@ -488,18 +548,20 @@ pub(crate) struct IfExpressionLetSomeSub {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_if_expression_missing_condition)]
+#[diag("missing condition for `if` expression")]
 pub(crate) struct IfExpressionMissingCondition {
     #[primary_span]
-    #[label(parse_condition_label)]
+    #[label("expected condition here")]
     pub if_span: Span,
-    #[label(parse_block_label)]
+    #[label(
+        "if this block is the condition of the `if` expression, then it must be followed by another block"
+    )]
     pub block_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_expression_found_let)]
-#[note]
+#[diag("expected expression, found `let` statement")]
+#[note("only supported directly in conditions of `if` and `while` expressions")]
 pub(crate) struct ExpectedExpressionFoundLet {
     #[primary_span]
     pub span: Span,
@@ -512,7 +574,7 @@ pub(crate) struct ExpectedExpressionFoundLet {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_or_in_let_chain)]
+#[diag("`||` operators are not supported in let chain conditions")]
 pub(crate) struct OrInLetChain {
     #[primary_span]
     pub span: Span,
@@ -520,7 +582,7 @@ pub(crate) struct OrInLetChain {
 
 #[derive(Subdiagnostic, Clone, Copy)]
 #[multipart_suggestion(
-    parse_maybe_missing_let,
+    "you might have meant to continue the let-chain",
     applicability = "maybe-incorrect",
     style = "verbose"
 )]
@@ -531,7 +593,7 @@ pub(crate) struct MaybeMissingLet {
 
 #[derive(Subdiagnostic, Clone, Copy)]
 #[multipart_suggestion(
-    parse_maybe_comparison,
+    "you might have meant to compare for equality",
     applicability = "maybe-incorrect",
     style = "verbose"
 )]
@@ -541,56 +603,71 @@ pub(crate) struct MaybeComparison {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expect_eq_instead_of_eqeq)]
+#[diag("expected `=`, found `==`")]
 pub(crate) struct ExpectedEqForLetExpr {
     #[primary_span]
     pub span: Span,
-    #[suggestion(applicability = "maybe-incorrect", code = "=", style = "verbose")]
+    #[suggestion(
+        "consider using `=` here",
+        applicability = "maybe-incorrect",
+        code = "=",
+        style = "verbose"
+    )]
     pub sugg_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_else_block)]
+#[diag("expected `{\"{\"}`, found {$first_tok}")]
 pub(crate) struct ExpectedElseBlock {
     #[primary_span]
     pub first_tok_span: Span,
     pub first_tok: String,
-    #[label]
+    #[label("expected an `if` or a block after this `else`")]
     pub else_span: Span,
-    #[suggestion(applicability = "maybe-incorrect", code = "if ", style = "verbose")]
+    #[suggestion(
+        "add an `if` if this is the condition of a chained `else if` statement",
+        applicability = "maybe-incorrect",
+        code = "if ",
+        style = "verbose"
+    )]
     pub condition_start: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_struct_field)]
+#[diag("expected one of `,`, `:`, or `{\"}\"}`, found `{$token}`")]
 pub(crate) struct ExpectedStructField {
     #[primary_span]
-    #[label]
+    #[label("expected one of `,`, `:`, or `{\"}\"}`")]
     pub span: Span,
     pub token: Token,
-    #[label(parse_ident_label)]
+    #[label("while parsing this struct field")]
     pub ident_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_outer_attribute_not_allowed_on_if_else)]
+#[diag("outer attributes are not allowed on `if` and `else` branches")]
 pub(crate) struct OuterAttributeNotAllowedOnIfElse {
     #[primary_span]
     pub last: Span,
 
-    #[label(parse_branch_label)]
+    #[label("the attributes are attached to this branch")]
     pub branch_span: Span,
 
-    #[label(parse_ctx_label)]
+    #[label("the branch belongs to this `{$ctx}`")]
     pub ctx_span: Span,
     pub ctx: String,
 
-    #[suggestion(applicability = "machine-applicable", code = "", style = "verbose")]
+    #[suggestion(
+        "remove the attributes",
+        applicability = "machine-applicable",
+        code = "",
+        style = "verbose"
+    )]
     pub attributes: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_in_in_for_loop)]
+#[diag("missing `in` in `for` loop")]
 pub(crate) struct MissingInInForLoop {
     #[primary_span]
     pub span: Span,
@@ -602,20 +679,36 @@ pub(crate) struct MissingInInForLoop {
 pub(crate) enum MissingInInForLoopSub {
     // User wrote `for pat of expr {}`
     // Has been misleading, at least in the past (closed Issue #48492), thus maybe-incorrect
-    #[suggestion(parse_use_in, style = "verbose", applicability = "maybe-incorrect", code = "in")]
+    #[suggestion(
+        "try using `in` here instead",
+        style = "verbose",
+        applicability = "maybe-incorrect",
+        code = "in"
+    )]
     InNotOf(#[primary_span] Span),
     // User wrote `for pat = expr {}`
-    #[suggestion(parse_use_in, style = "verbose", applicability = "maybe-incorrect", code = "in")]
+    #[suggestion(
+        "try using `in` here instead",
+        style = "verbose",
+        applicability = "maybe-incorrect",
+        code = "in"
+    )]
     InNotEq(#[primary_span] Span),
-    #[suggestion(parse_add_in, style = "verbose", applicability = "maybe-incorrect", code = " in ")]
+    #[suggestion(
+        "try adding `in` here",
+        style = "verbose",
+        applicability = "maybe-incorrect",
+        code = " in "
+    )]
     AddIn(#[primary_span] Span),
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_expression_in_for_loop)]
+#[diag("missing expression to iterate on in `for` loop")]
 pub(crate) struct MissingExpressionInForLoop {
     #[primary_span]
     #[suggestion(
+        "try adding an expression to the `for` loop",
         code = "/* expression */ ",
         applicability = "has-placeholders",
         style = "verbose"
@@ -624,63 +717,80 @@ pub(crate) struct MissingExpressionInForLoop {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_loop_else)]
-#[note]
+#[diag("`{$loop_kind}...else` loops are not supported")]
+#[note(
+    "consider moving this `else` clause to a separate `if` statement and use a `bool` variable to control if it should run"
+)]
 pub(crate) struct LoopElseNotSupported {
     #[primary_span]
     pub span: Span,
     pub loop_kind: &'static str,
-    #[label(parse_loop_keyword)]
+    #[label("`else` is attached to this loop")]
     pub loop_kw: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_comma_after_match_arm)]
+#[diag("expected `,` following `match` arm")]
 pub(crate) struct MissingCommaAfterMatchArm {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = ",", style = "verbose")]
+    #[suggestion(
+        "missing a comma here to end this `match` arm",
+        applicability = "machine-applicable",
+        code = ",",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_catch_after_try)]
-#[help]
+#[diag("keyword `catch` cannot follow a `try` block")]
+#[help("try using `match` on the result of the `try` block instead")]
 pub(crate) struct CatchAfterTry {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_comma_after_base_struct)]
-#[note]
+#[diag("cannot use a comma after the base struct")]
+#[note("the base struct must always be the last field")]
 pub(crate) struct CommaAfterBaseStruct {
     #[primary_span]
     pub span: Span,
-    #[suggestion(style = "verbose", applicability = "machine-applicable", code = "")]
+    #[suggestion(
+        "remove this comma",
+        style = "verbose",
+        applicability = "machine-applicable",
+        code = ""
+    )]
     pub comma: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_eq_field_init)]
+#[diag("expected `:`, found `=`")]
 pub(crate) struct EqFieldInit {
     #[primary_span]
     pub span: Span,
-    #[suggestion(applicability = "machine-applicable", code = ":", style = "verbose")]
+    #[suggestion(
+        "replace equals symbol with a colon",
+        applicability = "machine-applicable",
+        code = ":",
+        style = "verbose"
+    )]
     pub eq: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dotdotdot)]
+#[diag("unexpected token: `...`")]
 pub(crate) struct DotDotDot {
     #[primary_span]
     #[suggestion(
-        parse_suggest_exclusive_range,
+        "use `..` for an exclusive range",
         applicability = "maybe-incorrect",
         code = "..",
         style = "verbose"
     )]
     #[suggestion(
-        parse_suggest_inclusive_range,
+        "or `..=` for an inclusive range",
         applicability = "maybe-incorrect",
         code = "..=",
         style = "verbose"
@@ -689,66 +799,86 @@ pub(crate) struct DotDotDot {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_left_arrow_operator)]
+#[diag("unexpected token: `<-`")]
 pub(crate) struct LeftArrowOperator {
     #[primary_span]
-    #[suggestion(applicability = "maybe-incorrect", code = "< -", style = "verbose")]
+    #[suggestion(
+        "if you meant to write a comparison against a negative value, add a space in between `<` and `-`",
+        applicability = "maybe-incorrect",
+        code = "< -",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_remove_let)]
+#[diag("expected pattern, found `let`")]
 pub(crate) struct RemoveLet {
     #[primary_span]
     pub span: Span,
-    #[suggestion(applicability = "machine-applicable", code = "", style = "verbose")]
+    #[suggestion(
+        "remove the unnecessary `let` keyword",
+        applicability = "machine-applicable",
+        code = "",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_use_eq_instead)]
+#[diag("unexpected `==`")]
 pub(crate) struct UseEqInstead {
     #[primary_span]
-    #[suggestion(style = "verbose", applicability = "machine-applicable", code = "=")]
+    #[suggestion(
+        "try using `=` instead",
+        style = "verbose",
+        applicability = "machine-applicable",
+        code = "="
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_use_empty_block_not_semi)]
+#[diag("expected { \"`{}`\" }, found `;`")]
 pub(crate) struct UseEmptyBlockNotSemi {
     #[primary_span]
-    #[suggestion(style = "hidden", applicability = "machine-applicable", code = "{{}}")]
+    #[suggestion(
+        r#"try using { "`{}`" } instead"#,
+        style = "hidden",
+        applicability = "machine-applicable",
+        code = "{{}}"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_comparison_interpreted_as_generic)]
+#[diag("`<` is interpreted as a start of generic arguments for `{$type}`, not a comparison")]
 pub(crate) struct ComparisonInterpretedAsGeneric {
     #[primary_span]
-    #[label(parse_label_comparison)]
+    #[label("not interpreted as comparison")]
     pub comparison: Span,
     pub r#type: Path,
-    #[label(parse_label_args)]
+    #[label("interpreted as generic arguments")]
     pub args: Span,
     #[subdiagnostic]
     pub suggestion: ComparisonOrShiftInterpretedAsGenericSugg,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_shift_interpreted_as_generic)]
+#[diag("`<<` is interpreted as a start of generic arguments for `{$type}`, not a shift")]
 pub(crate) struct ShiftInterpretedAsGeneric {
     #[primary_span]
-    #[label(parse_label_comparison)]
+    #[label("not interpreted as shift")]
     pub shift: Span,
     pub r#type: Path,
-    #[label(parse_label_args)]
+    #[label("interpreted as generic arguments")]
     pub args: Span,
     #[subdiagnostic]
     pub suggestion: ComparisonOrShiftInterpretedAsGenericSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("BROKEN", applicability = "machine-applicable")]
 pub(crate) struct ComparisonOrShiftInterpretedAsGenericSugg {
     #[suggestion_part(code = "(")]
     pub left: Span,
@@ -757,10 +887,10 @@ pub(crate) struct ComparisonOrShiftInterpretedAsGenericSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_found_expr_would_be_stmt)]
+#[diag("expected expression, found `{$token}`")]
 pub(crate) struct FoundExprWouldBeStmt {
     #[primary_span]
-    #[label]
+    #[label("expected expression")]
     pub span: Span,
     pub token: Token,
     #[subdiagnostic]
@@ -768,81 +898,83 @@ pub(crate) struct FoundExprWouldBeStmt {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_extra_characters_after_close)]
+#[diag("extra characters after frontmatter close are not allowed")]
 pub(crate) struct FrontmatterExtraCharactersAfterClose {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_invalid_infostring)]
-#[note]
+#[diag("invalid infostring for frontmatter")]
+#[note("frontmatter infostrings must be a single identifier immediately following the opening")]
 pub(crate) struct FrontmatterInvalidInfostring {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_invalid_opening_preceding_whitespace)]
+#[diag("invalid preceding whitespace for frontmatter opening")]
 pub(crate) struct FrontmatterInvalidOpeningPrecedingWhitespace {
     #[primary_span]
     pub span: Span,
-    #[note]
+    #[note("frontmatter opening should not be preceded by whitespace")]
     pub note_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_unclosed)]
+#[diag("unclosed frontmatter")]
 pub(crate) struct FrontmatterUnclosed {
     #[primary_span]
     pub span: Span,
-    #[note]
+    #[note("frontmatter opening here was not closed")]
     pub note_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_invalid_close_preceding_whitespace)]
+#[diag("invalid preceding whitespace for frontmatter close")]
 pub(crate) struct FrontmatterInvalidClosingPrecedingWhitespace {
     #[primary_span]
     pub span: Span,
-    #[note]
+    #[note("frontmatter close should not be preceded by whitespace")]
     pub note_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_length_mismatch)]
+#[diag("frontmatter close does not match the opening")]
 pub(crate) struct FrontmatterLengthMismatch {
     #[primary_span]
     pub span: Span,
-    #[label(parse_label_opening)]
+    #[label("the opening here has {$len_opening} dashes...")]
     pub opening: Span,
-    #[label(parse_label_close)]
+    #[label("...while the close has {$len_close} dashes")]
     pub close: Span,
     pub len_opening: usize,
     pub len_close: usize,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_frontmatter_too_many_dashes)]
+#[diag(
+    "too many `-` symbols: frontmatter openings may be delimited by up to 255 `-` symbols, but found {$len_opening}"
+)]
 pub(crate) struct FrontmatterTooManyDashes {
     pub len_opening: usize,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_bare_cr_in_frontmatter)]
+#[diag("bare CR not allowed in frontmatter")]
 pub(crate) struct BareCrFrontmatter {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_leading_plus_not_supported)]
+#[diag("leading `+` is not supported")]
 pub(crate) struct LeadingPlusNotSupported {
     #[primary_span]
-    #[label]
+    #[label("unexpected `+`")]
     pub span: Span,
     #[suggestion(
-        parse_suggestion_remove_plus,
+        "try removing the `+`",
         style = "verbose",
         code = "",
         applicability = "machine-applicable"
@@ -853,7 +985,7 @@ pub(crate) struct LeadingPlusNotSupported {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_parentheses_with_struct_fields)]
+#[diag("invalid `struct` delimiters or `fn` call arguments")]
 pub(crate) struct ParenthesesWithStructFields {
     #[primary_span]
     pub span: Span,
@@ -865,7 +997,10 @@ pub(crate) struct ParenthesesWithStructFields {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion_braces_for_struct, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "if `{$type}` is a struct, use braces as delimiters",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct BracesForStructLiteral {
     #[suggestion_part(code = " {{ ")]
     pub first: Span,
@@ -874,14 +1009,19 @@ pub(crate) struct BracesForStructLiteral {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion_no_fields_for_fn, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "if `{$type}` is a function, use the arguments directly",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct NoFieldsForFnCall {
     #[suggestion_part(code = "")]
     pub fields: Vec,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_labeled_loop_in_break)]
+#[diag(
+    "parentheses are required around this expression to avoid confusion with a labeled break expression"
+)]
 pub(crate) struct LabeledLoopInBreak {
     #[primary_span]
     pub span: Span,
@@ -892,7 +1032,7 @@ pub(crate) struct LabeledLoopInBreak {
 #[derive(Subdiagnostic)]
 pub(crate) enum WrapInParentheses {
     #[multipart_suggestion(
-        parse_sugg_wrap_expression_in_parentheses,
+        "wrap the expression in parentheses",
         applicability = "machine-applicable"
     )]
     Expression {
@@ -902,7 +1042,7 @@ pub(crate) enum WrapInParentheses {
         right: Span,
     },
     #[multipart_suggestion(
-        parse_sugg_wrap_macro_in_parentheses,
+        "use parentheses instead of braces for this macro",
         applicability = "machine-applicable"
     )]
     MacroArgs {
@@ -914,7 +1054,7 @@ pub(crate) enum WrapInParentheses {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_array_brackets_instead_of_braces)]
+#[diag("this is a block expression, not an array")]
 pub(crate) struct ArrayBracketsInsteadOfBraces {
     #[primary_span]
     pub span: Span,
@@ -923,7 +1063,10 @@ pub(crate) struct ArrayBracketsInsteadOfBraces {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "to make an array, use square brackets instead of curly braces",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct ArrayBracketsInsteadOfBracesSugg {
     #[suggestion_part(code = "[")]
     pub left: Span,
@@ -932,12 +1075,17 @@ pub(crate) struct ArrayBracketsInsteadOfBracesSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_match_arm_body_without_braces)]
+#[diag("`match` arm body without braces")]
 pub(crate) struct MatchArmBodyWithoutBraces {
     #[primary_span]
-    #[label(parse_label_statements)]
+    #[label(
+        "{$num_statements ->
+            [one] this statement is not surrounded by a body
+            *[other] these statements are not surrounded by a body
+        }"
+    )]
     pub statements: Span,
-    #[label(parse_label_arrow)]
+    #[label("while parsing the `match` arm starting here")]
     pub arrow: Span,
     pub num_statements: usize,
     #[subdiagnostic]
@@ -945,12 +1093,12 @@ pub(crate) struct MatchArmBodyWithoutBraces {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_inclusive_range_extra_equals)]
-#[note]
+#[diag("unexpected `=` after inclusive range")]
+#[note("inclusive ranges end with a single equals sign (`..=`)")]
 pub(crate) struct InclusiveRangeExtraEquals {
     #[primary_span]
     #[suggestion(
-        parse_suggestion_remove_eq,
+        "use `..=` instead",
         style = "verbose",
         code = "..=",
         applicability = "maybe-incorrect"
@@ -959,24 +1107,29 @@ pub(crate) struct InclusiveRangeExtraEquals {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_inclusive_range_match_arrow)]
+#[diag("unexpected `>` after inclusive range")]
 pub(crate) struct InclusiveRangeMatchArrow {
     #[primary_span]
     pub arrow: Span,
-    #[label]
+    #[label("this is parsed as an inclusive range `..=`")]
     pub span: Span,
-    #[suggestion(style = "verbose", code = " ", applicability = "machine-applicable")]
+    #[suggestion(
+        "add a space between the pattern and `=>`",
+        style = "verbose",
+        code = " ",
+        applicability = "machine-applicable"
+    )]
     pub after_pat: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_inclusive_range_no_end, code = E0586)]
-#[note]
+#[diag("inclusive range with no end", code = E0586)]
+#[note("inclusive ranges must be bounded at the end (`..=b` or `a..=b`)")]
 pub(crate) struct InclusiveRangeNoEnd {
     #[primary_span]
     pub span: Span,
     #[suggestion(
-        parse_suggestion_open_range,
+        "use `..` instead",
         code = "",
         applicability = "machine-applicable",
         style = "verbose"
@@ -986,7 +1139,13 @@ pub(crate) struct InclusiveRangeNoEnd {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum MatchArmBodyWithoutBracesSugg {
-    #[multipart_suggestion(parse_suggestion_add_braces, applicability = "machine-applicable")]
+    #[multipart_suggestion(
+        "surround the {$num_statements ->
+            [one] statement
+           *[other] statements
+        } with a body",
+        applicability = "machine-applicable"
+    )]
     AddBraces {
         #[suggestion_part(code = "{{ ")]
         left: Span,
@@ -994,7 +1153,7 @@ pub(crate) enum MatchArmBodyWithoutBracesSugg {
         right: Span,
     },
     #[suggestion(
-        parse_suggestion_use_comma_not_semicolon,
+        "replace `;` with `,` to end a `match` arm expression",
         code = ",",
         applicability = "machine-applicable",
         style = "verbose"
@@ -1006,7 +1165,7 @@ pub(crate) enum MatchArmBodyWithoutBracesSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_struct_literal_not_allowed_here)]
+#[diag("struct literals are not allowed here")]
 pub(crate) struct StructLiteralNotAllowedHere {
     #[primary_span]
     pub span: Span,
@@ -1015,7 +1174,10 @@ pub(crate) struct StructLiteralNotAllowedHere {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "surround the struct literal with parentheses",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct StructLiteralNotAllowedHereSugg {
     #[suggestion_part(code = "(")]
     pub left: Span,
@@ -1024,80 +1186,105 @@ pub(crate) struct StructLiteralNotAllowedHereSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_literal_suffix_on_tuple_index)]
+#[diag("suffixes on a tuple index are invalid")]
 pub(crate) struct InvalidLiteralSuffixOnTupleIndex {
     #[primary_span]
-    #[label]
+    #[label("invalid suffix `{$suffix}`")]
     pub span: Span,
     pub suffix: Symbol,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_non_string_abi_literal)]
+#[diag("non-string ABI literal")]
 pub(crate) struct NonStringAbiLiteral {
     #[primary_span]
-    #[suggestion(code = "\"C\"", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "specify the ABI with a string literal",
+        code = "\"C\"",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_mismatched_closing_delimiter)]
+#[diag("mismatched closing delimiter: `{$delimiter}`")]
 pub(crate) struct MismatchedClosingDelimiter {
     #[primary_span]
     pub spans: Vec,
     pub delimiter: String,
-    #[label(parse_label_unmatched)]
+    #[label("mismatched closing delimiter")]
     pub unmatched: Span,
-    #[label(parse_label_opening_candidate)]
+    #[label("closing delimiter possibly meant for this")]
     pub opening_candidate: Option,
-    #[label(parse_label_unclosed)]
+    #[label("unclosed delimiter")]
     pub unclosed: Option,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_visibility_restriction, code = E0704)]
-#[help]
+#[diag("incorrect visibility restriction", code = E0704)]
+#[help(
+    "some possible visibility restrictions are:
+    `pub(crate)`: visible only on the current crate
+    `pub(super)`: visible only in the current module's parent
+    `pub(in path::to::module)`: visible only on the specified path"
+)]
 pub(crate) struct IncorrectVisibilityRestriction {
     #[primary_span]
-    #[suggestion(code = "in {inner_str}", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "make this visible only to module `{$inner_str}` with `in`",
+        code = "in {inner_str}",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
     pub inner_str: String,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_assignment_else_not_allowed)]
+#[diag(" ... else {\"{\"} ... {\"}\"} is not allowed")]
 pub(crate) struct AssignmentElseNotAllowed {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_statement_after_outer_attr)]
+#[diag("expected statement after outer attribute")]
 pub(crate) struct ExpectedStatementAfterOuterAttr {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_doc_comment_does_not_document_anything, code = E0585)]
-#[help]
+#[diag("found a documentation comment that doesn't document anything", code = E0585)]
+#[help("doc comments must come before what they document, if a comment was intended use `//`")]
 pub(crate) struct DocCommentDoesNotDocumentAnything {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = ",", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "missing comma here",
+        code = ",",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub missing_comma: Option,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_const_let_mutually_exclusive)]
+#[diag("`const` and `let` are mutually exclusive")]
 pub(crate) struct ConstLetMutuallyExclusive {
     #[primary_span]
-    #[suggestion(code = "const", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "remove `let`",
+        code = "const",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_expression_in_let_else)]
+#[diag("a `{$operator}` expression cannot be directly assigned in `let...else`")]
 pub(crate) struct InvalidExpressionInLetElse {
     #[primary_span]
     pub span: Span,
@@ -1107,7 +1294,7 @@ pub(crate) struct InvalidExpressionInLetElse {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_curly_in_let_else)]
+#[diag("right curly brace `{\"}\"}` before `else` in a `let...else` statement not allowed")]
 pub(crate) struct InvalidCurlyInLetElse {
     #[primary_span]
     pub span: Span,
@@ -1116,25 +1303,32 @@ pub(crate) struct InvalidCurlyInLetElse {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_compound_assignment_expression_in_let)]
-#[help]
+#[diag("can't reassign to an uninitialized variable")]
+#[help("if you meant to overwrite, remove the `let` binding")]
 pub(crate) struct CompoundAssignmentExpressionInLet {
     #[primary_span]
     pub span: Span,
-    #[suggestion(style = "verbose", code = "", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "initialize the variable",
+        style = "verbose",
+        code = "",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_suffixed_literal_in_attribute)]
-#[help]
+#[diag("suffixed literals are not allowed in attributes")]
+#[help(
+    "instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), use an unsuffixed version (`1`, `1.0`, etc.)"
+)]
 pub(crate) struct SuffixedLiteralInAttribute {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_meta_item)]
+#[diag("expected unsuffixed literal, found {$descr}")]
 pub(crate) struct InvalidMetaItem {
     #[primary_span]
     pub span: Span,
@@ -1144,7 +1338,10 @@ pub(crate) struct InvalidMetaItem {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_quote_ident_sugg, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "surround the identifier with quotation marks to make it into a string literal",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct InvalidMetaItemQuoteIdentSugg {
     #[suggestion_part(code = "\"")]
     pub before: Span,
@@ -1154,7 +1351,7 @@ pub(crate) struct InvalidMetaItemQuoteIdentSugg {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_sugg_escape_identifier,
+    "escape `{$ident_name}` to use it as an identifier",
     style = "verbose",
     applicability = "maybe-incorrect",
     code = "r#"
@@ -1167,7 +1364,7 @@ pub(crate) struct SuggEscapeIdentifier {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_sugg_remove_comma,
+    "remove this comma",
     applicability = "machine-applicable",
     code = "",
     style = "verbose"
@@ -1179,7 +1376,7 @@ pub(crate) struct SuggRemoveComma {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_sugg_add_let_for_stmt,
+    "you might have meant to introduce a new binding",
     style = "verbose",
     applicability = "maybe-incorrect",
     code = "let "
@@ -1191,17 +1388,17 @@ pub(crate) struct SuggAddMissingLetStmt {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum ExpectedIdentifierFound {
-    #[label(parse_expected_identifier_found_reserved_identifier)]
+    #[label("expected identifier, found reserved identifier")]
     ReservedIdentifier(#[primary_span] Span),
-    #[label(parse_expected_identifier_found_keyword)]
+    #[label("expected identifier, found keyword")]
     Keyword(#[primary_span] Span),
-    #[label(parse_expected_identifier_found_reserved_keyword)]
+    #[label("expected identifier, found reserved keyword")]
     ReservedKeyword(#[primary_span] Span),
-    #[label(parse_expected_identifier_found_doc_comment)]
+    #[label("expected identifier, found doc comment")]
     DocComment(#[primary_span] Span),
-    #[label(parse_expected_identifier_found_metavar)]
+    #[label("expected identifier, found metavariable")]
     MetaVar(#[primary_span] Span),
-    #[label(parse_expected_identifier)]
+    #[label("expected identifier")]
     Other(#[primary_span] Span),
 }
 
@@ -1239,22 +1436,22 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedIdentifier {
             level,
             match token_descr {
                 Some(TokenDescription::ReservedIdentifier) => {
-                    fluent::parse_expected_identifier_found_reserved_identifier_str
+                    inline_fluent!("expected identifier, found reserved identifier `{$token}`")
                 }
                 Some(TokenDescription::Keyword) => {
-                    fluent::parse_expected_identifier_found_keyword_str
+                    inline_fluent!("expected identifier, found keyword `{$token}`")
                 }
                 Some(TokenDescription::ReservedKeyword) => {
-                    fluent::parse_expected_identifier_found_reserved_keyword_str
+                    inline_fluent!("expected identifier, found reserved keyword `{$token}`")
                 }
                 Some(TokenDescription::DocComment) => {
-                    fluent::parse_expected_identifier_found_doc_comment_str
+                    inline_fluent!("expected identifier, found doc comment `{$token}`")
                 }
                 Some(TokenDescription::MetaVar(_)) => {
                     add_token = false;
-                    fluent::parse_expected_identifier_found_metavar_str
+                    inline_fluent!("expected identifier, found metavariable")
                 }
-                None => fluent::parse_expected_identifier_found_str,
+                None => inline_fluent!("expected identifier, found `{$token}`"),
             },
         );
         diag.span(self.span);
@@ -1281,7 +1478,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedIdentifier {
 }
 
 #[derive(Subdiagnostic)]
-#[help(parse_invalid_identifier_with_leading_number)]
+#[help("identifiers cannot start with a number")]
 pub(crate) struct HelpIdentifierStartsWithNumber {
     #[primary_span]
     pub num_span: Span,
@@ -1306,20 +1503,22 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedSemi {
             level,
             match token_descr {
                 Some(TokenDescription::ReservedIdentifier) => {
-                    fluent::parse_expected_semi_found_reserved_identifier_str
+                    inline_fluent!("expected `;`, found reserved identifier `{$token}`")
+                }
+                Some(TokenDescription::Keyword) => {
+                    inline_fluent!("expected `;`, found keyword `{$token}`")
                 }
-                Some(TokenDescription::Keyword) => fluent::parse_expected_semi_found_keyword_str,
                 Some(TokenDescription::ReservedKeyword) => {
-                    fluent::parse_expected_semi_found_reserved_keyword_str
+                    inline_fluent!("expected `;`, found reserved keyword `{$token}`")
                 }
                 Some(TokenDescription::DocComment) => {
-                    fluent::parse_expected_semi_found_doc_comment_str
+                    inline_fluent!("expected `;`, found doc comment `{$token}`")
                 }
                 Some(TokenDescription::MetaVar(_)) => {
                     add_token = false;
-                    fluent::parse_expected_semi_found_metavar_str
+                    inline_fluent!("expected `;`, found metavariable")
                 }
-                None => fluent::parse_expected_semi_found_str,
+                None => inline_fluent!("expected `;`, found `{$token}`"),
             },
         );
         diag.span(self.span);
@@ -1328,7 +1527,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedSemi {
         }
 
         if let Some(unexpected_token_label) = self.unexpected_token_label {
-            diag.span_label(unexpected_token_label, fluent::parse_label_unexpected_token);
+            diag.span_label(unexpected_token_label, inline_fluent!("unexpected token"));
         }
 
         self.sugg.add_to_diag(&mut diag);
@@ -1340,23 +1539,18 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedSemi {
 #[derive(Subdiagnostic)]
 pub(crate) enum ExpectedSemiSugg {
     #[suggestion(
-        parse_sugg_change_this_to_semi,
+        "change this to `;`",
         code = ";",
         applicability = "machine-applicable",
         style = "short"
     )]
     ChangeToSemi(#[primary_span] Span),
-    #[suggestion(
-        parse_sugg_add_semi,
-        code = ";",
-        applicability = "machine-applicable",
-        style = "short"
-    )]
+    #[suggestion("add `;` here", code = ";", applicability = "machine-applicable", style = "short")]
     AddSemi(#[primary_span] Span),
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_struct_literal_body_without_path)]
+#[diag("struct literal body without path")]
 pub(crate) struct StructLiteralBodyWithoutPath {
     #[primary_span]
     pub span: Span,
@@ -1365,7 +1559,10 @@ pub(crate) struct StructLiteralBodyWithoutPath {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "has-placeholders")]
+#[multipart_suggestion(
+    "you might have forgotten to add the struct literal inside the block",
+    applicability = "has-placeholders"
+)]
 pub(crate) struct StructLiteralBodyWithoutPathSugg {
     #[suggestion_part(code = "{{ SomeStruct ")]
     pub before: Span,
@@ -1374,16 +1571,29 @@ pub(crate) struct StructLiteralBodyWithoutPathSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unmatched_angle_brackets)]
+#[diag(
+    "{$num_extra_brackets ->
+        [one] unmatched angle bracket
+        *[other] unmatched angle brackets
+    }"
+)]
 pub(crate) struct UnmatchedAngleBrackets {
     #[primary_span]
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "{$num_extra_brackets ->
+            [one] remove extra angle bracket
+            *[other] remove extra angle brackets
+        }",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
     pub num_extra_brackets: usize,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_generic_parameters_without_angle_brackets)]
+#[diag("generic parameters without surrounding angle brackets")]
 pub(crate) struct GenericParamsWithoutAngleBrackets {
     #[primary_span]
     pub span: Span,
@@ -1392,7 +1602,10 @@ pub(crate) struct GenericParamsWithoutAngleBrackets {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "surround the type parameters with angle brackets",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct GenericParamsWithoutAngleBracketsSugg {
     #[suggestion_part(code = "<")]
     pub left: Span,
@@ -1401,19 +1614,19 @@ pub(crate) struct GenericParamsWithoutAngleBracketsSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_comparison_operators_cannot_be_chained)]
+#[diag("comparison operators cannot be chained")]
 pub(crate) struct ComparisonOperatorsCannotBeChained {
     #[primary_span]
     pub span: Vec,
     #[suggestion(
-        parse_sugg_turbofish_syntax,
+        "use `::<...>` instead of `<...>` to specify lifetime, type, or const arguments",
         style = "verbose",
         code = "::",
         applicability = "maybe-incorrect"
     )]
     pub suggest_turbofish: Option,
-    #[help(parse_sugg_turbofish_syntax)]
-    #[help(parse_sugg_parentheses_for_function_args)]
+    #[help("use `::<...>` instead of `<...>` to specify lifetime, type, or const arguments")]
+    #[help("or use `(...)` if you meant to specify fn arguments")]
     pub help_turbofish: bool,
     #[subdiagnostic]
     pub chaining_sugg: Option,
@@ -1422,7 +1635,7 @@ pub(crate) struct ComparisonOperatorsCannotBeChained {
 #[derive(Subdiagnostic)]
 pub(crate) enum ComparisonOperatorsCannotBeChainedSugg {
     #[suggestion(
-        parse_sugg_split_comparison,
+        "split the comparison into two",
         style = "verbose",
         code = " && {middle_term}",
         applicability = "maybe-incorrect"
@@ -1432,7 +1645,7 @@ pub(crate) enum ComparisonOperatorsCannotBeChainedSugg {
         span: Span,
         middle_term: String,
     },
-    #[multipart_suggestion(parse_sugg_parenthesize, applicability = "maybe-incorrect")]
+    #[multipart_suggestion("parenthesize the comparison", applicability = "maybe-incorrect")]
     Parenthesize {
         #[suggestion_part(code = "(")]
         left: Span,
@@ -1442,17 +1655,20 @@ pub(crate) enum ComparisonOperatorsCannotBeChainedSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_question_mark_in_type)]
+#[diag("invalid `?` in type")]
 pub(crate) struct QuestionMarkInType {
     #[primary_span]
-    #[label]
+    #[label("`?` is only allowed on expressions, not types")]
     pub span: Span,
     #[subdiagnostic]
     pub sugg: QuestionMarkInTypeSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "if you meant to express that the type might not contain a value, use the `Option` wrapper type",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct QuestionMarkInTypeSugg {
     #[suggestion_part(code = "Option<")]
     pub left: Span,
@@ -1461,7 +1677,7 @@ pub(crate) struct QuestionMarkInTypeSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_parentheses_in_for_head)]
+#[diag("unexpected parentheses surrounding `for` loop head")]
 pub(crate) struct ParenthesesInForHead {
     #[primary_span]
     pub span: Vec,
@@ -1470,7 +1686,7 @@ pub(crate) struct ParenthesesInForHead {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("remove parentheses in `for` loop", applicability = "machine-applicable")]
 pub(crate) struct ParenthesesInForHeadSugg {
     #[suggestion_part(code = " ")]
     pub left: Span,
@@ -1479,7 +1695,7 @@ pub(crate) struct ParenthesesInForHeadSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_parentheses_in_match_arm_pattern)]
+#[diag("unexpected parentheses surrounding `match` arm pattern")]
 pub(crate) struct ParenthesesInMatchPat {
     #[primary_span]
     pub span: Vec,
@@ -1488,7 +1704,10 @@ pub(crate) struct ParenthesesInMatchPat {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "remove parentheses surrounding the pattern",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct ParenthesesInMatchPatSugg {
     #[suggestion_part(code = "")]
     pub left: Span,
@@ -1497,67 +1716,82 @@ pub(crate) struct ParenthesesInMatchPatSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_doc_comment_on_param_type)]
+#[diag("documentation comments cannot be applied to a function parameter's type")]
 pub(crate) struct DocCommentOnParamType {
     #[primary_span]
-    #[label]
+    #[label("doc comments are not allowed here")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_attribute_on_param_type)]
+#[diag("attributes cannot be applied to a function parameter's type")]
 pub(crate) struct AttributeOnParamType {
     #[primary_span]
-    #[label]
+    #[label("attributes are not allowed here")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_attribute_on_type)]
+#[diag("attributes cannot be applied to types")]
 pub(crate) struct AttributeOnType {
     #[primary_span]
-    #[label]
+    #[label("attributes are not allowed here")]
     pub span: Span,
-    #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")]
+    #[suggestion(
+        "remove attribute from here",
+        code = "",
+        applicability = "machine-applicable",
+        style = "tool-only"
+    )]
     pub fix_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_attribute_on_generic_arg)]
+#[diag("attributes cannot be applied to generic arguments")]
 pub(crate) struct AttributeOnGenericArg {
     #[primary_span]
-    #[label]
+    #[label("attributes are not allowed here")]
     pub span: Span,
-    #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")]
+    #[suggestion(
+        "remove attribute from here",
+        code = "",
+        applicability = "machine-applicable",
+        style = "tool-only"
+    )]
     pub fix_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_attribute_on_empty_type)]
+#[diag("attributes cannot be applied here")]
 pub(crate) struct AttributeOnEmptyType {
     #[primary_span]
-    #[label]
+    #[label("attributes are not allowed here")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_pattern_method_param_without_body, code = E0642)]
+#[diag("patterns aren't allowed in methods without bodies", code = E0642)]
 pub(crate) struct PatternMethodParamWithoutBody {
     #[primary_span]
-    #[suggestion(code = "_", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "give this argument a name or use an underscore to ignore it",
+        code = "_",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_self_param_not_first)]
+#[diag("unexpected `self` parameter in function")]
 pub(crate) struct SelfParamNotFirst {
     #[primary_span]
-    #[label]
+    #[label("must be the first parameter of an associated function")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_const_generic_without_braces)]
+#[diag("expressions must be enclosed in braces to be used as const generic arguments")]
 pub(crate) struct ConstGenericWithoutBraces {
     #[primary_span]
     pub span: Span,
@@ -1566,7 +1800,10 @@ pub(crate) struct ConstGenericWithoutBraces {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "enclose the `const` expression in braces",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct ConstGenericWithoutBracesSugg {
     #[suggestion_part(code = "{{ ")]
     pub left: Span,
@@ -1575,10 +1812,10 @@ pub(crate) struct ConstGenericWithoutBracesSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_const_param_declaration)]
+#[diag("unexpected `const` parameter declaration")]
 pub(crate) struct UnexpectedConstParamDeclaration {
     #[primary_span]
-    #[label]
+    #[label("expected a `const` expression, not a parameter declaration")]
     pub span: Span,
     #[subdiagnostic]
     pub sugg: Option,
@@ -1586,7 +1823,10 @@ pub(crate) struct UnexpectedConstParamDeclaration {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum UnexpectedConstParamDeclarationSugg {
-    #[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+    #[multipart_suggestion(
+        "`const` parameters must be declared for the `impl`",
+        applicability = "machine-applicable"
+    )]
     AddParam {
         #[suggestion_part(code = "<{snippet}>")]
         impl_generics: Span,
@@ -1595,7 +1835,10 @@ pub(crate) enum UnexpectedConstParamDeclarationSugg {
         snippet: String,
         ident: String,
     },
-    #[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+    #[multipart_suggestion(
+        "`const` parameters must be declared for the `impl`",
+        applicability = "machine-applicable"
+    )]
     AppendParam {
         #[suggestion_part(code = ", {snippet}")]
         impl_generics_end: Span,
@@ -1607,41 +1850,61 @@ pub(crate) enum UnexpectedConstParamDeclarationSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_const_in_generic_param)]
+#[diag("expected lifetime, type, or constant, found keyword `const`")]
 pub(crate) struct UnexpectedConstInGenericParam {
     #[primary_span]
     pub span: Span,
-    #[suggestion(style = "verbose", code = "", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "the `const` keyword is only needed in the definition of the type",
+        style = "verbose",
+        code = "",
+        applicability = "maybe-incorrect"
+    )]
     pub to_remove: Option,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_async_move_order_incorrect)]
+#[diag("the order of `move` and `async` is incorrect")]
 pub(crate) struct AsyncMoveOrderIncorrect {
     #[primary_span]
-    #[suggestion(style = "verbose", code = "async move", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "try switching the order",
+        style = "verbose",
+        code = "async move",
+        applicability = "maybe-incorrect"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_async_use_order_incorrect)]
+#[diag("the order of `use` and `async` is incorrect")]
 pub(crate) struct AsyncUseOrderIncorrect {
     #[primary_span]
-    #[suggestion(style = "verbose", code = "async use", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "try switching the order",
+        style = "verbose",
+        code = "async use",
+        applicability = "maybe-incorrect"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_double_colon_in_bound)]
+#[diag("expected `:` followed by trait or lifetime")]
 pub(crate) struct DoubleColonInBound {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = ": ", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "use single colon",
+        code = ": ",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub between: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_fn_ptr_with_generics)]
+#[diag("function pointer types may not have generic parameters")]
 pub(crate) struct FnPtrWithGenerics {
     #[primary_span]
     pub span: Span,
@@ -1651,7 +1914,7 @@ pub(crate) struct FnPtrWithGenerics {
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    parse_misplaced_return_type,
+    "place the return type after the function parameters",
     style = "verbose",
     applicability = "maybe-incorrect"
 )]
@@ -1664,7 +1927,16 @@ pub(crate) struct MisplacedReturnType {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "consider moving the lifetime {$arity ->
+        [one] parameter
+        *[other] parameters
+    } to {$for_param_list_exists ->
+        [true] the
+        *[false] a
+    } `for` parameter list",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct FnPtrWithGenericsSugg {
     #[suggestion_part(code = "{snippet}")]
     pub left: Span,
@@ -1681,10 +1953,10 @@ pub(crate) struct FnTraitMissingParen {
 
 impl Subdiagnostic for FnTraitMissingParen {
     fn add_to_diag(self, diag: &mut Diag<'_, G>) {
-        diag.span_label(self.span, crate::fluent_generated::parse_fn_trait_missing_paren);
+        diag.span_label(self.span, inline_fluent!("`Fn` bounds require arguments in parentheses"));
         diag.span_suggestion_short(
             self.span.shrink_to_hi(),
-            crate::fluent_generated::parse_add_paren,
+            inline_fluent!("try adding parentheses"),
             "()",
             Applicability::MachineApplicable,
         );
@@ -1692,95 +1964,128 @@ impl Subdiagnostic for FnTraitMissingParen {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_if_with_if)]
+#[diag("unexpected `if` in the condition expression")]
 pub(crate) struct UnexpectedIfWithIf(
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = " ", style = "verbose")]
+    #[suggestion(
+        "remove the `if`",
+        applicability = "machine-applicable",
+        code = " ",
+        style = "verbose"
+    )]
     pub Span,
 );
 
 #[derive(Diagnostic)]
-#[diag(parse_maybe_fn_typo_with_impl)]
+#[diag("you might have meant to write `impl` instead of `fn`")]
 pub(crate) struct FnTypoWithImpl {
     #[primary_span]
-    #[suggestion(applicability = "maybe-incorrect", code = "impl", style = "verbose")]
+    #[suggestion(
+        "replace `fn` with `impl` here",
+        applicability = "maybe-incorrect",
+        code = "impl",
+        style = "verbose"
+    )]
     pub fn_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_fn_path_found_fn_keyword)]
+#[diag("expected identifier, found keyword `fn`")]
 pub(crate) struct ExpectedFnPathFoundFnKeyword {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = "Fn", style = "verbose")]
+    #[suggestion(
+        "use `Fn` to refer to the trait",
+        applicability = "machine-applicable",
+        code = "Fn",
+        style = "verbose"
+    )]
     pub fn_token_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_path_found_named_params)]
+#[diag("`Trait(...)` syntax does not support named parameters")]
 pub(crate) struct FnPathFoundNamedParams {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = "")]
+    #[suggestion("remove the parameter name", applicability = "machine-applicable", code = "")]
     pub named_param_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_path_found_c_variadic_params)]
+#[diag("`Trait(...)` syntax does not support c_variadic parameters")]
 pub(crate) struct PathFoundCVariadicParams {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = "")]
+    #[suggestion("remove the `...`", applicability = "machine-applicable", code = "")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_path_found_attribute_in_params)]
+#[diag("`Trait(...)` syntax does not support attributes in parameters")]
 pub(crate) struct PathFoundAttributeInParams {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = "")]
+    #[suggestion("remove the attributes", applicability = "machine-applicable", code = "")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_path_double_colon)]
+#[diag("path separator must be a double colon")]
 pub(crate) struct PathSingleColon {
     #[primary_span]
     pub span: Span,
 
-    #[suggestion(applicability = "machine-applicable", code = ":", style = "verbose")]
+    #[suggestion(
+        "use a double colon instead",
+        applicability = "machine-applicable",
+        code = ":",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_path_double_colon)]
+#[diag("path separator must be a double colon")]
 pub(crate) struct PathTripleColon {
     #[primary_span]
-    #[suggestion(applicability = "maybe-incorrect", code = "", style = "verbose")]
+    #[suggestion(
+        "use a double colon instead",
+        applicability = "maybe-incorrect",
+        code = "",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_colon_as_semi)]
+#[diag("statements are terminated with a semicolon")]
 pub(crate) struct ColonAsSemi {
     #[primary_span]
-    #[suggestion(applicability = "machine-applicable", code = ";", style = "verbose")]
+    #[suggestion(
+        "use a semicolon instead",
+        applicability = "machine-applicable",
+        code = ";",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_where_clause_before_tuple_struct_body)]
+#[diag("where clauses are not allowed before tuple struct bodies")]
 pub(crate) struct WhereClauseBeforeTupleStructBody {
     #[primary_span]
-    #[label]
+    #[label("unexpected where clause")]
     pub span: Span,
-    #[label(parse_name_label)]
+    #[label("while parsing this tuple struct")]
     pub name: Span,
-    #[label(parse_body_label)]
+    #[label("the struct body")]
     pub body: Span,
     #[subdiagnostic]
     pub sugg: Option,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "move the body before the where clause",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct WhereClauseBeforeTupleStructBodySugg {
     #[suggestion_part(code = "{snippet}")]
     pub left: Span,
@@ -1790,38 +2095,38 @@ pub(crate) struct WhereClauseBeforeTupleStructBodySugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_async_fn_in_2015, code = E0670)]
+#[diag("`async fn` is not permitted in Rust 2015", code = E0670)]
 pub(crate) struct AsyncFnIn2015 {
     #[primary_span]
-    #[label]
+    #[label("to use `async fn`, switch to Rust 2018 or later")]
     pub span: Span,
     #[subdiagnostic]
     pub help: HelpUseLatestEdition,
 }
 
 #[derive(Subdiagnostic)]
-#[label(parse_async_block_in_2015)]
+#[label("`async` blocks are only allowed in Rust 2018 or later")]
 pub(crate) struct AsyncBlockIn2015 {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_async_move_block_in_2015)]
+#[diag("`async move` blocks are only allowed in Rust 2018 or later")]
 pub(crate) struct AsyncMoveBlockIn2015 {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_async_use_block_in_2015)]
+#[diag("`async use` blocks are only allowed in Rust 2018 or later")]
 pub(crate) struct AsyncUseBlockIn2015 {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_async_bound_modifier_in_2015)]
+#[diag("`async` trait bounds are only allowed in Rust 2018 or later")]
 pub(crate) struct AsyncBoundModifierIn2015 {
     #[primary_span]
     pub span: Span,
@@ -1830,22 +2135,22 @@ pub(crate) struct AsyncBoundModifierIn2015 {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_let_chain_pre_2024)]
+#[diag("let chains are only allowed in Rust 2024 or later")]
 pub(crate) struct LetChainPre2024 {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_self_argument_pointer)]
+#[diag("cannot pass `self` by raw pointer")]
 pub(crate) struct SelfArgumentPointer {
     #[primary_span]
-    #[label]
+    #[label("cannot pass `self` by raw pointer")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_token_after_dot)]
+#[diag("unexpected token: {$actual}")]
 pub(crate) struct UnexpectedTokenAfterDot {
     #[primary_span]
     pub span: Span,
@@ -1853,64 +2158,84 @@ pub(crate) struct UnexpectedTokenAfterDot {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_visibility_not_followed_by_item)]
-#[help]
+#[diag("visibility `{$vis}` is not followed by an item")]
+#[help("you likely meant to define an item, e.g., `{$vis} fn foo() {\"{}\"}`")]
 pub(crate) struct VisibilityNotFollowedByItem {
     #[primary_span]
-    #[label]
+    #[label("the visibility")]
     pub span: Span,
     pub vis: Visibility,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_default_not_followed_by_item)]
-#[note]
+#[diag("`default` is not followed by an item")]
+#[note("only `fn`, `const`, `type`, or `impl` items may be prefixed by `default`")]
 pub(crate) struct DefaultNotFollowedByItem {
     #[primary_span]
-    #[label]
+    #[label("the `default` qualifier")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
 pub(crate) enum MissingKeywordForItemDefinition {
-    #[diag(parse_missing_enum_for_enum_definition)]
+    #[diag("missing `enum` for enum definition")]
     Enum {
         #[primary_span]
         span: Span,
-        #[suggestion(style = "verbose", applicability = "maybe-incorrect", code = "enum ")]
+        #[suggestion(
+            "add `enum` here to parse `{$ident}` as an enum",
+            style = "verbose",
+            applicability = "maybe-incorrect",
+            code = "enum "
+        )]
         insert_span: Span,
         ident: Ident,
     },
-    #[diag(parse_missing_enum_or_struct_for_item_definition)]
+    #[diag("missing `enum` or `struct` for enum or struct definition")]
     EnumOrStruct {
         #[primary_span]
         span: Span,
     },
-    #[diag(parse_missing_struct_for_struct_definition)]
+    #[diag("missing `struct` for struct definition")]
     Struct {
         #[primary_span]
         span: Span,
-        #[suggestion(style = "verbose", applicability = "maybe-incorrect", code = "struct ")]
+        #[suggestion(
+            "add `struct` here to parse `{$ident}` as a struct",
+            style = "verbose",
+            applicability = "maybe-incorrect",
+            code = "struct "
+        )]
         insert_span: Span,
         ident: Ident,
     },
-    #[diag(parse_missing_fn_for_function_definition)]
+    #[diag("missing `fn` for function definition")]
     Function {
         #[primary_span]
         span: Span,
-        #[suggestion(style = "verbose", applicability = "maybe-incorrect", code = "fn ")]
+        #[suggestion(
+            "add `fn` here to parse `{$ident}` as a function",
+            style = "verbose",
+            applicability = "maybe-incorrect",
+            code = "fn "
+        )]
         insert_span: Span,
         ident: Ident,
     },
-    #[diag(parse_missing_fn_for_method_definition)]
+    #[diag("missing `fn` for method definition")]
     Method {
         #[primary_span]
         span: Span,
-        #[suggestion(style = "verbose", applicability = "maybe-incorrect", code = "fn ")]
+        #[suggestion(
+            "add `fn` here to parse `{$ident}` as a method",
+            style = "verbose",
+            applicability = "maybe-incorrect",
+            code = "fn "
+        )]
         insert_span: Span,
         ident: Ident,
     },
-    #[diag(parse_missing_fn_or_struct_for_item_definition)]
+    #[diag("missing `fn` or `struct` for function or struct definition")]
     Ambiguous {
         #[primary_span]
         span: Span,
@@ -1922,7 +2247,7 @@ pub(crate) enum MissingKeywordForItemDefinition {
 #[derive(Subdiagnostic)]
 pub(crate) enum AmbiguousMissingKwForItemSub {
     #[suggestion(
-        parse_suggestion,
+        "if you meant to call a macro, try",
         applicability = "maybe-incorrect",
         code = "{snippet}!",
         style = "verbose"
@@ -1932,39 +2257,49 @@ pub(crate) enum AmbiguousMissingKwForItemSub {
         span: Span,
         snippet: String,
     },
-    #[help(parse_help)]
+    #[help("you likely meant to define an item, e.g., `{$vis} fn foo() {\"{}\"}`")]
     HelpMacro,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_fn_params)]
+#[diag("missing parameters for function definition")]
 pub(crate) struct MissingFnParams {
     #[primary_span]
-    #[suggestion(code = "()", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "add a parameter list",
+        code = "()",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_path_sep_in_fn_definition)]
+#[diag("invalid path separator in function definition")]
 pub(crate) struct InvalidPathSepInFnDefinition {
     #[primary_span]
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove invalid path separator",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_trait_in_trait_impl)]
+#[diag("missing trait in a trait impl")]
 pub(crate) struct MissingTraitInTraitImpl {
     #[primary_span]
     #[suggestion(
-        parse_suggestion_add_trait,
+        "add a trait here",
         code = " Trait ",
         applicability = "has-placeholders",
         style = "verbose"
     )]
     pub span: Span,
     #[suggestion(
-        parse_suggestion_remove_for,
+        "for an inherent impl, drop this `for`",
         code = "",
         applicability = "maybe-incorrect",
         style = "verbose"
@@ -1973,102 +2308,130 @@ pub(crate) struct MissingTraitInTraitImpl {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_for_in_trait_impl)]
+#[diag("missing `for` in a trait impl")]
 pub(crate) struct MissingForInTraitImpl {
     #[primary_span]
-    #[suggestion(style = "verbose", code = " for ", applicability = "machine-applicable")]
+    #[suggestion(
+        "add `for` here",
+        style = "verbose",
+        code = " for ",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_trait_in_trait_impl_found_type)]
+#[diag("expected a trait, found type")]
 pub(crate) struct ExpectedTraitInTraitImplFoundType {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_extra_impl_keyword_in_trait_impl)]
+#[diag("unexpected `impl` keyword")]
 pub(crate) struct ExtraImplKeywordInTraitImpl {
     #[primary_span]
-    #[suggestion(code = "", applicability = "maybe-incorrect", style = "short")]
+    #[suggestion(
+        "remove the extra `impl`",
+        code = "",
+        applicability = "maybe-incorrect",
+        style = "short"
+    )]
     pub extra_impl_kw: Span,
-    #[note]
+    #[note("this is parsed as an `impl Trait` type, but a trait is expected at this position")]
     pub impl_trait_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_bounds_not_allowed_on_trait_aliases)]
+#[diag("bounds are not allowed on trait aliases")]
 pub(crate) struct BoundsNotAllowedOnTraitAliases {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_trait_alias_cannot_be_auto)]
+#[diag("trait aliases cannot be `auto`")]
 pub(crate) struct TraitAliasCannotBeAuto {
     #[primary_span]
-    #[label(parse_trait_alias_cannot_be_auto)]
+    #[label("trait aliases cannot be `auto`")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_trait_alias_cannot_be_unsafe)]
+#[diag("trait aliases cannot be `unsafe`")]
 pub(crate) struct TraitAliasCannotBeUnsafe {
     #[primary_span]
-    #[label(parse_trait_alias_cannot_be_unsafe)]
+    #[label("trait aliases cannot be `unsafe`")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_associated_static_item_not_allowed)]
+#[diag("associated `static` items are not allowed")]
 pub(crate) struct AssociatedStaticItemNotAllowed {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_extern_crate_name_with_dashes)]
+#[diag("crate name using dashes are not valid in `extern crate` statements")]
 pub(crate) struct ExternCrateNameWithDashes {
     #[primary_span]
-    #[label]
+    #[label("dash-separated idents are not valid")]
     pub span: Span,
     #[subdiagnostic]
     pub sugg: ExternCrateNameWithDashesSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "if the original crate name uses dashes you need to use underscores in the code",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct ExternCrateNameWithDashesSugg {
     #[suggestion_part(code = "_")]
     pub dashes: Vec,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_extern_item_cannot_be_const)]
-#[note]
+#[diag("extern items cannot be `const`")]
+#[note("for more information, visit https://doc.rust-lang.org/std/keyword.extern.html")]
 pub(crate) struct ExternItemCannotBeConst {
     #[primary_span]
     pub ident_span: Span,
-    #[suggestion(code = "static ", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "try using a static value",
+        code = "static ",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub const_span: Option,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_const_global_cannot_be_mutable)]
+#[diag("const globals cannot be mutable")]
 pub(crate) struct ConstGlobalCannotBeMutable {
     #[primary_span]
-    #[label]
+    #[label("cannot be mutable")]
     pub ident_span: Span,
-    #[suggestion(code = "static", style = "verbose", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "you might want to declare a static instead",
+        code = "static",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
     pub const_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_const_type)]
+#[diag("missing type for `{$kind}` item")]
 pub(crate) struct MissingConstType {
     #[primary_span]
-    #[suggestion(code = "{colon} ", style = "verbose", applicability = "has-placeholders")]
+    #[suggestion(
+        "provide a type for the item",
+        code = "{colon} ",
+        style = "verbose",
+        applicability = "has-placeholders"
+    )]
     pub span: Span,
 
     pub kind: &'static str,
@@ -2076,53 +2439,64 @@ pub(crate) struct MissingConstType {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_enum_struct_mutually_exclusive)]
+#[diag("`enum` and `struct` are mutually exclusive")]
 pub(crate) struct EnumStructMutuallyExclusive {
     #[primary_span]
-    #[suggestion(code = "enum", style = "verbose", applicability = "machine-applicable")]
+    #[suggestion(
+        "replace `enum struct` with",
+        code = "enum",
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
 pub(crate) enum UnexpectedTokenAfterStructName {
-    #[diag(parse_unexpected_token_after_struct_name_found_reserved_identifier)]
+    #[diag(
+        "expected `where`, `{\"{\"}`, `(`, or `;` after struct name, found reserved identifier `{$token}`"
+    )]
     ReservedIdentifier {
         #[primary_span]
-        #[label(parse_unexpected_token_after_struct_name)]
+        #[label("expected `where`, `{\"{\"}`, `(`, or `;` after struct name")]
         span: Span,
         token: Token,
     },
-    #[diag(parse_unexpected_token_after_struct_name_found_keyword)]
+    #[diag("expected `where`, `{\"{\"}`, `(`, or `;` after struct name, found keyword `{$token}`")]
     Keyword {
         #[primary_span]
-        #[label(parse_unexpected_token_after_struct_name)]
+        #[label("expected `where`, `{\"{\"}`, `(`, or `;` after struct name")]
         span: Span,
         token: Token,
     },
-    #[diag(parse_unexpected_token_after_struct_name_found_reserved_keyword)]
+    #[diag(
+        "expected `where`, `{\"{\"}`, `(`, or `;` after struct name, found reserved keyword `{$token}`"
+    )]
     ReservedKeyword {
         #[primary_span]
-        #[label(parse_unexpected_token_after_struct_name)]
+        #[label("expected `where`, `{\"{\"}`, `(`, or `;` after struct name")]
         span: Span,
         token: Token,
     },
-    #[diag(parse_unexpected_token_after_struct_name_found_doc_comment)]
+    #[diag(
+        "expected `where`, `{\"{\"}`, `(`, or `;` after struct name, found doc comment `{$token}`"
+    )]
     DocComment {
         #[primary_span]
-        #[label(parse_unexpected_token_after_struct_name)]
+        #[label("expected `where`, `{\"{\"}`, `(`, or `;` after struct name")]
         span: Span,
         token: Token,
     },
-    #[diag(parse_unexpected_token_after_struct_name_found_metavar)]
+    #[diag("expected `where`, `{\"{\"}`, `(`, or `;` after struct name, found metavar")]
     MetaVar {
         #[primary_span]
-        #[label(parse_unexpected_token_after_struct_name)]
+        #[label("expected `where`, `{\"{\"}`, `(`, or `;` after struct name")]
         span: Span,
     },
-    #[diag(parse_unexpected_token_after_struct_name_found_other)]
+    #[diag("expected `where`, `{\"{\"}`, `(`, or `;` after struct name, found `{$token}`")]
     Other {
         #[primary_span]
-        #[label(parse_unexpected_token_after_struct_name)]
+        #[label("expected `where`, `{\"{\"}`, `(`, or `;` after struct name")]
         span: Span,
         token: Token,
     },
@@ -2142,45 +2516,50 @@ impl UnexpectedTokenAfterStructName {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_self_in_generic_parameters)]
-#[note]
+#[diag("unexpected keyword `Self` in generic parameters")]
+#[note("you cannot use `Self` as a generic parameter because it is reserved for associated items")]
 pub(crate) struct UnexpectedSelfInGenericParameters {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_default_value_for_lifetime_in_generic_parameters)]
+#[diag("unexpected default lifetime parameter")]
 pub(crate) struct UnexpectedDefaultValueForLifetimeInGenericParameters {
     #[primary_span]
-    #[label]
+    #[label("lifetime parameters cannot have default values")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_multiple_where_clauses)]
+#[diag("cannot define duplicate `where` clauses on an item")]
 pub(crate) struct MultipleWhereClauses {
     #[primary_span]
     pub span: Span,
-    #[label]
+    #[label("previous `where` clause starts here")]
     pub previous: Span,
-    #[suggestion(style = "verbose", code = ",", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "consider joining the two `where` clauses into one",
+        style = "verbose",
+        code = ",",
+        applicability = "maybe-incorrect"
+    )]
     pub between: Span,
 }
 
 #[derive(Diagnostic)]
 pub(crate) enum UnexpectedNonterminal {
-    #[diag(parse_nonterminal_expected_item_keyword)]
+    #[diag("expected an item keyword")]
     Item(#[primary_span] Span),
-    #[diag(parse_nonterminal_expected_statement)]
+    #[diag("expected a statement")]
     Statement(#[primary_span] Span),
-    #[diag(parse_nonterminal_expected_ident)]
+    #[diag("expected ident, found `{$token}`")]
     Ident {
         #[primary_span]
         span: Span,
         token: Token,
     },
-    #[diag(parse_nonterminal_expected_lifetime)]
+    #[diag("expected a lifetime, found `{$token}`")]
     Lifetime {
         #[primary_span]
         span: Span,
@@ -2190,14 +2569,14 @@ pub(crate) enum UnexpectedNonterminal {
 
 #[derive(Diagnostic)]
 pub(crate) enum TopLevelOrPatternNotAllowed {
-    #[diag(parse_or_pattern_not_allowed_in_let_binding)]
+    #[diag("`let` bindings require top-level or-patterns in parentheses")]
     LetBinding {
         #[primary_span]
         span: Span,
         #[subdiagnostic]
         sub: Option,
     },
-    #[diag(parse_or_pattern_not_allowed_in_fn_parameters)]
+    #[diag("function parameters require top-level or-patterns in parentheses")]
     FunctionParameter {
         #[primary_span]
         span: Span,
@@ -2207,7 +2586,7 @@ pub(crate) enum TopLevelOrPatternNotAllowed {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_cannot_be_raw_ident)]
+#[diag("`{$ident}` cannot be a raw identifier")]
 pub(crate) struct CannotBeRawIdent {
     #[primary_span]
     pub span: Span,
@@ -2215,7 +2594,7 @@ pub(crate) struct CannotBeRawIdent {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_cannot_be_raw_lifetime)]
+#[diag("`{$ident}` cannot be a raw lifetime")]
 pub(crate) struct CannotBeRawLifetime {
     #[primary_span]
     pub span: Span,
@@ -2223,21 +2602,26 @@ pub(crate) struct CannotBeRawLifetime {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_keyword_lifetime)]
+#[diag("lifetimes cannot use keyword names")]
 pub(crate) struct KeywordLifetime {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_keyword_label)]
+#[diag("labels cannot use keyword names")]
 pub(crate) struct KeywordLabel {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_cr_doc_comment)]
+#[diag(
+    "bare CR not allowed in {$block ->
+        [true] block doc-comment
+        *[false] doc-comment
+    }"
+)]
 pub(crate) struct CrDocComment {
     #[primary_span]
     pub span: Span,
@@ -2245,14 +2629,14 @@ pub(crate) struct CrDocComment {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_no_digits_literal, code = E0768)]
+#[diag("no valid digits found for number", code = E0768)]
 pub(crate) struct NoDigitsLiteral {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_digit_literal)]
+#[diag("invalid digit for a base {$base} literal")]
 pub(crate) struct InvalidDigitLiteral {
     #[primary_span]
     pub span: Span,
@@ -2260,14 +2644,14 @@ pub(crate) struct InvalidDigitLiteral {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_empty_exponent_float)]
+#[diag("expected at least one digit in exponent")]
 pub(crate) struct EmptyExponentFloat {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_float_literal_unsupported_base)]
+#[diag("{$base} float literal is not supported")]
 pub(crate) struct FloatLiteralUnsupportedBase {
     #[primary_span]
     pub span: Span,
@@ -2275,11 +2659,11 @@ pub(crate) struct FloatLiteralUnsupportedBase {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unknown_prefix)]
-#[note]
+#[diag("prefix `{$prefix}` is unknown")]
+#[note("prefixed identifiers and literals are reserved since Rust 2021")]
 pub(crate) struct UnknownPrefix<'a> {
     #[primary_span]
-    #[label]
+    #[label("unknown prefix")]
     pub span: Span,
     pub prefix: &'a str,
     #[subdiagnostic]
@@ -2287,7 +2671,7 @@ pub(crate) struct UnknownPrefix<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[note(parse_macro_expands_to_adt_field)]
+#[note("macros cannot expand to {$adt_ty} fields")]
 pub(crate) struct MacroExpandsToAdtField<'a> {
     pub adt_ty: &'a str,
 }
@@ -2295,28 +2679,28 @@ pub(crate) struct MacroExpandsToAdtField<'a> {
 #[derive(Subdiagnostic)]
 pub(crate) enum UnknownPrefixSugg {
     #[suggestion(
-        parse_suggestion_br,
+        "use `br` for a raw byte string",
         code = "br",
         applicability = "maybe-incorrect",
         style = "verbose"
     )]
     UseBr(#[primary_span] Span),
     #[suggestion(
-        parse_suggestion_cr,
+        "use `cr` for a raw C-string",
         code = "cr",
         applicability = "maybe-incorrect",
         style = "verbose"
     )]
     UseCr(#[primary_span] Span),
     #[suggestion(
-        parse_suggestion_whitespace,
+        "consider inserting whitespace here",
         code = " ",
         applicability = "maybe-incorrect",
         style = "verbose"
     )]
     Whitespace(#[primary_span] Span),
     #[multipart_suggestion(
-        parse_suggestion_str,
+        "if you meant to write a string literal, use double quotes",
         applicability = "maybe-incorrect",
         style = "verbose"
     )]
@@ -2329,8 +2713,8 @@ pub(crate) enum UnknownPrefixSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_reserved_multihash)]
-#[note]
+#[diag("reserved multi-hash token is forbidden")]
+#[note("sequences of two or more # are reserved for future use since Rust 2024")]
 pub(crate) struct ReservedMultihash {
     #[primary_span]
     pub span: Span,
@@ -2338,8 +2722,8 @@ pub(crate) struct ReservedMultihash {
     pub sugg: Option,
 }
 #[derive(Diagnostic)]
-#[diag(parse_reserved_string)]
-#[note]
+#[diag("invalid string literal")]
+#[note("unprefixed guarded string literals are reserved for future use since Rust 2024")]
 pub(crate) struct ReservedString {
     #[primary_span]
     pub span: Span,
@@ -2348,7 +2732,7 @@ pub(crate) struct ReservedString {
 }
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_suggestion_whitespace,
+    "consider inserting whitespace here",
     code = " ",
     applicability = "maybe-incorrect",
     style = "verbose"
@@ -2356,7 +2740,9 @@ pub(crate) struct ReservedString {
 pub(crate) struct GuardedStringSugg(#[primary_span] pub Span);
 
 #[derive(Diagnostic)]
-#[diag(parse_too_many_hashes)]
+#[diag(
+    "too many `#` symbols: raw strings may be delimited by up to 255 `#` symbols, but found {$num}"
+)]
 pub(crate) struct TooManyHashes {
     #[primary_span]
     pub span: Span,
@@ -2364,7 +2750,7 @@ pub(crate) struct TooManyHashes {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unknown_start_of_token)]
+#[diag("unknown start of token: {$escaped}")]
 pub(crate) struct UnknownTokenStart {
     #[primary_span]
     pub span: Span,
@@ -2382,7 +2768,7 @@ pub(crate) struct UnknownTokenStart {
 #[derive(Subdiagnostic)]
 pub(crate) enum TokenSubstitution {
     #[suggestion(
-        parse_sugg_quotes,
+        "Unicode characters '“' (Left Double Quotation Mark) and '”' (Right Double Quotation Mark) look like '{$ascii_str}' ({$ascii_name}), but are not",
         code = "{suggestion}",
         applicability = "maybe-incorrect",
         style = "verbose"
@@ -2395,7 +2781,7 @@ pub(crate) enum TokenSubstitution {
         ascii_name: &'static str,
     },
     #[suggestion(
-        parse_sugg_other,
+        "Unicode character '{$ch}' ({$u_name}) looks like '{$ascii_str}' ({$ascii_name}), but it is not",
         code = "{suggestion}",
         applicability = "maybe-incorrect",
         style = "verbose"
@@ -2412,35 +2798,52 @@ pub(crate) enum TokenSubstitution {
 }
 
 #[derive(Subdiagnostic)]
-#[note(parse_note_repeats)]
+#[note(
+    "character appears {$repeats ->
+        [one] once more
+        *[other] {$repeats} more times
+    }"
+)]
 pub(crate) struct UnknownTokenRepeat {
     pub repeats: usize,
 }
 
 #[derive(Subdiagnostic)]
-#[help(parse_help_invisible_char)]
+#[help("invisible characters like '{$escaped}' are not usually visible in text editors")]
 pub(crate) struct InvisibleCharacter;
 
 #[derive(Subdiagnostic)]
-#[help(parse_help_null)]
+#[help(
+    "source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used"
+)]
 pub(crate) struct UnknownTokenNull;
 
 #[derive(Diagnostic)]
 pub(crate) enum UnescapeError {
-    #[diag(parse_invalid_unicode_escape)]
-    #[help]
+    #[diag("invalid unicode character escape")]
+    #[help(
+        "unicode escape must {$surrogate ->
+            [true] not be a surrogate
+            *[false] be at most 10FFFF
+        }"
+    )]
     InvalidUnicodeEscape {
         #[primary_span]
-        #[label]
+        #[label("invalid escape")]
         span: Span,
         surrogate: bool,
     },
-    #[diag(parse_escape_only_char)]
+    #[diag(
+        "{$byte ->
+            [true] byte
+            *[false] character
+        } constant must be escaped: `{$escaped_msg}`"
+    )]
     EscapeOnlyChar {
         #[primary_span]
         span: Span,
         #[suggestion(
-            parse_escape,
+            "escape the character",
             applicability = "machine-applicable",
             code = "{escaped_sugg}",
             style = "verbose"
@@ -2450,11 +2853,16 @@ pub(crate) enum UnescapeError {
         escaped_msg: String,
         byte: bool,
     },
-    #[diag(parse_bare_cr)]
+    #[diag(
+        r#"{$double_quotes ->
+            [true] bare CR not allowed in string, use `\r` instead
+            *[false] character constant must be escaped: `\r`
+        }"#
+    )]
     BareCr {
         #[primary_span]
         #[suggestion(
-            parse_escape,
+            "escape the character",
             applicability = "machine-applicable",
             code = "\\r",
             style = "verbose"
@@ -2462,93 +2870,103 @@ pub(crate) enum UnescapeError {
         span: Span,
         double_quotes: bool,
     },
-    #[diag(parse_bare_cr_in_raw_string)]
+    #[diag("bare CR not allowed in raw string")]
     BareCrRawString(#[primary_span] Span),
-    #[diag(parse_too_short_hex_escape)]
+    #[diag("numeric character escape is too short")]
     TooShortHexEscape(#[primary_span] Span),
-    #[diag(parse_invalid_char_in_escape)]
+    #[diag(
+        "invalid character in {$is_hex ->
+            [true] numeric character
+            *[false] unicode
+        } escape: `{$ch}`"
+    )]
     InvalidCharInEscape {
         #[primary_span]
-        #[label]
+        #[label(
+            "invalid character in {$is_hex ->
+                [true] numeric character
+                *[false] unicode
+            } escape"
+        )]
         span: Span,
         is_hex: bool,
         ch: String,
     },
-    #[diag(parse_leading_underscore_unicode_escape)]
+    #[diag("invalid start of unicode escape: `_`")]
     LeadingUnderscoreUnicodeEscape {
         #[primary_span]
-        #[label(parse_leading_underscore_unicode_escape_label)]
+        #[label("invalid start of unicode escape")]
         span: Span,
         ch: String,
     },
-    #[diag(parse_overlong_unicode_escape)]
+    #[diag("overlong unicode escape")]
     OverlongUnicodeEscape(
         #[primary_span]
-        #[label]
+        #[label("must have at most 6 hex digits")]
         Span,
     ),
-    #[diag(parse_unclosed_unicode_escape)]
+    #[diag("unterminated unicode escape")]
     UnclosedUnicodeEscape(
         #[primary_span]
-        #[label]
+        #[label(r#"missing a closing `{"}"}`"#)]
         Span,
         #[suggestion(
-            parse_terminate,
+            "terminate the unicode escape",
             code = "}}",
             applicability = "maybe-incorrect",
             style = "verbose"
         )]
         Span,
     ),
-    #[diag(parse_no_brace_unicode_escape)]
+    #[diag("incorrect unicode escape sequence")]
     NoBraceInUnicodeEscape {
         #[primary_span]
         span: Span,
-        #[label]
+        #[label("incorrect unicode escape sequence")]
         label: Option,
         #[subdiagnostic]
         sub: NoBraceUnicodeSub,
     },
-    #[diag(parse_unicode_escape_in_byte)]
-    #[help]
+    #[diag("unicode escape in byte string")]
+    #[help("unicode escape sequences cannot be used as a byte or in a byte string")]
     UnicodeEscapeInByte(
         #[primary_span]
-        #[label]
+        #[label("unicode escape in byte string")]
         Span,
     ),
-    #[diag(parse_empty_unicode_escape)]
+    #[diag("empty unicode escape")]
     EmptyUnicodeEscape(
         #[primary_span]
-        #[label]
+        #[label("this escape must have at least 1 hex digit")]
         Span,
     ),
-    #[diag(parse_zero_chars)]
+    #[diag("empty character literal")]
     ZeroChars(
         #[primary_span]
-        #[label]
+        #[label("empty character literal")]
         Span,
     ),
-    #[diag(parse_lone_slash)]
+    #[diag("invalid trailing slash in literal")]
     LoneSlash(
         #[primary_span]
-        #[label]
+        #[label("invalid trailing slash in literal")]
         Span,
     ),
-    #[diag(parse_unskipped_whitespace)]
+    #[diag("whitespace symbol '{$ch}' is not skipped")]
     UnskippedWhitespace {
         #[primary_span]
         span: Span,
-        #[label]
+        #[label("whitespace symbol '{$ch}' is not skipped")]
         char_span: Span,
         ch: String,
     },
-    #[diag(parse_multiple_skipped_lines)]
+    #[diag("multiple lines skipped by escaped newline")]
     MultipleSkippedLinesWarning(
         #[primary_span]
-        #[label]
+        #[label("skipping everything up to and including this point")]
         Span,
     ),
-    #[diag(parse_more_than_one_char)]
+    #[diag("character literal may only contain one codepoint")]
     MoreThanOneChar {
         #[primary_span]
         span: Span,
@@ -2557,7 +2975,7 @@ pub(crate) enum UnescapeError {
         #[subdiagnostic]
         suggestion: MoreThanOneCharSugg,
     },
-    #[diag(parse_nul_in_c_str)]
+    #[diag("null characters in C string literals are not supported")]
     NulInCStr {
         #[primary_span]
         span: Span,
@@ -2567,7 +2985,7 @@ pub(crate) enum UnescapeError {
 #[derive(Subdiagnostic)]
 pub(crate) enum MoreThanOneCharSugg {
     #[suggestion(
-        parse_consider_normalized,
+        "consider using the normalized form `{$ch}` of this character",
         code = "{normalized}",
         applicability = "machine-applicable",
         style = "verbose"
@@ -2579,7 +2997,7 @@ pub(crate) enum MoreThanOneCharSugg {
         normalized: String,
     },
     #[suggestion(
-        parse_remove_non,
+        "consider removing the non-printing characters",
         code = "{ch}",
         applicability = "maybe-incorrect",
         style = "verbose"
@@ -2590,7 +3008,10 @@ pub(crate) enum MoreThanOneCharSugg {
         ch: String,
     },
     #[suggestion(
-        parse_use_double_quotes,
+        "if you meant to write a {$is_byte ->
+            [true] byte string
+            *[false] string
+        } literal, use double quotes",
         code = "{sugg}",
         applicability = "machine-applicable",
         style = "verbose"
@@ -2601,7 +3022,13 @@ pub(crate) enum MoreThanOneCharSugg {
         is_byte: bool,
         sugg: String,
     },
-    #[multipart_suggestion(parse_use_double_quotes, applicability = "machine-applicable")]
+    #[multipart_suggestion(
+        "if you meant to write a {$is_byte ->
+            [true] byte string
+            *[false] string
+        } literal, use double quotes",
+        applicability = "machine-applicable"
+    )]
     Quotes {
         #[suggestion_part(code = "{prefix}\"")]
         start: Span,
@@ -2614,7 +3041,12 @@ pub(crate) enum MoreThanOneCharSugg {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum MoreThanOneCharNote {
-    #[note(parse_followed_by)]
+    #[note(
+        "this `{$chr}` is followed by the combining {$len ->
+            [one] mark
+            *[other] marks
+        } `{$escaped_marks}`"
+    )]
     AllCombining {
         #[primary_span]
         span: Span,
@@ -2622,7 +3054,7 @@ pub(crate) enum MoreThanOneCharNote {
         len: usize,
         escaped_marks: String,
     },
-    #[note(parse_non_printing)]
+    #[note("there are non-printing characters, the full sequence is `{$escaped}`")]
     NonPrinting {
         #[primary_span]
         span: Span,
@@ -2633,7 +3065,7 @@ pub(crate) enum MoreThanOneCharNote {
 #[derive(Subdiagnostic)]
 pub(crate) enum NoBraceUnicodeSub {
     #[suggestion(
-        parse_use_braces,
+        "format of unicode escape sequences uses braces",
         code = "{suggestion}",
         applicability = "maybe-incorrect",
         style = "verbose"
@@ -2643,12 +3075,12 @@ pub(crate) enum NoBraceUnicodeSub {
         span: Span,
         suggestion: String,
     },
-    #[help(parse_format_of_unicode)]
+    #[help(r#"format of unicode escape sequences is `\u{"{...}"}`"#)]
     Help,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_sugg_wrap_pattern_in_parens, applicability = "machine-applicable")]
+#[multipart_suggestion("wrap the pattern in parentheses", applicability = "machine-applicable")]
 pub(crate) struct WrapInParens {
     #[suggestion_part(code = "(")]
     pub(crate) lo: Span,
@@ -2659,7 +3091,7 @@ pub(crate) struct WrapInParens {
 #[derive(Subdiagnostic)]
 pub(crate) enum TopLevelOrPatternNotAllowedSugg {
     #[suggestion(
-        parse_sugg_remove_leading_vert_in_pattern,
+        "remove the `|`",
         code = "",
         applicability = "machine-applicable",
         style = "tool-only"
@@ -2677,27 +3109,37 @@ pub(crate) enum TopLevelOrPatternNotAllowedSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_vert_vert_before_function_parameter)]
-#[note(parse_note_pattern_alternatives_use_single_vert)]
+#[diag("unexpected `||` before function parameter")]
+#[note("alternatives in or-patterns are separated with `|`, not `||`")]
 pub(crate) struct UnexpectedVertVertBeforeFunctionParam {
     #[primary_span]
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove the `||`",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_vert_vert_in_pattern)]
+#[diag("unexpected token `||` in pattern")]
 pub(crate) struct UnexpectedVertVertInPattern {
     #[primary_span]
-    #[suggestion(code = "|", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "use a single `|` to separate multiple alternative patterns",
+        code = "|",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
-    #[label(parse_label_while_parsing_or_pattern_here)]
+    #[label("while parsing this or-pattern starting here")]
     pub start: Option,
 }
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_trailing_vert_not_allowed,
+    "a trailing `{$token}` is not allowed in an or-pattern",
     code = "",
     applicability = "machine-applicable",
     style = "tool-only"
@@ -2708,59 +3150,71 @@ pub(crate) struct TrailingVertSuggestion {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_trailing_vert_not_allowed)]
+#[diag("a trailing `{$token}` is not allowed in an or-pattern")]
 pub(crate) struct TrailingVertNotAllowed {
     #[primary_span]
     pub span: Span,
     #[subdiagnostic]
     pub suggestion: TrailingVertSuggestion,
-    #[label(parse_label_while_parsing_or_pattern_here)]
+    #[label("while parsing this or-pattern starting here")]
     pub start: Option,
     pub token: Token,
-    #[note(parse_note_pattern_alternatives_use_single_vert)]
+    #[note("alternatives in or-patterns are separated with `|`, not `||`")]
     pub note_double_vert: bool,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dotdotdot_rest_pattern)]
+#[diag("unexpected `...`")]
 pub(crate) struct DotDotDotRestPattern {
     #[primary_span]
-    #[label]
+    #[label("not a valid pattern")]
     pub span: Span,
-    #[suggestion(style = "verbose", code = "", applicability = "machine-applicable")]
+    #[suggestion(
+        "for a rest pattern, use `..` instead of `...`",
+        style = "verbose",
+        code = "",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Option,
-    #[note]
+    #[note(
+        "only `extern \"C\"` and `extern \"C-unwind\"` functions may have a C variable argument list"
+    )]
     pub var_args: Option<()>,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_pattern_on_wrong_side_of_at)]
+#[diag("pattern on wrong side of `@`")]
 pub(crate) struct PatternOnWrongSideOfAt {
     #[primary_span]
-    #[suggestion(code = "{whole_pat}", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "switch the order",
+        code = "{whole_pat}",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub whole_span: Span,
     pub whole_pat: String,
-    #[label(parse_label_pattern)]
+    #[label("pattern on the left, should be on the right")]
     pub pattern: Span,
-    #[label(parse_label_binding)]
+    #[label("binding on the right, should be on the left")]
     pub binding: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_binding_left_of_at)]
-#[note]
+#[diag("left-hand side of `@` must be a binding")]
+#[note("bindings are `x`, `mut x`, `ref x`, and `ref mut x`")]
 pub(crate) struct ExpectedBindingLeftOfAt {
     #[primary_span]
     pub whole_span: Span,
-    #[label(parse_label_lhs)]
+    #[label("interpreted as a pattern, not a binding")]
     pub lhs: Span,
-    #[label(parse_label_rhs)]
+    #[label("also a pattern")]
     pub rhs: Span,
 }
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    parse_ambiguous_range_pattern_suggestion,
+    "add parentheses to clarify the precedence",
     applicability = "machine-applicable"
 )]
 pub(crate) struct ParenRangeSuggestion {
@@ -2771,7 +3225,7 @@ pub(crate) struct ParenRangeSuggestion {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_ambiguous_range_pattern)]
+#[diag("the range pattern here has ambiguous interpretation")]
 pub(crate) struct AmbiguousRangePattern {
     #[primary_span]
     pub span: Span,
@@ -2780,100 +3234,144 @@ pub(crate) struct AmbiguousRangePattern {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_lifetime_in_pattern)]
+#[diag("unexpected lifetime `{$symbol}` in pattern")]
 pub(crate) struct UnexpectedLifetimeInPattern {
     #[primary_span]
     pub span: Span,
     pub symbol: Symbol,
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove the lifetime",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
 pub(crate) enum InvalidMutInPattern {
-    #[diag(parse_mut_on_nested_ident_pattern)]
-    #[note(parse_note_mut_pattern_usage)]
+    #[diag("`mut` must be attached to each individual binding")]
+    #[note("`mut` may be followed by `variable` and `variable @ pattern`")]
     NestedIdent {
         #[primary_span]
-        #[suggestion(code = "{pat}", applicability = "machine-applicable", style = "verbose")]
+        #[suggestion(
+            "add `mut` to each binding",
+            code = "{pat}",
+            applicability = "machine-applicable",
+            style = "verbose"
+        )]
         span: Span,
         pat: String,
     },
-    #[diag(parse_mut_on_non_ident_pattern)]
-    #[note(parse_note_mut_pattern_usage)]
+    #[diag("`mut` must be followed by a named binding")]
+    #[note("`mut` may be followed by `variable` and `variable @ pattern`")]
     NonIdent {
         #[primary_span]
-        #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+        #[suggestion(
+            "remove the `mut` prefix",
+            code = "",
+            applicability = "machine-applicable",
+            style = "verbose"
+        )]
         span: Span,
     },
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_repeated_mut_in_pattern)]
+#[diag("`mut` on a binding may not be repeated")]
 pub(crate) struct RepeatedMutInPattern {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove the additional `mut`s",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dot_dot_dot_range_to_pattern_not_allowed)]
+#[diag("range-to patterns with `...` are not allowed")]
 pub(crate) struct DotDotDotRangeToPatternNotAllowed {
     #[primary_span]
-    #[suggestion(style = "verbose", code = "..=", applicability = "machine-applicable")]
+    #[suggestion(
+        "use `..=` instead",
+        style = "verbose",
+        code = "..=",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_enum_pattern_instead_of_identifier)]
+#[diag("expected identifier, found enum pattern")]
 pub(crate) struct EnumPatternInsteadOfIdentifier {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_at_dot_dot_in_struct_pattern)]
+#[diag("`@ ..` is not supported in struct patterns")]
 pub(crate) struct AtDotDotInStructPattern {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "", style = "verbose", applicability = "machine-applicable")]
+    #[suggestion(
+        "bind to each field separately or, if you don't need them, just remove `{$ident} @`",
+        code = "",
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
     pub remove: Span,
     pub ident: Ident,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_at_in_struct_pattern)]
-#[note]
-#[help]
+#[diag("unexpected `@` in struct pattern")]
+#[note("struct patterns use `field: pattern` syntax to bind to fields")]
+#[help(
+    "consider replacing `new_name @ field_name` with `field_name: new_name` if that is what you intended"
+)]
 pub(crate) struct AtInStructPattern {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dot_dot_dot_for_remaining_fields)]
+#[diag("expected field pattern, found `{$token_str}`")]
 pub(crate) struct DotDotDotForRemainingFields {
     #[primary_span]
-    #[suggestion(code = "..", style = "verbose", applicability = "machine-applicable")]
+    #[suggestion(
+        "to omit remaining fields, use `..`",
+        code = "..",
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
     pub token_str: Cow<'static, str>,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_comma_after_pattern_field)]
+#[diag("expected `,`")]
 pub(crate) struct ExpectedCommaAfterPatternField {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_expr_in_pat)]
-#[note]
+#[diag(
+    "expected {$is_bound ->
+        [true] a pattern range bound
+        *[false] a pattern
+    }, found an expression"
+)]
+#[note(
+    "arbitrary expressions are not allowed in patterns: "
+)]
 pub(crate) struct UnexpectedExpressionInPattern {
     /// The unexpected expr's span.
     #[primary_span]
-    #[label]
+    #[label("not a pattern")]
     pub span: Span,
     /// Was a `RangePatternBound` expected?
     pub is_bound: bool,
@@ -2884,7 +3382,7 @@ pub(crate) struct UnexpectedExpressionInPattern {
 #[derive(Subdiagnostic)]
 pub(crate) enum UnexpectedExpressionInPatternSugg {
     #[multipart_suggestion(
-        parse_unexpected_expr_in_pat_create_guard_sugg,
+        "consider moving the expression to a match arm guard",
         applicability = "maybe-incorrect"
     )]
     CreateGuard {
@@ -2901,7 +3399,7 @@ pub(crate) enum UnexpectedExpressionInPatternSugg {
     },
 
     #[multipart_suggestion(
-        parse_unexpected_expr_in_pat_update_guard_sugg,
+        "consider moving the expression to the match arm guard",
         applicability = "maybe-incorrect"
     )]
     UpdateGuard {
@@ -2923,7 +3421,7 @@ pub(crate) enum UnexpectedExpressionInPatternSugg {
     },
 
     #[multipart_suggestion(
-        parse_unexpected_expr_in_pat_const_sugg,
+        "consider extracting the expression into a `const`",
         applicability = "has-placeholders"
     )]
     Const {
@@ -2943,7 +3441,7 @@ pub(crate) enum UnexpectedExpressionInPatternSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unexpected_paren_in_range_pat)]
+#[diag("range pattern bounds cannot have parentheses")]
 pub(crate) struct UnexpectedParenInRangePat {
     #[primary_span]
     pub span: Vec,
@@ -2952,10 +3450,7 @@ pub(crate) struct UnexpectedParenInRangePat {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(
-    parse_unexpected_paren_in_range_pat_sugg,
-    applicability = "machine-applicable"
-)]
+#[multipart_suggestion("remove these parentheses", applicability = "machine-applicable")]
 pub(crate) struct UnexpectedParenInRangePatSugg {
     #[suggestion_part(code = "")]
     pub start_span: Span,
@@ -2964,104 +3459,145 @@ pub(crate) struct UnexpectedParenInRangePatSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_return_types_use_thin_arrow)]
+#[diag("return types are denoted using `->`")]
 pub(crate) struct ReturnTypesUseThinArrow {
     #[primary_span]
     pub span: Span,
-    #[suggestion(style = "verbose", code = " -> ", applicability = "machine-applicable")]
+    #[suggestion(
+        "use `->` instead",
+        style = "verbose",
+        code = " -> ",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_need_plus_after_trait_object_lifetime)]
+#[diag("lifetimes must be followed by `+` to form a trait object type")]
 pub(crate) struct NeedPlusAfterTraitObjectLifetime {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = " + /* Trait */", applicability = "has-placeholders")]
+    #[suggestion(
+        "consider adding a trait bound after the potential lifetime bound",
+        code = " + /* Trait */",
+        applicability = "has-placeholders"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_mut_or_const_in_raw_pointer_type)]
+#[diag("expected `mut` or `const` keyword in raw pointer type")]
 pub(crate) struct ExpectedMutOrConstInRawPointerType {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code("mut ", "const "), applicability = "has-placeholders", style = "verbose")]
+    #[suggestion(
+        "add `mut` or `const` here",
+        code("mut ", "const "),
+        applicability = "has-placeholders",
+        style = "verbose"
+    )]
     pub after_asterisk: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_lifetime_after_mut)]
+#[diag("lifetime must precede `mut`")]
 pub(crate) struct LifetimeAfterMut {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "&{snippet} mut", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "place the lifetime before `mut`",
+        code = "&{snippet} mut",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub suggest_lifetime: Option,
     pub snippet: String,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dyn_after_mut)]
+#[diag("`mut` must precede `dyn`")]
 pub(crate) struct DynAfterMut {
     #[primary_span]
-    #[suggestion(code = "&mut dyn", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "place `mut` before `dyn`",
+        code = "&mut dyn",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_fn_pointer_cannot_be_const)]
-#[note]
+#[diag("an `fn` pointer type cannot be `const`")]
+#[note("allowed qualifiers are: `unsafe` and `extern`")]
 pub(crate) struct FnPointerCannotBeConst {
     #[primary_span]
-    #[label]
+    #[label("`const` because of this")]
     pub span: Span,
-    #[suggestion(code = "", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "remove the `const` qualifier",
+        code = "",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_fn_pointer_cannot_be_async)]
-#[note]
+#[diag("an `fn` pointer type cannot be `async`")]
+#[note("allowed qualifiers are: `unsafe` and `extern`")]
 pub(crate) struct FnPointerCannotBeAsync {
     #[primary_span]
-    #[label]
+    #[label("`async` because of this")]
     pub span: Span,
-    #[suggestion(code = "", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "remove the `async` qualifier",
+        code = "",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_nested_c_variadic_type, code = E0743)]
+#[diag("C-variadic type `...` may not be nested inside another type", code = E0743)]
 pub(crate) struct NestedCVariadicType {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dotdotdot_rest_type)]
-#[note]
+#[diag("unexpected `...`")]
+#[note(
+    "only `extern \"C\"` and `extern \"C-unwind\"` functions may have a C variable argument list"
+)]
 pub(crate) struct InvalidCVariadicType {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_dyn_keyword)]
-#[help]
+#[diag("invalid `dyn` keyword")]
+#[help("`dyn` is only needed at the start of a trait `+`-separated list")]
 pub(crate) struct InvalidDynKeyword {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove this keyword",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum HelpUseLatestEdition {
-    #[help(parse_help_set_edition_cargo)]
-    #[note(parse_note_edition_guide)]
+    #[help("set `edition = \"{$edition}\"` in `Cargo.toml`")]
+    #[note("for more on editions, read https://doc.rust-lang.org/edition-guide")]
     Cargo { edition: Edition },
-    #[help(parse_help_set_edition_standalone)]
-    #[note(parse_note_edition_guide)]
+    #[help("pass `--edition {$edition}` to `rustc`")]
+    #[note("for more on editions, read https://doc.rust-lang.org/edition-guide")]
     Standalone { edition: Edition },
 }
 
@@ -3077,7 +3613,7 @@ impl HelpUseLatestEdition {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_box_syntax_removed)]
+#[diag("`box_syntax` has been removed")]
 pub(crate) struct BoxSyntaxRemoved {
     #[primary_span]
     pub span: Span,
@@ -3087,7 +3623,7 @@ pub(crate) struct BoxSyntaxRemoved {
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    parse_box_syntax_removed_suggestion,
+    "use `Box::new()` instead",
     applicability = "machine-applicable",
     style = "verbose"
 )]
@@ -3099,65 +3635,70 @@ pub(crate) struct AddBoxNew {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_bad_return_type_notation_output)]
+#[diag("return type not allowed with return type notation")]
 pub(crate) struct BadReturnTypeNotationOutput {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "remove the return type",
+        code = "",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_bad_assoc_type_bounds)]
+#[diag("bounds on associated types do not belong here")]
 pub(crate) struct BadAssocTypeBounds {
     #[primary_span]
-    #[label]
+    #[label("belongs in `where` clause")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_attr_after_generic)]
+#[diag("trailing attribute after generic parameter")]
 pub(crate) struct AttrAfterGeneric {
     #[primary_span]
-    #[label]
+    #[label("attributes must go before parameters")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_attr_without_generics)]
+#[diag("attribute without generic parameters")]
 pub(crate) struct AttrWithoutGenerics {
     #[primary_span]
-    #[label]
+    #[label("attributes are only permitted when preceding parameters")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_where_generics)]
+#[diag("generic parameters on `where` clauses are reserved for future use")]
 pub(crate) struct WhereOnGenerics {
     #[primary_span]
-    #[label]
+    #[label("currently unsupported")]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_generics_in_path)]
+#[diag("unexpected generic arguments in path")]
 pub(crate) struct GenericsInPath {
     #[primary_span]
     pub span: Vec,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_lifetime_in_eq_constraint)]
-#[help]
+#[diag("lifetimes are not permitted in this context")]
+#[help("if you meant to specify a trait object, write `dyn /* Trait */ + {$lifetime}`")]
 pub(crate) struct LifetimeInEqConstraint {
     #[primary_span]
-    #[label]
+    #[label("lifetime is not allowed here")]
     pub span: Span,
     pub lifetime: Ident,
-    #[label(parse_context_label)]
+    #[label("this introduces an associated item binding")]
     pub binding_label: Span,
     #[suggestion(
-        parse_colon_sugg,
+        "you might have meant to write a bound here",
         style = "verbose",
         applicability = "maybe-incorrect",
         code = ": "
@@ -3166,119 +3707,159 @@ pub(crate) struct LifetimeInEqConstraint {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_modifier_lifetime)]
+#[diag("`{$modifier}` may only modify trait bounds, not lifetime bounds")]
 pub(crate) struct ModifierLifetime {
     #[primary_span]
-    #[suggestion(style = "tool-only", applicability = "maybe-incorrect", code = "")]
+    #[suggestion(
+        "remove the `{$modifier}`",
+        style = "tool-only",
+        applicability = "maybe-incorrect",
+        code = ""
+    )]
     pub span: Span,
     pub modifier: &'static str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_underscore_literal_suffix)]
+#[diag("underscore literal suffix is not allowed")]
 pub(crate) struct UnderscoreLiteralSuffix {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expect_label_found_ident)]
+#[diag("expected a label, found an identifier")]
 pub(crate) struct ExpectedLabelFoundIdent {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "'", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "labels start with a tick",
+        code = "'",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub start: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_inappropriate_default)]
-#[note]
+#[diag("{$article} {$descr} cannot be `default`")]
+#[note("only associated `fn`, `const`, and `type` items can be `default`")]
 pub(crate) struct InappropriateDefault {
     #[primary_span]
-    #[label]
+    #[label("`default` because of this")]
     pub span: Span,
     pub article: &'static str,
     pub descr: &'static str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_recover_import_as_use)]
+#[diag("expected item, found {$token_name}")]
 pub(crate) struct RecoverImportAsUse {
     #[primary_span]
-    #[suggestion(code = "use", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "items are imported using the `use` keyword",
+        code = "use",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
     pub token_name: String,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_single_colon_import_path)]
-#[note]
+#[diag("expected `::`, found `:`")]
+#[note("import paths are delimited using `::`")]
 pub(crate) struct SingleColonImportPath {
     #[primary_span]
-    #[suggestion(code = "::", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "use double colon",
+        code = "::",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_bad_item_kind)]
+#[diag("{$descr} is not supported in {$ctx}")]
 pub(crate) struct BadItemKind {
     #[primary_span]
     pub span: Span,
     pub descr: &'static str,
     pub ctx: &'static str,
-    #[help]
+    #[help("consider moving the {$descr} out to a nearby module scope")]
     pub help: bool,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_macro_rules_missing_bang)]
+#[diag("expected `!` after `macro_rules`")]
 pub(crate) struct MacroRulesMissingBang {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "!", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion("add a `!`", code = "!", applicability = "machine-applicable", style = "verbose")]
     pub hi: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_macro_name_remove_bang)]
+#[diag("macro names aren't followed by a `!`")]
 pub(crate) struct MacroNameRemoveBang {
     #[primary_span]
-    #[suggestion(code = "", applicability = "machine-applicable", style = "short")]
+    #[suggestion(
+        "remove the `!`",
+        code = "",
+        applicability = "machine-applicable",
+        style = "short"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_macro_rules_visibility)]
+#[diag("can't qualify macro_rules invocation with `{$vis}`")]
 pub(crate) struct MacroRulesVisibility<'a> {
     #[primary_span]
-    #[suggestion(code = "#[macro_export]", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "try exporting the macro",
+        code = "#[macro_export]",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub span: Span,
     pub vis: &'a str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_macro_invocation_visibility)]
-#[help]
+#[diag("can't qualify macro invocation with `pub`")]
+#[help("try adjusting the macro to put `{$vis}` inside the invocation")]
 pub(crate) struct MacroInvocationVisibility<'a> {
     #[primary_span]
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove the visibility",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
     pub vis: &'a str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_nested_adt)]
+#[diag("`{$kw_str}` definition cannot be nested inside `{$keyword}`")]
 pub(crate) struct NestedAdt<'a> {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = "", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "consider creating a new `{$kw_str}` definition instead of nesting",
+        code = "",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub item: Span,
     pub keyword: &'a str,
     pub kw_str: Cow<'a, str>,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_function_body_equals_expr)]
+#[diag("function body cannot be `= expression;`")]
 pub(crate) struct FunctionBodyEqualsExpr {
     #[primary_span]
     pub span: Span,
@@ -3287,7 +3868,10 @@ pub(crate) struct FunctionBodyEqualsExpr {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    r#"surround the expression with `{"{"}` and `{"}"}` instead of `=` and `;`"#,
+    applicability = "machine-applicable"
+)]
 pub(crate) struct FunctionBodyEqualsExprSugg {
     #[suggestion_part(code = "{{")]
     pub eq: Span,
@@ -3296,38 +3880,56 @@ pub(crate) struct FunctionBodyEqualsExprSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_box_not_pat)]
+#[diag("expected pattern, found {$descr}")]
 pub(crate) struct BoxNotPat {
     #[primary_span]
     pub span: Span,
-    #[note]
+    #[note("`box` is a reserved keyword")]
     pub kw: Span,
-    #[suggestion(code = "r#", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion(
+        "escape `box` to use it as an identifier",
+        code = "r#",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
     pub lo: Span,
     pub descr: String,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unmatched_angle)]
+#[diag(
+    "unmatched angle {$plural ->
+        [true] brackets
+        *[false] bracket
+    }"
+)]
 pub(crate) struct UnmatchedAngle {
     #[primary_span]
-    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
+    #[suggestion(
+        "remove extra angle {$plural ->
+            [true] brackets
+            *[false] bracket
+        }",
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
     pub span: Span,
     pub plural: bool,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_missing_plus_in_bounds)]
+#[diag("expected `+` between lifetime and {$sym}")]
 pub(crate) struct MissingPlusBounds {
     #[primary_span]
     pub span: Span,
-    #[suggestion(code = " +", applicability = "maybe-incorrect", style = "verbose")]
+    #[suggestion("add `+`", code = " +", applicability = "maybe-incorrect", style = "verbose")]
     pub hi: Span,
     pub sym: Symbol,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_parens_trait_bounds)]
+#[diag("incorrect parentheses around trait bounds")]
 pub(crate) struct IncorrectParensTraitBounds {
     #[primary_span]
     pub span: Vec,
@@ -3336,10 +3938,7 @@ pub(crate) struct IncorrectParensTraitBounds {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(
-    parse_incorrect_parens_trait_bounds_sugg,
-    applicability = "machine-applicable"
-)]
+#[multipart_suggestion("fix the parentheses", applicability = "machine-applicable")]
 pub(crate) struct IncorrectParensTraitBoundsSugg {
     #[suggestion_part(code = " ")]
     pub wrong_span: Span,
@@ -3348,10 +3947,15 @@ pub(crate) struct IncorrectParensTraitBoundsSugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_kw_bad_case)]
+#[diag("keyword `{$kw}` is written in the wrong case")]
 pub(crate) struct KwBadCase<'a> {
     #[primary_span]
-    #[suggestion(code = "{kw}", style = "verbose", applicability = "machine-applicable")]
+    #[suggestion(
+        "write it in {$case}",
+        code = "{kw}",
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
     pub kw: &'a str,
     pub case: Case,
@@ -3375,7 +3979,7 @@ impl IntoDiagArg for Case {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_unknown_builtin_construct)]
+#[diag("unknown `builtin #` construct `{$name}`")]
 pub(crate) struct UnknownBuiltinConstruct {
     #[primary_span]
     pub span: Span,
@@ -3383,35 +3987,38 @@ pub(crate) struct UnknownBuiltinConstruct {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expected_builtin_ident)]
+#[diag("expected identifier after `builtin #`")]
 pub(crate) struct ExpectedBuiltinIdent {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_static_with_generics)]
+#[diag("static items may not have generic parameters")]
 pub(crate) struct StaticWithGenerics {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_where_clause_before_const_body)]
+#[diag("where clauses are not allowed before const item bodies")]
 pub(crate) struct WhereClauseBeforeConstBody {
     #[primary_span]
-    #[label]
+    #[label("unexpected where clause")]
     pub span: Span,
-    #[label(parse_name_label)]
+    #[label("while parsing this const item")]
     pub name: Span,
-    #[label(parse_body_label)]
+    #[label("the item body")]
     pub body: Span,
     #[subdiagnostic]
     pub sugg: Option,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "move the body before the where clause",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct WhereClauseBeforeConstBodySugg {
     #[suggestion_part(code = "= {snippet} ")]
     pub left: Span,
@@ -3421,12 +4028,12 @@ pub(crate) struct WhereClauseBeforeConstBodySugg {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_generic_args_in_pat_require_turbofish_syntax)]
+#[diag("generic args in patterns require the turbofish syntax")]
 pub(crate) struct GenericArgsInPatRequireTurbofishSyntax {
     #[primary_span]
     pub span: Span,
     #[suggestion(
-        parse_sugg_turbofish_syntax,
+        "use `::<...>` instead of `<...>` to specify lifetime, type, or const arguments",
         style = "verbose",
         code = "::",
         applicability = "maybe-incorrect"
@@ -3435,7 +4042,7 @@ pub(crate) struct GenericArgsInPatRequireTurbofishSyntax {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_transpose_dyn_or_impl)]
+#[diag("`for<...>` expected after `{$kw}`, not before")]
 pub(crate) struct TransposeDynOrImpl<'a> {
     #[primary_span]
     pub span: Span,
@@ -3445,7 +4052,7 @@ pub(crate) struct TransposeDynOrImpl<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("move `{$kw}` before the `for<...>`", applicability = "machine-applicable")]
 pub(crate) struct TransposeDynOrImplSugg<'a> {
     #[suggestion_part(code = "")]
     pub removal_span: Span,
@@ -3455,68 +4062,77 @@ pub(crate) struct TransposeDynOrImplSugg<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_array_index_offset_of)]
+#[diag("array indexing not supported in offset_of")]
 pub(crate) struct ArrayIndexInOffsetOf(#[primary_span] pub Span);
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_offset_of)]
+#[diag("offset_of expects dot-separated field and variant names")]
 pub(crate) struct InvalidOffsetOf(#[primary_span] pub Span);
 
 #[derive(Diagnostic)]
-#[diag(parse_async_impl)]
+#[diag("`async` trait implementations are unsupported")]
 pub(crate) struct AsyncImpl {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_expr_rarrow_call)]
-#[help]
+#[diag("`->` is not valid syntax for field accesses and method calls")]
+#[help(
+    "the `.` operator will automatically dereference the value, except if the value is a raw pointer"
+)]
 pub(crate) struct ExprRArrowCall {
     #[primary_span]
-    #[suggestion(style = "verbose", applicability = "machine-applicable", code = ".")]
+    #[suggestion(
+        "try using `.` instead",
+        style = "verbose",
+        applicability = "machine-applicable",
+        code = "."
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_dot_dot_range_attribute)]
+#[diag("attributes are not allowed on range expressions starting with `..`")]
 pub(crate) struct DotDotRangeAttribute {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_binder_before_modifiers)]
+#[diag("`for<...>` binder should be placed before trait bound modifiers")]
 pub(crate) struct BinderBeforeModifiers {
     #[primary_span]
     pub binder_span: Span,
-    #[label]
+    #[label("place the `for<...>` binder before any modifiers")]
     pub modifiers_span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_binder_and_polarity)]
+#[diag("`for<...>` binder not allowed with `{$polarity}` trait polarity modifier")]
 pub(crate) struct BinderAndPolarity {
     #[primary_span]
     pub polarity_span: Span,
-    #[label]
+    #[label("there is not a well-defined meaning for a higher-ranked `{$polarity}` trait")]
     pub binder_span: Span,
     pub polarity: &'static str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_modifiers_and_polarity)]
+#[diag("`{$modifiers_concatenated}` trait not allowed with `{$polarity}` trait polarity modifier")]
 pub(crate) struct PolarityAndModifiers {
     #[primary_span]
     pub polarity_span: Span,
-    #[label]
+    #[label(
+        "there is not a well-defined meaning for a `{$modifiers_concatenated} {$polarity}` trait"
+    )]
     pub modifiers_span: Span,
     pub polarity: &'static str,
     pub modifiers_concatenated: String,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_incorrect_type_on_self)]
+#[diag("type not allowed for shorthand `self` parameter")]
 pub(crate) struct IncorrectTypeOnSelf {
     #[primary_span]
     pub span: Span,
@@ -3525,7 +4141,10 @@ pub(crate) struct IncorrectTypeOnSelf {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "move the modifiers on `self` to the type",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct MoveSelfModifier {
     #[suggestion_part(code = "")]
     pub removal_span: Span,
@@ -3535,82 +4154,104 @@ pub(crate) struct MoveSelfModifier {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_unsupported_operand)]
+#[diag("the `{$symbol}` operand cannot be used with `{$macro_name}!`")]
 pub(crate) struct AsmUnsupportedOperand<'a> {
     #[primary_span]
-    #[label]
+    #[label(
+        "the `{$symbol}` operand is not meaningful for global-scoped inline assembly, remove it"
+    )]
     pub(crate) span: Span,
     pub(crate) symbol: &'a str,
     pub(crate) macro_name: &'static str,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_underscore_input)]
+#[diag("_ cannot be used for input operands")]
 pub(crate) struct AsmUnderscoreInput {
     #[primary_span]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_sym_no_path)]
+#[diag("expected a path for argument to `sym`")]
 pub(crate) struct AsmSymNoPath {
     #[primary_span]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_requires_template)]
+#[diag("requires at least a template string argument")]
 pub(crate) struct AsmRequiresTemplate {
     #[primary_span]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_expected_comma)]
+#[diag("expected token: `,`")]
 pub(crate) struct AsmExpectedComma {
     #[primary_span]
-    #[label]
+    #[label("expected `,`")]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_expected_other)]
+#[diag(
+    "expected operand, {$is_inline_asm ->
+        [false] options
+        *[true] clobber_abi, options
+    }, or additional template string"
+)]
 pub(crate) struct AsmExpectedOther {
     #[primary_span]
-    #[label(parse_asm_expected_other)]
+    #[label(
+        "expected operand, {$is_inline_asm ->
+            [false] options
+            *[true] clobber_abi, options
+        }, or additional template string"
+    )]
     pub(crate) span: Span,
     pub(crate) is_inline_asm: bool,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_non_abi)]
+#[diag("at least one abi must be provided as an argument to `clobber_abi`")]
 pub(crate) struct NonABI {
     #[primary_span]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_expected_string_literal)]
+#[diag("expected string literal")]
 pub(crate) struct AsmExpectedStringLiteral {
     #[primary_span]
-    #[label]
+    #[label("not a string literal")]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_asm_expected_register_class_or_explicit_register)]
+#[diag("expected register class or explicit register")]
 pub(crate) struct ExpectedRegisterClassOrExplicitRegister {
     #[primary_span]
     pub(crate) span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(parse_hidden_unicode_codepoints)]
-#[note]
+#[diag("unicode codepoint changing visible direction of text present in {$label}")]
+#[note(
+    "these kind of unicode codepoints change the way text flows on applications that support them, but can cause confusion because they change the order of characters on the screen"
+)]
 pub(crate) struct HiddenUnicodeCodepointsDiag {
     pub label: String,
     pub count: usize,
-    #[label]
+    #[label(
+        "this {$label} contains {$count ->
+            [one] an invisible
+            *[other] invisible
+        } unicode text flow control {$count ->
+            [one] codepoint
+            *[other] codepoints
+        }"
+    )]
     pub span_label: Span,
     #[subdiagnostic]
     pub labels: Option,
@@ -3641,13 +4282,13 @@ impl Subdiagnostic for HiddenUnicodeCodepointsDiagSub {
         match self {
             HiddenUnicodeCodepointsDiagSub::Escape { spans } => {
                 diag.multipart_suggestion_with_style(
-                    fluent::parse_suggestion_remove,
+                    inline_fluent!("if their presence wasn't intentional, you can remove them"),
                     spans.iter().map(|(_, span)| (*span, "".to_string())).collect(),
                     Applicability::MachineApplicable,
                     SuggestionStyle::HideCodeAlways,
                 );
                 diag.multipart_suggestion(
-                    fluent::parse_suggestion_escape,
+                    inline_fluent!("if you want to keep them but make them visible in your source code, you can escape them"),
                     spans
                         .into_iter()
                         .map(|(c, span)| {
@@ -3670,43 +4311,59 @@ impl Subdiagnostic for HiddenUnicodeCodepointsDiagSub {
                         .collect::>()
                         .join(", "),
                 );
-                diag.note(fluent::parse_suggestion_remove);
-                diag.note(fluent::parse_no_suggestion_note_escape);
+                diag.note(inline_fluent!(
+                    "if their presence wasn't intentional, you can remove them"
+                ));
+                diag.note(inline_fluent!("if you want to keep them but make them visible in your source code, you can escape them: {$escaped}"));
             }
         }
     }
 }
 
 #[derive(LintDiagnostic)]
-#[diag(parse_varargs_without_pattern)]
+#[diag("missing pattern for `...` argument")]
 pub(crate) struct VarargsWithoutPattern {
-    #[suggestion(code = "_: ...", applicability = "machine-applicable")]
+    #[suggestion(
+        "name the argument, or use `_` to continue ignoring it",
+        code = "_: ...",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_delegation_non_trait_impl_reuse)]
+#[diag("only trait impls can be reused")]
 pub(crate) struct ImplReuseInherentImpl {
     #[primary_span]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_struct_literal_placeholder_path)]
+#[diag("placeholder `_` is not allowed for the path in struct literals")]
 pub(crate) struct StructLiteralPlaceholderPath {
     #[primary_span]
-    #[label]
-    #[suggestion(applicability = "has-placeholders", code = "/* Type */", style = "verbose")]
+    #[label("not allowed in struct literals")]
+    #[suggestion(
+        "replace it with the correct type",
+        applicability = "has-placeholders",
+        code = "/* Type */",
+        style = "verbose"
+    )]
     pub span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_struct_literal_body_without_path_late)]
+#[diag("struct literal body without path")]
 pub(crate) struct StructLiteralWithoutPathLate {
     #[primary_span]
-    #[label]
+    #[label("struct name missing for struct literal")]
     pub span: Span,
-    #[suggestion(applicability = "has-placeholders", code = "/* Type */ ", style = "verbose")]
+    #[suggestion(
+        "add the correct type",
+        applicability = "has-placeholders",
+        code = "/* Type */ ",
+        style = "verbose"
+    )]
     pub suggestion_span: Span,
 }
 
@@ -3716,19 +4373,22 @@ pub(crate) enum ForbiddenLetReason {
     /// `let` is not valid and the source environment is not important
     OtherForbidden,
     /// A let chain with the `||` operator
-    #[note(parse_not_supported_or)]
+    #[note("`||` operators are not supported in let chain expressions")]
     NotSupportedOr(#[primary_span] Span),
     /// A let chain with invalid parentheses
     ///
     /// For example, `let 1 = 1 && (expr && expr)` is allowed
     /// but `(let 1 = 1 && (let 1 = 1 && (let 1 = 1))) && let a = 1` is not
-    #[note(parse_not_supported_parentheses)]
+    #[note("`let`s wrapped in parentheses are not supported in a context with let chains")]
     NotSupportedParentheses(#[primary_span] Span),
 }
 
 #[derive(Debug, rustc_macros::Subdiagnostic)]
 #[suggestion(
-    parse_misspelled_kw,
+    "{$is_incorrect_case ->
+        [true] write keyword `{$similar_kw}` in lowercase
+        *[false] there is a keyword `{$similar_kw}` with a similar name
+    }",
     applicability = "machine-applicable",
     code = "{similar_kw}",
     style = "verbose"
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index c836d52a71e4..40cdcfc35c2f 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -75,8 +75,6 @@ const _: () = {
     }
 };
 
-rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
-
 // Unwrap the result if `Ok`, otherwise emit the diagnostics and abort.
 pub fn unwrap_or_emit_fatal(expr: Result>>) -> T {
     match expr {
diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
index fe8c1deedff7..3830d4b4cd41 100644
--- a/compiler/rustc_parse/src/parser/attr.rs
+++ b/compiler/rustc_parse/src/parser/attr.rs
@@ -3,7 +3,7 @@ use rustc_ast::token::{self, MetaVarKind};
 use rustc_ast::tokenstream::ParserRange;
 use rustc_ast::{AttrItemKind, Attribute, attr};
 use rustc_errors::codes::*;
-use rustc_errors::{Diag, PResult};
+use rustc_errors::{Diag, PResult, inline_fluent};
 use rustc_span::{BytePos, Span};
 use thin_vec::ThinVec;
 use tracing::debug;
@@ -13,7 +13,7 @@ use super::{
     Trailing, UsePreAttrPos,
 };
 use crate::parser::FnContext;
-use crate::{errors, exp, fluent_generated as fluent};
+use crate::{errors, exp};
 
 // Public for rustfmt usage
 #[derive(Debug)]
@@ -68,7 +68,7 @@ impl<'a> Parser<'a> {
                     let span = self.token.span;
                     let mut err = self
                         .dcx()
-                        .struct_span_err(span, fluent::parse_inner_doc_comment_not_permitted);
+                        .struct_span_err(span, inline_fluent!("expected outer doc comment"));
                     err.code(E0753);
                     if let Some(replacement_span) = self.annotate_following_item_if_applicable(
                         &mut err,
@@ -79,10 +79,12 @@ impl<'a> Parser<'a> {
                         },
                         true,
                     ) {
-                        err.note(fluent::parse_note);
+                        err.note(inline_fluent!(
+                            "inner doc comments like this (starting with `//!` or `/*!`) can only appear before items"
+                        ));
                         err.span_suggestion_verbose(
                             replacement_span,
-                            fluent::parse_suggestion,
+                            inline_fluent!("you might have meant to write a regular comment"),
                             "",
                             rustc_errors::Applicability::MachineApplicable,
                         );
@@ -209,11 +211,11 @@ impl<'a> Parser<'a> {
             Ok(Some(item)) => {
                 // FIXME(#100717)
                 err.arg("item", item.kind.descr());
-                err.span_label(item.span, fluent::parse_label_does_not_annotate_this);
+                err.span_label(item.span, inline_fluent!("BROKEN"));
                 if suggest_to_outer {
                     err.span_suggestion_verbose(
                         replacement_span,
-                        fluent::parse_sugg_change_inner_to_outer,
+                        inline_fluent!("BROKEN"),
                         match attr_type {
                             OuterAttributeType::Attribute => "",
                             OuterAttributeType::DocBlockComment => "*",
@@ -244,28 +246,42 @@ impl<'a> Parser<'a> {
                     self.dcx()
                         .struct_span_err(
                             attr_sp,
-                            fluent::parse_inner_attr_not_permitted_after_outer_doc_comment,
+                            inline_fluent!(
+                                "an inner attribute is not permitted following an outer doc comment"
+                            ),
+                        )
+                        .with_span_label(
+                            attr_sp,
+                            inline_fluent!("not permitted following an outer doc comment"),
                         )
-                        .with_span_label(attr_sp, fluent::parse_label_attr)
                         .with_span_label(
                             prev_doc_comment_span,
-                            fluent::parse_label_prev_doc_comment,
+                            inline_fluent!("previous doc comment"),
                         )
                 }
                 Some(InnerAttrForbiddenReason::AfterOuterAttribute { prev_outer_attr_sp }) => self
                     .dcx()
                     .struct_span_err(
                         attr_sp,
-                        fluent::parse_inner_attr_not_permitted_after_outer_attr,
+                        inline_fluent!(
+                            "an inner attribute is not permitted following an outer attribute"
+                        ),
                     )
-                    .with_span_label(attr_sp, fluent::parse_label_attr)
-                    .with_span_label(prev_outer_attr_sp, fluent::parse_label_prev_attr),
-                Some(InnerAttrForbiddenReason::InCodeBlock) | None => {
-                    self.dcx().struct_span_err(attr_sp, fluent::parse_inner_attr_not_permitted)
-                }
+                    .with_span_label(
+                        attr_sp,
+                        inline_fluent!("not permitted following an outer attribute"),
+                    )
+                    .with_span_label(
+                        prev_outer_attr_sp,
+                        inline_fluent!("previous outer attribute"),
+                    ),
+                Some(InnerAttrForbiddenReason::InCodeBlock) | None => self.dcx().struct_span_err(
+                    attr_sp,
+                    inline_fluent!("an inner attribute is not permitted in this context"),
+                ),
             };
 
-            diag.note(fluent::parse_inner_attr_explanation);
+            diag.note(inline_fluent!("inner attributes, like `#![no_std]`, annotate the item enclosing them, and are usually found at the beginning of source files"));
             if self
                 .annotate_following_item_if_applicable(
                     &mut diag,
@@ -275,7 +291,9 @@ impl<'a> Parser<'a> {
                 )
                 .is_some()
             {
-                diag.note(fluent::parse_outer_attr_explanation);
+                diag.note(inline_fluent!(
+                    "outer attributes, like `#[test]`, annotate the item following them"
+                ));
             };
             diag.emit();
         }
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index cb8a291c7fa0..adba2065c457 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -13,7 +13,7 @@ use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{
     Applicability, Diag, DiagCtxtHandle, ErrorGuaranteed, PResult, Subdiagnostic, Suggestions,
-    pluralize,
+    inline_fluent, pluralize,
 };
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
@@ -41,10 +41,10 @@ use crate::errors::{
     TernaryOperatorSuggestion, UnexpectedConstInGenericParam, UnexpectedConstParamDeclaration,
     UnexpectedConstParamDeclarationSugg, UnmatchedAngleBrackets, UseEqInstead, WrapType,
 };
+use crate::exp;
 use crate::parser::FnContext;
 use crate::parser::attr::InnerAttrPolicy;
 use crate::parser::item::IsDotDotDot;
-use crate::{exp, fluent_generated as fluent};
 
 /// Creates a placeholder argument.
 pub(super) fn dummy_arg(ident: Ident, guar: ErrorGuaranteed) -> Param {
@@ -1272,7 +1272,7 @@ impl<'a> Parser<'a> {
                         // We made sense of it. Improve the error message.
                         e.span_suggestion_verbose(
                             binop.span.shrink_to_lo(),
-                            fluent::parse_sugg_turbofish_syntax,
+                            inline_fluent!("use `::<...>` instead of `<...>` to specify lifetime, type, or const arguments"),
                             "::",
                             Applicability::MaybeIncorrect,
                         );
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 6b61504f2327..c35ddc4dc943 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -11,7 +11,7 @@ use rustc_ast::{
 };
 use rustc_ast_pretty::pprust;
 use rustc_errors::codes::*;
-use rustc_errors::{Applicability, PResult, StashKey, struct_span_code_err};
+use rustc_errors::{Applicability, PResult, StashKey, inline_fluent, struct_span_code_err};
 use rustc_session::lint::builtin::VARARGS_WITHOUT_PATTERN;
 use rustc_span::edit_distance::edit_distance;
 use rustc_span::edition::Edition;
@@ -26,7 +26,7 @@ use super::{
     Parser, PathStyle, Recovered, Trailing, UsePreAttrPos,
 };
 use crate::errors::{self, FnPointerCannotBeAsync, FnPointerCannotBeConst, MacroExpandsToAdtField};
-use crate::{exp, fluent_generated as fluent};
+use crate::exp;
 
 impl<'a> Parser<'a> {
     /// Parses a source module as a crate. This is the main entry point for the parser.
@@ -1721,7 +1721,7 @@ impl<'a> Parser<'a> {
 
             if this.token == token::Bang {
                 if let Err(err) = this.unexpected() {
-                    err.with_note(fluent::parse_macro_expands_to_enum_variant).emit();
+                    err.with_note(inline_fluent!("macros cannot expand to enum variants")).emit();
                 }
 
                 this.bump();
diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs
index 61b498431b04..1b17f422345f 100644
--- a/compiler/rustc_parse/src/parser/tests.rs
+++ b/compiler/rustc_parse/src/parser/tests.rs
@@ -25,10 +25,6 @@ use crate::lexer::StripTokens;
 use crate::parser::{AllowConstBlockItems, ForceCollect, Parser};
 use crate::{new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
 
-fn psess() -> ParseSess {
-    ParseSess::new(vec![crate::DEFAULT_LOCALE_RESOURCE])
-}
-
 fn filename(sm: &SourceMap, path: &str) -> FileName {
     FileName::Real(sm.path_mapping().to_real_filename(sm.working_dir(), PathBuf::from(path)))
 }
@@ -46,7 +42,7 @@ fn string_to_parser(psess: &ParseSess, source_str: String) -> Parser<'_> {
 fn create_test_handler(theme: OutputTheme) -> (DiagCtxt, Arc, Arc>>) {
     let output = Arc::new(Mutex::new(Vec::new()));
     let source_map = Arc::new(SourceMap::new(FilePathMapping::empty()));
-    let translator = Translator::with_fallback_bundle(vec![crate::DEFAULT_LOCALE_RESOURCE], false);
+    let translator = Translator::with_fallback_bundle(vec![], false);
     let shared: Box = Box::new(Shared { data: output.clone() });
     let auto_stream = AutoStream::never(shared);
     let dcx = DiagCtxt::new(Box::new(
@@ -93,7 +89,7 @@ where
 
 /// Maps a string to tts, using a made-up filename.
 pub(crate) fn string_to_stream(source_str: String) -> TokenStream {
-    let psess = psess();
+    let psess = ParseSess::new(vec![]);
     unwrap_or_emit_fatal(source_str_to_stream(
         &psess,
         filename(psess.source_map(), "bogofile"),
@@ -2243,12 +2239,12 @@ fn sp(a: u32, b: u32) -> Span {
 
 /// Parses a string, return an expression.
 fn string_to_expr(source_str: String) -> Box {
-    with_error_checking_parse(source_str, &psess(), |p| p.parse_expr())
+    with_error_checking_parse(source_str, &ParseSess::new(vec![]), |p| p.parse_expr())
 }
 
 /// Parses a string, returns an item.
 fn string_to_item(source_str: String) -> Option> {
-    with_error_checking_parse(source_str, &psess(), |p| {
+    with_error_checking_parse(source_str, &ParseSess::new(vec![]), |p| {
         p.parse_item(ForceCollect::No, AllowConstBlockItems::Yes)
     })
 }
@@ -2484,7 +2480,7 @@ let mut fflags: c_int = wb();
 #[test]
 fn crlf_doc_comments() {
     create_default_session_globals_then(|| {
-        let psess = psess();
+        let psess = ParseSess::new(vec![]);
 
         let name_1 = FileName::Custom("crlf_source_1".to_string());
         let source = "/// doc comment\r\nfn foo() {}".to_string();
@@ -2519,7 +2515,7 @@ fn ttdelim_span() {
     }
 
     create_default_session_globals_then(|| {
-        let psess = psess();
+        let psess = ParseSess::new(vec![]);
         let expr = parse_expr_from_source_str(
             filename(psess.source_map(), "foo"),
             "foo!( fn main() { body } )".to_string(),
@@ -2555,7 +2551,7 @@ fn look_ahead() {
         let sym_S = Symbol::intern("S");
         let raw_no = IdentIsRaw::No;
 
-        let psess = psess();
+        let psess = ParseSess::new(vec![]);
         let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
 
         // Current position is the `fn`.
@@ -2630,7 +2626,7 @@ fn look_ahead_non_outermost_stream() {
         let sym_S = Symbol::intern("S");
         let raw_no = IdentIsRaw::No;
 
-        let psess = psess();
+        let psess = ParseSess::new(vec![]);
         let mut p = string_to_parser(&psess, "mod m { fn f(x: u32) { x } struct S; }".to_string());
 
         // Move forward to the `fn`, which is not within the outermost token
@@ -2662,7 +2658,7 @@ fn look_ahead_non_outermost_stream() {
 #[test]
 fn debug_lookahead() {
     create_default_session_globals_then(|| {
-        let psess = psess();
+        let psess = ParseSess::new(vec![]);
         let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
 
         // Current position is the `fn`.
@@ -2883,7 +2879,7 @@ fn debug_lookahead() {
 #[test]
 fn out_of_line_mod() {
     create_default_session_globals_then(|| {
-        let psess = psess();
+        let psess = ParseSess::new(vec![]);
         let item = parse_item_from_source_str(
             filename(psess.source_map(), "foo"),
             "mod foo { struct S; mod this_does_not_exist; }".to_owned(),
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index d198675c0b22..7bebbc579c43 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -313,8 +313,8 @@ impl ParseSess {
         }
     }
 
-    pub fn emitter_with_note(locale_resources: Vec<&'static str>, note: String) -> Self {
-        let translator = Translator::with_fallback_bundle(locale_resources, false);
+    pub fn emitter_with_note(note: String) -> Self {
+        let translator = Translator::with_fallback_bundle(vec![], false);
         let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
         let emitter = Box::new(AnnotateSnippetEmitter::new(
             stderr_destination(ColorConfig::Auto),
diff --git a/tests/ui-fulldeps/mod_dir_path_canonicalized.rs b/tests/ui-fulldeps/mod_dir_path_canonicalized.rs
index 86f2d5f5954a..894fec248f1f 100644
--- a/tests/ui-fulldeps/mod_dir_path_canonicalized.rs
+++ b/tests/ui-fulldeps/mod_dir_path_canonicalized.rs
@@ -29,7 +29,7 @@ pub fn main() {
 }
 
 fn parse() {
-    let psess = ParseSess::new(vec![rustc_parse::DEFAULT_LOCALE_RESOURCE]);
+    let psess = ParseSess::new(vec![]);
 
     let path = Path::new(file!());
     let path = path.canonicalize().unwrap();
diff --git a/tests/ui-fulldeps/pprust-expr-roundtrip.rs b/tests/ui-fulldeps/pprust-expr-roundtrip.rs
index 08ded2aee53d..7fe71004522a 100644
--- a/tests/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/tests/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -219,7 +219,7 @@ fn main() {
 }
 
 fn run() {
-    let psess = ParseSess::new(vec![rustc_parse::DEFAULT_LOCALE_RESOURCE]);
+    let psess = ParseSess::new(vec![]);
 
     iter_exprs(2, &mut |mut e| {
         // If the pretty printer is correct, then `parse(print(e))` should be identical to `e`,
diff --git a/tests/ui-fulldeps/pprust-parenthesis-insertion.rs b/tests/ui-fulldeps/pprust-parenthesis-insertion.rs
index 72b5cfb90630..90cf9b89a293 100644
--- a/tests/ui-fulldeps/pprust-parenthesis-insertion.rs
+++ b/tests/ui-fulldeps/pprust-parenthesis-insertion.rs
@@ -196,7 +196,7 @@ fn main() -> ExitCode {
     };
 
     rustc_span::create_default_session_globals_then(|| {
-        let psess = &ParseSess::new(vec![rustc_parse::DEFAULT_LOCALE_RESOURCE]);
+        let psess = &ParseSess::new(vec![]);
 
         for &source_code in EXPRS {
             let Some(expr) = parse_expr(psess, source_code) else {

From cda9b8c15752df635d693cfe73b147e6090faddc Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 16:44:53 +0100
Subject: [PATCH 1390/1843] Split `ComparisonOrShiftInterpretedAsGenericSugg`

---
 compiler/rustc_parse/src/errors.rs      | 17 +++++++++++++----
 compiler/rustc_parse/src/parser/expr.rs | 15 ++++++++-------
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 4a7031257700..418d56f632bd 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -861,7 +861,16 @@ pub(crate) struct ComparisonInterpretedAsGeneric {
     #[label("interpreted as generic arguments")]
     pub args: Span,
     #[subdiagnostic]
-    pub suggestion: ComparisonOrShiftInterpretedAsGenericSugg,
+    pub suggestion: ComparisonInterpretedAsGenericSugg,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion("try comparing the cast value", applicability = "machine-applicable")]
+pub(crate) struct ComparisonInterpretedAsGenericSugg {
+    #[suggestion_part(code = "(")]
+    pub left: Span,
+    #[suggestion_part(code = ")")]
+    pub right: Span,
 }
 
 #[derive(Diagnostic)]
@@ -874,12 +883,12 @@ pub(crate) struct ShiftInterpretedAsGeneric {
     #[label("interpreted as generic arguments")]
     pub args: Span,
     #[subdiagnostic]
-    pub suggestion: ComparisonOrShiftInterpretedAsGenericSugg,
+    pub suggestion: ShiftInterpretedAsGenericSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion("BROKEN", applicability = "machine-applicable")]
-pub(crate) struct ComparisonOrShiftInterpretedAsGenericSugg {
+#[multipart_suggestion("try shifting the cast value", applicability = "machine-applicable")]
+pub(crate) struct ShiftInterpretedAsGenericSugg {
     #[suggestion_part(code = "(")]
     pub left: Span,
     #[suggestion_part(code = ")")]
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 11cde35fab17..8bb22c2a831b 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -727,25 +727,26 @@ impl<'a> Parser<'a> {
                         );
 
                         let args_span = self.look_ahead(1, |t| t.span).to(span_after_type);
-                        let suggestion = errors::ComparisonOrShiftInterpretedAsGenericSugg {
-                            left: expr.span.shrink_to_lo(),
-                            right: expr.span.shrink_to_hi(),
-                        };
-
                         match self.token.kind {
                             token::Lt => {
                                 self.dcx().emit_err(errors::ComparisonInterpretedAsGeneric {
                                     comparison: self.token.span,
                                     r#type: path,
                                     args: args_span,
-                                    suggestion,
+                                    suggestion: errors::ComparisonInterpretedAsGenericSugg {
+                                        left: expr.span.shrink_to_lo(),
+                                        right: expr.span.shrink_to_hi(),
+                                    },
                                 })
                             }
                             token::Shl => self.dcx().emit_err(errors::ShiftInterpretedAsGeneric {
                                 shift: self.token.span,
                                 r#type: path,
                                 args: args_span,
-                                suggestion,
+                                suggestion: errors::ShiftInterpretedAsGenericSugg {
+                                    left: expr.span.shrink_to_lo(),
+                                    right: expr.span.shrink_to_hi(),
+                                },
                             }),
                             _ => {
                                 // We can end up here even without `<` being the next token, for

From c5587ca91967f91bf2f1467fbfa68e5d5448bce3 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 16:59:55 +0100
Subject: [PATCH 1391/1843] Split `parse_inner_attr` errors by case

---
 compiler/rustc_parse/src/parser/attr.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
index 3830d4b4cd41..3d16f37c000e 100644
--- a/compiler/rustc_parse/src/parser/attr.rs
+++ b/compiler/rustc_parse/src/parser/attr.rs
@@ -209,13 +209,25 @@ impl<'a> Parser<'a> {
             AllowConstBlockItems::Yes,
         ) {
             Ok(Some(item)) => {
-                // FIXME(#100717)
                 err.arg("item", item.kind.descr());
-                err.span_label(item.span, inline_fluent!("BROKEN"));
+                err.span_label(
+                    item.span,
+                    match attr_type {
+                        OuterAttributeType::Attribute => {
+                            inline_fluent!("the inner attribute doesn't annotate this {$item}")
+                        }
+                        OuterAttributeType::DocComment | OuterAttributeType::DocBlockComment => {
+                            inline_fluent!("the inner doc comment doesn't annotate this {$item}")
+                        }
+                    },
+                );
                 if suggest_to_outer {
                     err.span_suggestion_verbose(
                         replacement_span,
-                        inline_fluent!("BROKEN"),
+                        match attr_type {
+                            OuterAttributeType::Attribute =>  inline_fluent!("to annotate the {$item}, change the attribute from inner to outer style"),
+                            OuterAttributeType::DocComment | OuterAttributeType::DocBlockComment =>  inline_fluent!("to annotate the {$item}, change the doc comment from inner to outer style"),
+                        },
                         match attr_type {
                             OuterAttributeType::Attribute => "",
                             OuterAttributeType::DocBlockComment => "*",

From 0ef518c94695c7d852ec49ec9903d338efec109d Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 18:00:22 +0100
Subject: [PATCH 1392/1843] Disable the `run-make/translation` test for now

---
 tests/run-make/translation/rmake.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/run-make/translation/rmake.rs b/tests/run-make/translation/rmake.rs
index 86078888c2e1..4d7fd71a2f4a 100644
--- a/tests/run-make/translation/rmake.rs
+++ b/tests/run-make/translation/rmake.rs
@@ -10,6 +10,9 @@
 //@ needs-symlink
 //@ needs-subprocess
 
+// FIXME(151366) Currently `-Ztranslate-additional-ftl` is currently broken
+//@ ignore-test
+
 #![deny(warnings)]
 
 use std::path::{Path, PathBuf};

From 00dd7dbe57923937377ed81369ec3206bf61b161 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sat, 7 Feb 2026 10:31:36 +0100
Subject: [PATCH 1393/1843] Allow more capitalized words

---
 compiler/rustc_macros/src/diagnostics/message.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler/rustc_macros/src/diagnostics/message.rs b/compiler/rustc_macros/src/diagnostics/message.rs
index ad3ca4d1f8ea..2db8df2f69ae 100644
--- a/compiler/rustc_macros/src/diagnostics/message.rs
+++ b/compiler/rustc_macros/src/diagnostics/message.rs
@@ -156,6 +156,8 @@ const ALLOWED_CAPITALIZED_WORDS: &[&str] = &[
     "MIR",
     "NaNs",
     "OK",
+    "Rust",
+    "Unicode",
     "VS",
     // tidy-alphabetical-end
 ];

From c9d59bac1ec002137c1e53bc211f29996a849cea Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 04:39:20 -0600
Subject: [PATCH 1394/1843] ci: Update all docker images to the latest version

---
 .../ci/docker/aarch64-unknown-linux-gnu/Dockerfile          | 6 +++---
 .../ci/docker/arm-unknown-linux-gnueabi/Dockerfile          | 6 +++---
 .../ci/docker/arm-unknown-linux-gnueabihf/Dockerfile        | 6 +++---
 .../ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile      | 6 +++---
 .../ci/docker/i586-unknown-linux-gnu/Dockerfile             | 2 +-
 .../ci/docker/i686-unknown-linux-gnu/Dockerfile             | 2 +-
 .../ci/docker/loongarch64-unknown-linux-gnu/Dockerfile      | 6 +++---
 .../ci/docker/mips-unknown-linux-gnu/Dockerfile             | 6 +++---
 .../ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile      | 6 +++---
 .../ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile    | 6 +++---
 .../ci/docker/mipsel-unknown-linux-gnu/Dockerfile           | 6 +++---
 .../ci/docker/powerpc-unknown-linux-gnu/Dockerfile          | 6 +++---
 .../ci/docker/powerpc64-unknown-linux-gnu/Dockerfile        | 6 +++---
 .../ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile      | 6 +++---
 .../ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile        | 6 +++---
 .../ci/docker/thumbv6m-none-eabi/Dockerfile                 | 2 +-
 .../ci/docker/thumbv7em-none-eabi/Dockerfile                | 2 +-
 .../ci/docker/thumbv7em-none-eabihf/Dockerfile              | 2 +-
 .../ci/docker/thumbv7m-none-eabi/Dockerfile                 | 2 +-
 .../ci/docker/wasm32-unknown-unknown/Dockerfile             | 2 +-
 .../ci/docker/x86_64-unknown-linux-gnu/Dockerfile           | 2 +-
 library/compiler-builtins/ci/run-docker.sh                  | 2 +-
 22 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
index 69b99f5b6b32..683bd07fd47e 100644
--- a/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
     gcc-aarch64-linux-gnu m4 make libc6-dev-arm64-cross \
-    qemu-user-static
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=aarch64-linux-gnu-
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-aarch64-static \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-aarch64 \
     AR_aarch64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_aarch64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/aarch64-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
index 2fa6f8520520..781abd1b6e88 100644
--- a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
@@ -1,14 +1,14 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
-    gcc-arm-linux-gnueabi libc6-dev-armel-cross qemu-user-static
+    gcc-arm-linux-gnueabi libc6-dev-armel-cross qemu-user
 
 ENV TOOLCHAIN_PREFIX=arm-linux-gnueabi-
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm-static \
+    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm \
     AR_arm_unknown_linux_gnueabi="$TOOLCHAIN_PREFIX"ar \
     CC_arm_unknown_linux_gnueabi="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/arm-linux-gnueabi \
diff --git a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
index 85f7335f5a85..36ea4827dc52 100644
--- a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
@@ -1,14 +1,14 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
-    gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static
+    gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
 
 ENV TOOLCHAIN_PREFIX=arm-linux-gnueabihf-
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm-static \
+    CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm \
     AR_arm_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"ar \
     CC_arm_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \
diff --git a/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
index 42511479f36f..8b76693b2799 100644
--- a/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
@@ -1,14 +1,14 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
-    gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static
+    gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
 
 ENV TOOLCHAIN_PREFIX=arm-linux-gnueabihf-
 ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm-static \
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm \
     AR_armv7_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"ar \
     CC_armv7_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \
diff --git a/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile
index 35488c477493..9125038acbde 100644
--- a/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile
index 35488c477493..9125038acbde 100644
--- a/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
index e95a1b9163ff..a65223595877 100644
--- a/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
@@ -1,13 +1,13 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user-static ca-certificates \
+    gcc libc6-dev qemu-user ca-certificates \
     gcc-14-loongarch64-linux-gnu libc6-dev-loong64-cross
 
 ENV CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER=loongarch64-linux-gnu-gcc-14 \
-    CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-loongarch64-static \
+    CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-loongarch64 \
     AR_loongarch64_unknown_linux_gnu=loongarch64-linux-gnu-ar \
     CC_loongarch64_unknown_linux_gnu=loongarch64-linux-gnu-gcc-14 \
     QEMU_LD_PREFIX=/usr/loongarch64-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
index fd1877603100..0913f33c05ce 100644
--- a/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
     gcc-mips-linux-gnu libc6-dev-mips-cross \
-    binfmt-support qemu-user-static qemu-system-mips
+    binfmt-support qemu-user qemu-system-mips
 
 ENV TOOLCHAIN_PREFIX=mips-linux-gnu-
 ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_RUNNER=qemu-mips-static \
+    CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_RUNNER=qemu-mips \
     AR_mips_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_mips_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/mips-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
index 4e542ce6858c..d2f4e484b1aa 100644
--- a/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
@@ -8,12 +8,12 @@ RUN apt-get update && \
     gcc-mips64-linux-gnuabi64 \
     libc6-dev \
     libc6-dev-mips64-cross \
-    qemu-user-static \
+    qemu-user \
     qemu-system-mips
 
 ENV TOOLCHAIN_PREFIX=mips64-linux-gnuabi64-
 ENV CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64-static \
+    CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64 \
     AR_mips64_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"ar \
     CC_mips64_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/mips64-linux-gnuabi64 \
diff --git a/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
index 528dfd8940d5..873754b2793e 100644
--- a/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
@@ -8,11 +8,11 @@ RUN apt-get update && \
     gcc-mips64el-linux-gnuabi64 \
     libc6-dev \
     libc6-dev-mips64el-cross \
-    qemu-user-static
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=mips64el-linux-gnuabi64-
 ENV CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64el-static \
+    CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64el \
     AR_mips64el_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"ar \
     CC_mips64el_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/mips64el-linux-gnuabi64 \
diff --git a/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
index 2572180238e2..5768b68d6c95 100644
--- a/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
     gcc-mipsel-linux-gnu libc6-dev-mipsel-cross \
-    binfmt-support qemu-user-static
+    binfmt-support qemu-user
 
 ENV TOOLCHAIN_PREFIX=mipsel-linux-gnu-
 ENV CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_RUNNER=qemu-mipsel-static \
+    CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_RUNNER=qemu-mipsel \
     AR_mipsel_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_mipsel_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/mipsel-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
index cac1f23610aa..c625a4bcd5d7 100644
--- a/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user-static ca-certificates \
+    gcc libc6-dev qemu-user ca-certificates \
     gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
     qemu-system-ppc
 
 ENV TOOLCHAIN_PREFIX=powerpc-linux-gnu-
 ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc-static \
+    CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc \
     AR_powerpc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_powerpc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/powerpc-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
index 76127b7dbb8c..86a7a8cd46e4 100644
--- a/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
     gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
-    binfmt-support qemu-user-static qemu-system-ppc
+    binfmt-support qemu-user qemu-system-ppc
 
 ENV TOOLCHAIN_PREFIX=powerpc64-linux-gnu-
 ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64-static \
+    CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 \
     AR_powerpc64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_powerpc64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/powerpc64-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
index da1d56ca66f2..722b10b0a734 100644
--- a/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user-static ca-certificates \
+    gcc libc6-dev qemu-user ca-certificates \
     gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
     qemu-system-ppc
 
 ENV TOOLCHAIN_PREFIX=powerpc64le-linux-gnu-
 ENV CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64le-static \
+    CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64le \
     AR_powerpc64le_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_powerpc64le_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/powerpc64le-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
index 513efacd6d96..7a721ba05416 100644
--- a/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
@@ -1,15 +1,15 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user-static ca-certificates \
+    gcc libc6-dev qemu-user ca-certificates \
     gcc-riscv64-linux-gnu libc6-dev-riscv64-cross \
     qemu-system-riscv64
 
 ENV TOOLCHAIN_PREFIX=riscv64-linux-gnu-
 ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
-    CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER=qemu-riscv64-static \
+    CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER=qemu-riscv64 \
     AR_riscv64gc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_riscv64gc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
     QEMU_LD_PREFIX=/usr/riscv64-linux-gnu \
diff --git a/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile b/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile
index a9a172a21137..a1a6b3cf5cfd 100644
--- a/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile b/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile
index a9a172a21137..a1a6b3cf5cfd 100644
--- a/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile b/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile
index a9a172a21137..a1a6b3cf5cfd 100644
--- a/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile b/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile
index a9a172a21137..a1a6b3cf5cfd 100644
--- a/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile b/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile
index 2813d318670e..b646a72bb37c 100644
--- a/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile
+++ b/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:20.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
index 2ef800129d67..927515f90f32 100644
--- a/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,4 @@
-ARG IMAGE=ubuntu:25.04
+ARG IMAGE=ubuntu:25.10
 FROM $IMAGE
 
 RUN apt-get update && \
diff --git a/library/compiler-builtins/ci/run-docker.sh b/library/compiler-builtins/ci/run-docker.sh
index 4c1fe0fe2644..e65ada271904 100755
--- a/library/compiler-builtins/ci/run-docker.sh
+++ b/library/compiler-builtins/ci/run-docker.sh
@@ -97,7 +97,7 @@ if [ "${1:-}" = "--help" ] || [ "$#" -gt 1 ]; then
     usage: ./ci/run-docker.sh [target]
 
     you can also set DOCKER_BASE_IMAGE to use something other than the default
-    ubuntu:25.04 (or rustlang/rust:nightly).
+    ubuntu:25.10 (or rustlang/rust:nightly).
     "
     exit
 fi

From 4d24b508bb1004116eb71b93afaabfd83a3aa9e3 Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 04:10:45 -0600
Subject: [PATCH 1395/1843] meta: Upgrade all dependencies to the latest
 compatible versions

This allows us to drop wasm-specific configuration for `getrandom`.

Link: https://github.com/rust-random/getrandom/blob/314fd5ab3e6d9ef2ec90243894731865a725417d/CHANGELOG.md#major-change-to-wasm_js-backend
---
 library/compiler-builtins/ci/run.sh              |  5 -----
 .../crates/libm-macros/Cargo.toml                |  6 +++---
 .../crates/musl-math-sys/Cargo.toml              |  2 +-
 .../crates/symbol-check/Cargo.toml               |  4 ++--
 library/compiler-builtins/crates/util/Cargo.toml |  2 +-
 library/compiler-builtins/libm-test/Cargo.toml   | 16 ++++++++--------
 .../compiler-builtins/libm-test/src/precision.rs |  2 +-
 7 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh
index 0c07b32c74b9..52fb4b151ca5 100755
--- a/library/compiler-builtins/ci/run.sh
+++ b/library/compiler-builtins/ci/run.sh
@@ -13,11 +13,6 @@ if [ -z "$target" ]; then
     target="$host_target"
 fi
 
-if [[ "$target" = *"wasm"* ]]; then
-    # Enable the random backend
-    export RUSTFLAGS="${RUSTFLAGS:-} --cfg getrandom_backend=\"wasm_js\""
-fi
-
 if [ "${USING_CONTAINER_RUSTC:-}" = 1 ]; then
     # Install nonstandard components if we have control of the environment
     rustup target list --installed |
diff --git a/library/compiler-builtins/crates/libm-macros/Cargo.toml b/library/compiler-builtins/crates/libm-macros/Cargo.toml
index 100a8d0ec30e..f6697b783457 100644
--- a/library/compiler-builtins/crates/libm-macros/Cargo.toml
+++ b/library/compiler-builtins/crates/libm-macros/Cargo.toml
@@ -10,9 +10,9 @@ proc-macro = true
 
 [dependencies]
 heck = "0.5.0"
-proc-macro2 = "1.0.95"
-quote = "1.0.40"
-syn = { version = "2.0.104", features = ["full", "extra-traits", "visit-mut"] }
+proc-macro2 = "1.0.106"
+quote = "1.0.44"
+syn = { version = "2.0.114", features = ["full", "extra-traits", "visit-mut"] }
 
 [lints.rust]
 # Values used during testing
diff --git a/library/compiler-builtins/crates/musl-math-sys/Cargo.toml b/library/compiler-builtins/crates/musl-math-sys/Cargo.toml
index 39f6fa9065bd..60b0647b6dc6 100644
--- a/library/compiler-builtins/crates/musl-math-sys/Cargo.toml
+++ b/library/compiler-builtins/crates/musl-math-sys/Cargo.toml
@@ -11,4 +11,4 @@ license = "MIT OR Apache-2.0"
 libm = { path = "../../libm" }
 
 [build-dependencies]
-cc = "1.2.29"
+cc = "1.2.55"
diff --git a/library/compiler-builtins/crates/symbol-check/Cargo.toml b/library/compiler-builtins/crates/symbol-check/Cargo.toml
index e2218b491720..9f027df1e4f5 100644
--- a/library/compiler-builtins/crates/symbol-check/Cargo.toml
+++ b/library/compiler-builtins/crates/symbol-check/Cargo.toml
@@ -5,8 +5,8 @@ edition = "2024"
 publish = false
 
 [dependencies]
-object = "0.37.1"
-serde_json = "1.0.140"
+object = "0.37.3"
+serde_json = "1.0.149"
 
 [features]
 wasm = ["object/wasm"]
diff --git a/library/compiler-builtins/crates/util/Cargo.toml b/library/compiler-builtins/crates/util/Cargo.toml
index 614c54bd8355..b1ccd8a9e63c 100644
--- a/library/compiler-builtins/crates/util/Cargo.toml
+++ b/library/compiler-builtins/crates/util/Cargo.toml
@@ -16,4 +16,4 @@ libm = { path = "../../libm", default-features = false }
 libm-macros = { path = "../libm-macros" }
 libm-test = { path = "../../libm-test", default-features = false }
 musl-math-sys = { path = "../musl-math-sys", optional = true }
-rug = { version = "1.27.0", optional = true, default-features = false, features = ["float", "std"] }
+rug = { version = "1.28.1", optional = true, default-features = false, features = ["float", "std"] }
diff --git a/library/compiler-builtins/libm-test/Cargo.toml b/library/compiler-builtins/libm-test/Cargo.toml
index adecfc1af6b8..b813331a8552 100644
--- a/library/compiler-builtins/libm-test/Cargo.toml
+++ b/library/compiler-builtins/libm-test/Cargo.toml
@@ -28,25 +28,25 @@ icount = ["dep:gungraun"]
 short-benchmarks = []
 
 [dependencies]
-anyhow = "1.0.98"
+anyhow = "1.0.101"
 # This is not directly used but is required so we can enable `gmp-mpfr-sys/force-cross`.
-gmp-mpfr-sys = { version = "1.6.5", optional = true, default-features = false }
+gmp-mpfr-sys = { version = "1.6.8", optional = true, default-features = false }
 gungraun = { version = "0.17.0", optional = true }
-indicatif = { version = "0.18.0", default-features = false }
+indicatif = { version = "0.18.3", default-features = false }
 libm = { path = "../libm", features = ["unstable-public-internals"] }
 libm-macros = { path = "../crates/libm-macros" }
 musl-math-sys = { path = "../crates/musl-math-sys", optional = true }
 paste = "1.0.15"
-rand = "0.9.1"
+rand = "0.9.2"
 rand_chacha = "0.9.0"
-rayon = "1.10.0"
-rug = { version = "1.27.0", optional = true, default-features = false, features = ["float", "integer", "std"] }
+rayon = "1.11.0"
+rug = { version = "1.28.1", optional = true, default-features = false, features = ["float", "integer", "std"] }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
-getrandom = { version = "0.3.3", features = ["wasm_js"] }
+getrandom = { version = "0.3.4", features = ["wasm_js"] }
 
 [build-dependencies]
-rand = { version = "0.9.1", optional = true }
+rand = { version = "0.9.2", optional = true }
 
 [dev-dependencies]
 criterion = { version = "0.6.0", default-features = false, features = ["cargo_bench_support"] }
diff --git a/library/compiler-builtins/libm-test/src/precision.rs b/library/compiler-builtins/libm-test/src/precision.rs
index 897f21da78e7..a94fe429f5f3 100644
--- a/library/compiler-builtins/libm-test/src/precision.rs
+++ b/library/compiler-builtins/libm-test/src/precision.rs
@@ -498,7 +498,7 @@ fn int_float_common(
         if input.0 > 4000 {
             return XFAIL_NOCHECK;
         } else if input.0 > 100 {
-            return CheckAction::AssertWithUlp(1_000_000);
+            return CheckAction::AssertWithUlp(2_000_000);
         }
     }
     DEFAULT

From 6733cf42a6f5e26cd98b2ae7ad87b4e3c57ffe9b Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 05:04:20 -0600
Subject: [PATCH 1396/1843] symcheck: Enable wasm by default

The build time isn't very different so we can keep things simpler.
---
 library/compiler-builtins/ci/run.sh                      | 1 -
 library/compiler-builtins/crates/symbol-check/Cargo.toml | 5 +----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh
index 52fb4b151ca5..ca1304f3dc87 100755
--- a/library/compiler-builtins/ci/run.sh
+++ b/library/compiler-builtins/ci/run.sh
@@ -47,7 +47,6 @@ fi
 # `compiler-builtins` is built with various features. Symcheck invokes Cargo to
 # build with the arguments we provide it, then validates the built artifacts.
 symcheck=(cargo run -p symbol-check --release)
-[[ "$target" = "wasm"* ]] && symcheck+=(--features wasm)
 symcheck+=(-- build-and-check)
 
 "${symcheck[@]}" "$target" -- -p compiler_builtins
diff --git a/library/compiler-builtins/crates/symbol-check/Cargo.toml b/library/compiler-builtins/crates/symbol-check/Cargo.toml
index 9f027df1e4f5..9e4bc739da15 100644
--- a/library/compiler-builtins/crates/symbol-check/Cargo.toml
+++ b/library/compiler-builtins/crates/symbol-check/Cargo.toml
@@ -5,8 +5,5 @@ edition = "2024"
 publish = false
 
 [dependencies]
-object = "0.37.3"
+object = { version = "0.37.3", features = ["wasm"] }
 serde_json = "1.0.149"
-
-[features]
-wasm = ["object/wasm"]

From 9184a5f661e074690b59ab28d7ef16c1c6ae0d06 Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Fri, 6 Feb 2026 17:54:06 -0600
Subject: [PATCH 1397/1843] symcheck: Add tests for the symbol checker

Ensure that these are actually doing what is expected.
---
 library/compiler-builtins/ci/run.sh           |   4 +-
 .../crates/symbol-check/Cargo.toml            |   5 +
 .../crates/symbol-check/build.rs              |   6 +
 .../crates/symbol-check/src/main.rs           |  30 ++---
 .../crates/symbol-check/tests/all.rs          | 124 ++++++++++++++++++
 .../symbol-check/tests/input/core_symbols.rs  |  11 ++
 .../symbol-check/tests/input/duplicates.rs    |  12 ++
 .../crates/symbol-check/tests/input/good.rs   |   4 +
 8 files changed, 176 insertions(+), 20 deletions(-)
 create mode 100644 library/compiler-builtins/crates/symbol-check/build.rs
 create mode 100644 library/compiler-builtins/crates/symbol-check/tests/all.rs
 create mode 100644 library/compiler-builtins/crates/symbol-check/tests/input/core_symbols.rs
 create mode 100644 library/compiler-builtins/crates/symbol-check/tests/input/duplicates.rs
 create mode 100644 library/compiler-builtins/crates/symbol-check/tests/input/good.rs

diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh
index ca1304f3dc87..12b3f37889c9 100755
--- a/library/compiler-builtins/ci/run.sh
+++ b/library/compiler-builtins/ci/run.sh
@@ -46,6 +46,7 @@ fi
 # Ensure there are no duplicate symbols or references to `core` when
 # `compiler-builtins` is built with various features. Symcheck invokes Cargo to
 # build with the arguments we provide it, then validates the built artifacts.
+SYMCHECK_TEST_TARGET="$target" cargo test -p symbol-check --release
 symcheck=(cargo run -p symbol-check --release)
 symcheck+=(-- build-and-check)
 
@@ -151,7 +152,8 @@ if [ "${BUILD_ONLY:-}" = "1" ]; then
 
     echo "can't run tests on $target; skipping"
 else
-    mflags+=(--workspace --target "$target")
+    # symcheck tests need specific env setup, and is already tested above
+    mflags+=(--workspace --exclude symbol-check --target "$target")
     cmd=(cargo test "${mflags[@]}")
     profile_flag="--profile"
 
diff --git a/library/compiler-builtins/crates/symbol-check/Cargo.toml b/library/compiler-builtins/crates/symbol-check/Cargo.toml
index 9e4bc739da15..774d1c31a4b0 100644
--- a/library/compiler-builtins/crates/symbol-check/Cargo.toml
+++ b/library/compiler-builtins/crates/symbol-check/Cargo.toml
@@ -7,3 +7,8 @@ publish = false
 [dependencies]
 object = { version = "0.37.3", features = ["wasm"] }
 serde_json = "1.0.149"
+
+[dev-dependencies]
+assert_cmd = "2.1.2"
+cc = "1.2.55"
+tempfile = "3.24.0"
diff --git a/library/compiler-builtins/crates/symbol-check/build.rs b/library/compiler-builtins/crates/symbol-check/build.rs
new file mode 100644
index 000000000000..b3e53c38b0bd
--- /dev/null
+++ b/library/compiler-builtins/crates/symbol-check/build.rs
@@ -0,0 +1,6 @@
+use std::env;
+
+fn main() {
+    println!("cargo::rustc-env=HOST={}", env::var("HOST").unwrap());
+    println!("cargo::rustc-env=TARGET={}", env::var("TARGET").unwrap());
+}
diff --git a/library/compiler-builtins/crates/symbol-check/src/main.rs b/library/compiler-builtins/crates/symbol-check/src/main.rs
index 7d0b7e90addb..1413042e6c70 100644
--- a/library/compiler-builtins/crates/symbol-check/src/main.rs
+++ b/library/compiler-builtins/crates/symbol-check/src/main.rs
@@ -1,7 +1,10 @@
 //! Tool used by CI to inspect compiler-builtins archives and help ensure we won't run into any
 //! linking errors.
+//!
+//! Note that symcheck is a "hostprog", i.e. is built and run on the host target even when the
+//! actual target is cross compiled.
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, BTreeSet, HashSet};
 use std::fs;
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
@@ -42,8 +45,7 @@ fn main() {
             run_build_and_check(target, args);
         }
         ["build-and-check", "--", args @ ..] if !args.is_empty() => {
-            let target = &host_target();
-            run_build_and_check(target, args);
+            run_build_and_check(env!("HOST"), args);
         }
         ["check", paths @ ..] if !paths.is_empty() => {
             check_paths(paths);
@@ -80,20 +82,6 @@ fn check_paths>(paths: &[P]) {
     }
 }
 
-fn host_target() -> String {
-    let out = Command::new("rustc")
-        .arg("--version")
-        .arg("--verbose")
-        .output()
-        .unwrap();
-    assert!(out.status.success());
-    let out = String::from_utf8(out.stdout).unwrap();
-    out.lines()
-        .find_map(|s| s.strip_prefix("host: "))
-        .unwrap()
-        .to_owned()
-}
-
 /// Run `cargo build` with the provided additional arguments, collecting the list of created
 /// libraries.
 fn exec_cargo_with_args(target: &str, args: &[&str]) -> Vec {
@@ -257,8 +245,9 @@ fn verify_no_duplicates(archive: &BinFile) {
     assert!(found_any, "no symbols found");
 
     if !dups.is_empty() {
+        let count = dups.iter().map(|x| &x.name).collect::>().len();
         dups.sort_unstable_by(|a, b| a.name.cmp(&b.name));
-        panic!("found duplicate symbols: {dups:#?}");
+        panic!("found {count} duplicate symbols: {dups:#?}");
     }
 
     println!("    success: no duplicate symbols found");
@@ -293,7 +282,10 @@ fn verify_core_symbols(archive: &BinFile) {
 
     if !undefined.is_empty() {
         undefined.sort_unstable_by(|a, b| a.name.cmp(&b.name));
-        panic!("found undefined symbols from core: {undefined:#?}");
+        panic!(
+            "found {} undefined symbols from core: {undefined:#?}",
+            undefined.len()
+        );
     }
 
     println!("    success: no undefined references to core found");
diff --git a/library/compiler-builtins/crates/symbol-check/tests/all.rs b/library/compiler-builtins/crates/symbol-check/tests/all.rs
new file mode 100644
index 000000000000..4ad9509726fb
--- /dev/null
+++ b/library/compiler-builtins/crates/symbol-check/tests/all.rs
@@ -0,0 +1,124 @@
+use std::env;
+use std::ffi::OsString;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::LazyLock;
+
+use assert_cmd::assert::Assert;
+use assert_cmd::cargo::cargo_bin_cmd;
+use tempfile::tempdir;
+
+trait AssertExt {
+    fn stderr_contains(self, s: &str) -> Self;
+}
+
+impl AssertExt for Assert {
+    fn stderr_contains(self, s: &str) -> Self {
+        let out = String::from_utf8_lossy(&self.get_output().stderr);
+        assert!(out.contains(s), "looking for: `{s}`\nout:\n```\n{out}\n```");
+        self
+    }
+}
+
+#[test]
+fn test_duplicates() {
+    let dir = tempdir().unwrap();
+    let dup_out = dir.path().join("dup.o");
+    let lib_out = dir.path().join("libfoo.rlib");
+
+    // For the "bad" file, we need duplicate symbols from different object files in the archive. Do
+    // this reliably by building an archive and a separate object file then merging them.
+    rustc_build(&input_dir().join("duplicates.rs"), &lib_out, |cmd| cmd);
+    rustc_build(&input_dir().join("duplicates.rs"), &dup_out, |cmd| {
+        cmd.arg("--emit=obj")
+    });
+
+    let mut ar = cc_build().get_archiver();
+
+    if ar.get_program().to_string_lossy().contains("lib.exe") {
+        let mut out_arg = OsString::from("-out:");
+        out_arg.push(&lib_out);
+        ar.arg(&out_arg);
+        // Repeating the same file as the first arg makes lib.exe append (taken from the
+        // `cc` implementation).
+        ar.arg(&lib_out);
+    } else {
+        ar.arg("rs")
+            // Eat an `libfoo.rlib(lib.rmeta) has no symbols` info message on MacOS
+            .stderr(Stdio::null())
+            .arg(&lib_out);
+    }
+    let status = ar.arg(&dup_out).status().unwrap();
+    assert!(status.success());
+
+    let assert = cargo_bin_cmd!().arg("check").arg(&lib_out).assert();
+    assert
+        .failure()
+        .stderr_contains("duplicate symbols")
+        .stderr_contains("FDUP")
+        .stderr_contains("IDUP")
+        .stderr_contains("fndup");
+}
+
+#[test]
+fn test_core_symbols() {
+    let dir = tempdir().unwrap();
+    let lib_out = dir.path().join("libfoo.rlib");
+    rustc_build(&input_dir().join("core_symbols.rs"), &lib_out, |cmd| cmd);
+    let assert = cargo_bin_cmd!().arg("check").arg(&lib_out).assert();
+    // FIXME(symcheck): this should fail but we don't detect the new mangling.
+    assert.success();
+}
+
+#[test]
+fn test_good() {
+    let dir = tempdir().unwrap();
+    let lib_out = dir.path().join("libfoo.rlib");
+    rustc_build(&input_dir().join("good.rs"), &lib_out, |cmd| cmd);
+    let assert = cargo_bin_cmd!().arg("check").arg(&lib_out).assert();
+    assert.success();
+}
+
+/// Build i -> o with optional additional configuration.
+fn rustc_build(i: &Path, o: &Path, mut f: impl FnMut(&mut Command) -> &mut Command) {
+    let mut cmd = Command::new("rustc");
+    cmd.arg(i)
+        .arg("--target")
+        .arg(target())
+        .arg("--crate-type=lib")
+        .arg("-o")
+        .arg(o);
+    f(&mut cmd);
+    let status = cmd.status().unwrap();
+    assert!(status.success());
+}
+
+/// Configure `cc` with the host and target.
+fn cc_build() -> cc::Build {
+    let mut b = cc::Build::new();
+    b.host(env!("HOST")).target(&target());
+    b
+}
+
+/// Symcheck runs on the host but we want to verify that we find issues on all targets, so
+/// the cross target may be specified.
+fn target() -> String {
+    static TARGET: LazyLock = LazyLock::new(|| {
+        let target = match env::var("SYMCHECK_TEST_TARGET") {
+            Ok(t) => t,
+            // Require on CI so we don't accidentally always test the native target
+            _ if env::var("CI").is_ok() => panic!("SYMCHECK_TEST_TARGET must be set in CI"),
+            // Fall back to native for local convenience.
+            Err(_) => env!("HOST").to_string(),
+        };
+
+        println!("using target {target}");
+        target
+    });
+
+    TARGET.clone()
+}
+
+fn input_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/input")
+}
diff --git a/library/compiler-builtins/crates/symbol-check/tests/input/core_symbols.rs b/library/compiler-builtins/crates/symbol-check/tests/input/core_symbols.rs
new file mode 100644
index 000000000000..cf74f9779661
--- /dev/null
+++ b/library/compiler-builtins/crates/symbol-check/tests/input/core_symbols.rs
@@ -0,0 +1,11 @@
+//! Ensure we catch calls to `core`.
+
+#![no_std]
+
+#[unsafe(no_mangle)]
+pub fn call_from_core(s: &[u8]) -> &str {
+    match core::str::from_utf8(&s) {
+        Ok(s) => s,
+        Err(_) => "",
+    }
+}
diff --git a/library/compiler-builtins/crates/symbol-check/tests/input/duplicates.rs b/library/compiler-builtins/crates/symbol-check/tests/input/duplicates.rs
new file mode 100644
index 000000000000..92623a0b2e1a
--- /dev/null
+++ b/library/compiler-builtins/crates/symbol-check/tests/input/duplicates.rs
@@ -0,0 +1,12 @@
+//! Ensure we catch duplicate symbols (the duplicates are in the aux file). Gets built twice
+//! as separate object files.
+
+#![no_std]
+
+#[unsafe(no_mangle)]
+static IDUP: i32 = 0;
+#[unsafe(no_mangle)]
+static FDUP: f32 = 0.0;
+
+#[unsafe(no_mangle)]
+pub fn fndup() {}
diff --git a/library/compiler-builtins/crates/symbol-check/tests/input/good.rs b/library/compiler-builtins/crates/symbol-check/tests/input/good.rs
new file mode 100644
index 000000000000..6679ee1dd30c
--- /dev/null
+++ b/library/compiler-builtins/crates/symbol-check/tests/input/good.rs
@@ -0,0 +1,4 @@
+#![no_std]
+
+#[unsafe(no_mangle)]
+pub fn good() {}

From 0de74158757dcb2db583c27b412f14391ae9ce0c Mon Sep 17 00:00:00 2001
From: Yuki Okushi 
Date: Sat, 7 Feb 2026 19:44:34 +0900
Subject: [PATCH 1398/1843] borrowck: suggest `&mut *x` for pattern reborrows

---
 .../src/diagnostics/mutability_errors.rs      | 73 +++++++++++++++++--
 tests/ui/borrowck/mut-borrow-of-mut-ref.fixed | 37 ++++++++++
 tests/ui/borrowck/mut-borrow-of-mut-ref.rs    |  3 +-
 .../ui/borrowck/mut-borrow-of-mut-ref.stderr  | 16 +---
 .../reborrow-in-match-suggest-deref.fixed     | 23 ++++++
 .../reborrow-in-match-suggest-deref.rs        | 23 ++++++
 .../reborrow-in-match-suggest-deref.stderr    | 23 ++++++
 7 files changed, 175 insertions(+), 23 deletions(-)
 create mode 100644 tests/ui/borrowck/mut-borrow-of-mut-ref.fixed
 create mode 100644 tests/ui/borrowck/reborrow-in-match-suggest-deref.fixed
 create mode 100644 tests/ui/borrowck/reborrow-in-match-suggest-deref.rs
 create mode 100644 tests/ui/borrowck/reborrow-in-match-suggest-deref.stderr

diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index 4fb739403ced..07d2313cc7fd 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -354,14 +354,71 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                         self.infcx.tcx.sess.source_map().span_to_snippet(source_info.span)
                     {
                         if snippet.starts_with("&mut ") {
-                            // We don't have access to the HIR to get accurate spans, but we can
-                            // give a best effort structured suggestion.
-                            err.span_suggestion_verbose(
-                                source_info.span.with_hi(source_info.span.lo() + BytePos(5)),
-                                "if there is only one mutable reborrow, remove the `&mut`",
-                                "",
-                                Applicability::MaybeIncorrect,
-                            );
+                            // In calls, `&mut &mut T` may be deref-coerced to `&mut T`, and
+                            // removing the extra `&mut` is the most direct suggestion. But for
+                            // pattern-matching expressions (`match`, `if let`, `while let`), that
+                            // can easily turn into a move, so prefer suggesting an explicit
+                            // reborrow via `&mut *x` instead.
+                            let mut in_pat_scrutinee = false;
+                            let mut is_deref_coerced = false;
+                            if let Some(expr) = self.find_expr(source_info.span) {
+                                let tcx = self.infcx.tcx;
+                                let span = expr.span.source_callsite();
+                                for (_, node) in tcx.hir_parent_iter(expr.hir_id) {
+                                    if let Node::Expr(parent_expr) = node {
+                                        match parent_expr.kind {
+                                            ExprKind::Match(scrutinee, ..)
+                                                if scrutinee
+                                                    .span
+                                                    .source_callsite()
+                                                    .contains(span) =>
+                                            {
+                                                in_pat_scrutinee = true;
+                                                break;
+                                            }
+                                            ExprKind::Let(let_expr)
+                                                if let_expr
+                                                    .init
+                                                    .span
+                                                    .source_callsite()
+                                                    .contains(span) =>
+                                            {
+                                                in_pat_scrutinee = true;
+                                                break;
+                                            }
+                                            _ => {}
+                                        }
+                                    }
+                                }
+
+                                let typeck = tcx.typeck(expr.hir_id.owner.def_id);
+                                is_deref_coerced =
+                                    typeck.expr_adjustments(expr).iter().any(|adj| {
+                                        matches!(adj.kind, ty::adjustment::Adjust::Deref(_))
+                                    });
+                            }
+
+                            if in_pat_scrutinee {
+                                // Best-effort structured suggestion: insert `*` after `&mut `.
+                                err.span_suggestion_verbose(
+                                    source_info
+                                        .span
+                                        .with_lo(source_info.span.lo() + BytePos(5))
+                                        .shrink_to_lo(),
+                                    "to reborrow the mutable reference, add `*`",
+                                    "*",
+                                    Applicability::MaybeIncorrect,
+                                );
+                            } else if is_deref_coerced {
+                                // We don't have access to the HIR to get accurate spans, but we
+                                // can give a best effort structured suggestion.
+                                err.span_suggestion_verbose(
+                                    source_info.span.with_hi(source_info.span.lo() + BytePos(5)),
+                                    "if there is only one mutable reborrow, remove the `&mut`",
+                                    "",
+                                    Applicability::MaybeIncorrect,
+                                );
+                            }
                         } else {
                             // This can occur with things like `(&mut self).foo()`.
                             err.span_help(source_info.span, "try removing `&mut` here");
diff --git a/tests/ui/borrowck/mut-borrow-of-mut-ref.fixed b/tests/ui/borrowck/mut-borrow-of-mut-ref.fixed
new file mode 100644
index 000000000000..f70983c65223
--- /dev/null
+++ b/tests/ui/borrowck/mut-borrow-of-mut-ref.fixed
@@ -0,0 +1,37 @@
+// Suggest not mutably borrowing a mutable reference
+//@ run-rustfix
+#![crate_type = "rlib"]
+
+pub fn f(mut b: &mut i32) {
+    //~^ ERROR: cannot borrow
+    //~| NOTE: not mutable
+    //~| NOTE: the binding is already a mutable borrow
+    //~| HELP: consider making the binding mutable if you need to reborrow multiple times
+    h(b);
+    //~^ NOTE: cannot borrow as mutable
+    //~| HELP: if there is only one mutable reborrow, remove the `&mut`
+    g(&mut &mut b);
+    //~^ NOTE: cannot borrow as mutable
+}
+
+pub fn g(mut b: &mut i32) { //~ NOTE: the binding is already a mutable borrow
+    //~^ HELP: consider making the binding mutable if you need to reborrow multiple times
+    h(&mut &mut b);
+    //~^ ERROR: cannot borrow
+    //~| NOTE: cannot borrow as mutable
+}
+
+pub fn h(_: &mut i32) {}
+
+trait Foo {
+    fn bar(&mut self);
+}
+
+impl Foo for &mut String {
+    fn bar(&mut self) {}
+}
+
+pub fn baz(mut f: &mut String) { //~ HELP consider making the binding mutable
+    f.bar(); //~ ERROR cannot borrow `f` as mutable, as it is not declared as mutable
+    //~^ NOTE cannot borrow as mutable
+}
diff --git a/tests/ui/borrowck/mut-borrow-of-mut-ref.rs b/tests/ui/borrowck/mut-borrow-of-mut-ref.rs
index 33954592bf65..d150b249bf2d 100644
--- a/tests/ui/borrowck/mut-borrow-of-mut-ref.rs
+++ b/tests/ui/borrowck/mut-borrow-of-mut-ref.rs
@@ -1,4 +1,5 @@
 // Suggest not mutably borrowing a mutable reference
+//@ run-rustfix
 #![crate_type = "rlib"]
 
 pub fn f(b: &mut i32) {
@@ -11,7 +12,6 @@ pub fn f(b: &mut i32) {
     //~| HELP: if there is only one mutable reborrow, remove the `&mut`
     g(&mut &mut b);
     //~^ NOTE: cannot borrow as mutable
-    //~| HELP: if there is only one mutable reborrow, remove the `&mut`
 }
 
 pub fn g(b: &mut i32) { //~ NOTE: the binding is already a mutable borrow
@@ -19,7 +19,6 @@ pub fn g(b: &mut i32) { //~ NOTE: the binding is already a mutable borrow
     h(&mut &mut b);
     //~^ ERROR: cannot borrow
     //~| NOTE: cannot borrow as mutable
-    //~| HELP: if there is only one mutable reborrow, remove the `&mut`
 }
 
 pub fn h(_: &mut i32) {}
diff --git a/tests/ui/borrowck/mut-borrow-of-mut-ref.stderr b/tests/ui/borrowck/mut-borrow-of-mut-ref.stderr
index 4c4a5e718393..01ceba90c943 100644
--- a/tests/ui/borrowck/mut-borrow-of-mut-ref.stderr
+++ b/tests/ui/borrowck/mut-borrow-of-mut-ref.stderr
@@ -1,5 +1,5 @@
 error[E0596]: cannot borrow `b` as mutable, as it is not declared as mutable
-  --> $DIR/mut-borrow-of-mut-ref.rs:4:10
+  --> $DIR/mut-borrow-of-mut-ref.rs:5:10
    |
 LL | pub fn f(b: &mut i32) {
    |          ^ not mutable
@@ -11,7 +11,7 @@ LL |     g(&mut &mut b);
    |            ------ cannot borrow as mutable
    |
 note: the binding is already a mutable borrow
-  --> $DIR/mut-borrow-of-mut-ref.rs:4:13
+  --> $DIR/mut-borrow-of-mut-ref.rs:5:13
    |
 LL | pub fn f(b: &mut i32) {
    |             ^^^^^^^^
@@ -24,11 +24,6 @@ help: if there is only one mutable reborrow, remove the `&mut`
 LL -     h(&mut b);
 LL +     h(b);
    |
-help: if there is only one mutable reborrow, remove the `&mut`
-   |
-LL -     g(&mut &mut b);
-LL +     g(&mut b);
-   |
 
 error[E0596]: cannot borrow `b` as mutable, as it is not declared as mutable
   --> $DIR/mut-borrow-of-mut-ref.rs:19:12
@@ -45,14 +40,9 @@ help: consider making the binding mutable if you need to reborrow multiple times
    |
 LL | pub fn g(mut b: &mut i32) {
    |          +++
-help: if there is only one mutable reborrow, remove the `&mut`
-   |
-LL -     h(&mut &mut b);
-LL +     h(&mut b);
-   |
 
 error[E0596]: cannot borrow `f` as mutable, as it is not declared as mutable
-  --> $DIR/mut-borrow-of-mut-ref.rs:36:5
+  --> $DIR/mut-borrow-of-mut-ref.rs:35:5
    |
 LL |     f.bar();
    |     ^ cannot borrow as mutable
diff --git a/tests/ui/borrowck/reborrow-in-match-suggest-deref.fixed b/tests/ui/borrowck/reborrow-in-match-suggest-deref.fixed
new file mode 100644
index 000000000000..3f52e57a804c
--- /dev/null
+++ b/tests/ui/borrowck/reborrow-in-match-suggest-deref.fixed
@@ -0,0 +1,23 @@
+// Regression test for #81059.
+// edition:2024
+//@ run-rustfix
+
+#![allow(unused)]
+
+fn test(mut outer: &mut Option) {
+    //~^ NOTE: the binding is already a mutable borrow
+    //~| HELP: consider making the binding mutable if you need to reborrow multiple times
+    match (&mut *outer, 23) {
+        //~^ ERROR: cannot borrow `outer` as mutable, as it is not declared as mutable
+        //~| NOTE: cannot borrow as mutable
+        //~| HELP: to reborrow the mutable reference, add `*`
+        (Some(inner), _) => {
+            *inner = 17;
+        }
+        _ => {
+            *outer = Some(2);
+        }
+    }
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/reborrow-in-match-suggest-deref.rs b/tests/ui/borrowck/reborrow-in-match-suggest-deref.rs
new file mode 100644
index 000000000000..34c299f4f7b1
--- /dev/null
+++ b/tests/ui/borrowck/reborrow-in-match-suggest-deref.rs
@@ -0,0 +1,23 @@
+// Regression test for #81059.
+// edition:2024
+//@ run-rustfix
+
+#![allow(unused)]
+
+fn test(outer: &mut Option) {
+    //~^ NOTE: the binding is already a mutable borrow
+    //~| HELP: consider making the binding mutable if you need to reborrow multiple times
+    match (&mut outer, 23) {
+        //~^ ERROR: cannot borrow `outer` as mutable, as it is not declared as mutable
+        //~| NOTE: cannot borrow as mutable
+        //~| HELP: to reborrow the mutable reference, add `*`
+        (Some(inner), _) => {
+            *inner = 17;
+        }
+        _ => {
+            *outer = Some(2);
+        }
+    }
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/reborrow-in-match-suggest-deref.stderr b/tests/ui/borrowck/reborrow-in-match-suggest-deref.stderr
new file mode 100644
index 000000000000..db3129461518
--- /dev/null
+++ b/tests/ui/borrowck/reborrow-in-match-suggest-deref.stderr
@@ -0,0 +1,23 @@
+error[E0596]: cannot borrow `outer` as mutable, as it is not declared as mutable
+  --> $DIR/reborrow-in-match-suggest-deref.rs:10:12
+   |
+LL |     match (&mut outer, 23) {
+   |            ^^^^^^^^^^ cannot borrow as mutable
+   |
+note: the binding is already a mutable borrow
+  --> $DIR/reborrow-in-match-suggest-deref.rs:7:16
+   |
+LL | fn test(outer: &mut Option) {
+   |                ^^^^^^^^^^^^^^^^
+help: consider making the binding mutable if you need to reborrow multiple times
+   |
+LL | fn test(mut outer: &mut Option) {
+   |         +++
+help: to reborrow the mutable reference, add `*`
+   |
+LL |     match (&mut *outer, 23) {
+   |                 +
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0596`.

From 3ee0286396e9cfacd3c0cb1ce7417b6a7a8e6308 Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Sun, 18 Jan 2026 13:35:42 +0100
Subject: [PATCH 1399/1843] internal: Remove `Edition::CURRENT_FIXME`

---
 src/tools/rust-analyzer/Cargo.toml            |  2 -
 .../bench_data/glorious_old_parser            | 18 +------
 .../rust-analyzer/crates/edition/src/lib.rs   |  2 -
 .../crates/hir/src/term_search/tactics.rs     | 19 +++++--
 .../ide-assists/src/handlers/auto_import.rs   |  1 +
 .../crates/ide-db/src/imports/insert_use.rs   |  9 +++-
 .../ide-db/src/syntax_helpers/suggest_name.rs | 50 +++++++++++--------
 .../rust-analyzer/crates/ide/src/typing.rs    | 18 ++++---
 .../crates/project-model/src/sysroot.rs       |  6 ++-
 .../crates/project-model/src/workspace.rs     |  2 +-
 10 files changed, 70 insertions(+), 57 deletions(-)

diff --git a/src/tools/rust-analyzer/Cargo.toml b/src/tools/rust-analyzer/Cargo.toml
index 2288933a96cc..9f31e1903af5 100644
--- a/src/tools/rust-analyzer/Cargo.toml
+++ b/src/tools/rust-analyzer/Cargo.toml
@@ -192,8 +192,6 @@ unused_lifetimes = "warn"
 unreachable_pub = "warn"
 
 [workspace.lints.clippy]
-# FIXME Remove the tidy test once the lint table is stable
-
 ## lint groups
 complexity = { level = "warn", priority = -1 }
 correctness = { level = "deny", priority = -1 }
diff --git a/src/tools/rust-analyzer/bench_data/glorious_old_parser b/src/tools/rust-analyzer/bench_data/glorious_old_parser
index f593f2b2955a..8136daa8329f 100644
--- a/src/tools/rust-analyzer/bench_data/glorious_old_parser
+++ b/src/tools/rust-analyzer/bench_data/glorious_old_parser
@@ -724,7 +724,7 @@ impl<'a> Parser<'a> {
                 //  {foo(bar {}}
                 //      -      ^
                 //      |      |
-                //      |      help: `)` may belong here (FIXME: #58270)
+                //      |      help: `)` may belong here
                 //      |
                 //      unclosed delimiter
                 if let Some(sp) = unmatched.unclosed_span {
@@ -3217,7 +3217,6 @@ impl<'a> Parser<'a> {
 
                   }
                   _ => {
-                    // FIXME Could factor this out into non_fatal_unexpected or something.
                     let actual = self.this_token_to_string();
                     self.span_err(self.span, &format!("unexpected token: `{}`", actual));
                   }
@@ -5250,7 +5249,6 @@ impl<'a> Parser<'a> {
                 }
             }
         } else {
-            // FIXME: Bad copy of attrs
             let old_directory_ownership =
                 mem::replace(&mut self.directory.ownership, DirectoryOwnership::UnownedViaBlock);
             let item = self.parse_item_(attrs.clone(), false, true)?;
@@ -5953,23 +5951,14 @@ impl<'a> Parser<'a> {
                 });
                 assoc_ty_bindings.push(span);
             } else if self.check_const_arg() {
-                // FIXME(const_generics): to distinguish between idents for types and consts,
-                // we should introduce a GenericArg::Ident in the AST and distinguish when
-                // lowering to the HIR. For now, idents for const args are not permitted.
-
                 // Parse const argument.
                 let expr = if let token::OpenDelim(token::Brace) = self.token {
                     self.parse_block_expr(None, self.span, BlockCheckMode::Default, ThinVec::new())?
                 } else if self.token.is_ident() {
-                    // FIXME(const_generics): to distinguish between idents for types and consts,
-                    // we should introduce a GenericArg::Ident in the AST and distinguish when
-                    // lowering to the HIR. For now, idents for const args are not permitted.
                     return Err(
                         self.fatal("identifiers may currently not be used for const generics")
                     );
                 } else {
-                    // FIXME(const_generics): this currently conflicts with emplacement syntax
-                    // with negative integer literals.
                     self.parse_literal_maybe_minus()?
                 };
                 let value = AnonConst {
@@ -5991,9 +5980,6 @@ impl<'a> Parser<'a> {
             }
         }
 
-        // FIXME: we would like to report this in ast_validation instead, but we currently do not
-        // preserve ordering of generic parameters with respect to associated type binding, so we
-        // lose that information after parsing.
         if misplaced_assoc_ty_bindings.len() > 0 {
             let mut err = self.struct_span_err(
                 args_lo.to(self.prev_span),
@@ -6079,8 +6065,6 @@ impl<'a> Parser<'a> {
                             bounds,
                         }
                     ));
-                // FIXME: Decide what should be used here, `=` or `==`.
-                // FIXME: We are just dropping the binders in lifetime_defs on the floor here.
                 } else if self.eat(&token::Eq) || self.eat(&token::EqEq) {
                     let rhs_ty = self.parse_ty()?;
                     where_clause.predicates.push(ast::WherePredicate::EqPredicate(
diff --git a/src/tools/rust-analyzer/crates/edition/src/lib.rs b/src/tools/rust-analyzer/crates/edition/src/lib.rs
index f1a1fe596493..eb4cec39dce0 100644
--- a/src/tools/rust-analyzer/crates/edition/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/edition/src/lib.rs
@@ -16,8 +16,6 @@ impl Edition {
     pub const DEFAULT: Edition = Edition::Edition2015;
     pub const LATEST: Edition = Edition::Edition2024;
     pub const CURRENT: Edition = Edition::Edition2024;
-    /// The current latest stable edition, note this is usually not the right choice in code.
-    pub const CURRENT_FIXME: Edition = Edition::Edition2024;
 
     pub fn from_u32(u32: u32) -> Edition {
         match u32 {
diff --git a/src/tools/rust-analyzer/crates/hir/src/term_search/tactics.rs b/src/tools/rust-analyzer/crates/hir/src/term_search/tactics.rs
index 05a89e76529b..8622aa1378b3 100644
--- a/src/tools/rust-analyzer/crates/hir/src/term_search/tactics.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/term_search/tactics.rs
@@ -18,7 +18,6 @@ use hir_ty::{
 use itertools::Itertools;
 use rustc_hash::FxHashSet;
 use rustc_type_ir::inherent::Ty as _;
-use span::Edition;
 
 use crate::{
     Adt, AssocItem, GenericDef, GenericParam, HasAttrs, HasVisibility, Impl, ModuleDef, ScopeDef,
@@ -367,7 +366,11 @@ pub(super) fn free_function<'a, 'lt, 'db, DB: HirDatabase>(
                         let ret_ty = it.ret_type_with_args(db, generics.iter().cloned());
                         // Filter out private and unsafe functions
                         if !it.is_visible_from(db, module)
-                            || it.is_unsafe_to_call(db, None, Edition::CURRENT_FIXME)
+                            || it.is_unsafe_to_call(
+                                db,
+                                None,
+                                crate::Crate::from(ctx.scope.resolver().krate()).edition(db),
+                            )
                             || it.is_unstable(db)
                             || ctx.config.enable_borrowcheck && ret_ty.contains_reference(db)
                             || ret_ty.is_raw_ptr()
@@ -473,7 +476,11 @@ pub(super) fn impl_method<'a, 'lt, 'db, DB: HirDatabase>(
 
             // Filter out private and unsafe functions
             if !it.is_visible_from(db, module)
-                || it.is_unsafe_to_call(db, None, Edition::CURRENT_FIXME)
+                || it.is_unsafe_to_call(
+                    db,
+                    None,
+                    crate::Crate::from(ctx.scope.resolver().krate()).edition(db),
+                )
                 || it.is_unstable(db)
             {
                 return None;
@@ -667,7 +674,11 @@ pub(super) fn impl_static_method<'a, 'lt, 'db, DB: HirDatabase>(
 
             // Filter out private and unsafe functions
             if !it.is_visible_from(db, module)
-                || it.is_unsafe_to_call(db, None, Edition::CURRENT_FIXME)
+                || it.is_unsafe_to_call(
+                    db,
+                    None,
+                    crate::Crate::from(ctx.scope.resolver().krate()).edition(db),
+                )
                 || it.is_unstable(db)
             {
                 return None;
diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/auto_import.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/auto_import.rs
index cc2bf8174941..2694910aa609 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/auto_import.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/auto_import.rs
@@ -155,6 +155,7 @@ pub(crate) fn auto_import(acc: &mut Assists, ctx: &AssistContext<'_>) -> Option<
                             &scope,
                             mod_path_to_ast(&import_path, edition),
                             &ctx.config.insert_use,
+                            edition,
                         );
                     },
                 );
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/imports/insert_use.rs b/src/tools/rust-analyzer/crates/ide-db/src/imports/insert_use.rs
index 4444ef5d81d5..db1d599d550d 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/imports/insert_use.rs
+++ b/src/tools/rust-analyzer/crates/ide-db/src/imports/insert_use.rs
@@ -146,9 +146,14 @@ pub fn insert_use(scope: &ImportScope, path: ast::Path, cfg: &InsertUseConfig) {
     insert_use_with_alias_option(scope, path, cfg, None);
 }
 
-pub fn insert_use_as_alias(scope: &ImportScope, path: ast::Path, cfg: &InsertUseConfig) {
+pub fn insert_use_as_alias(
+    scope: &ImportScope,
+    path: ast::Path,
+    cfg: &InsertUseConfig,
+    edition: span::Edition,
+) {
     let text: &str = "use foo as _";
-    let parse = syntax::SourceFile::parse(text, span::Edition::CURRENT_FIXME);
+    let parse = syntax::SourceFile::parse(text, edition);
     let node = parse
         .tree()
         .syntax()
diff --git a/src/tools/rust-analyzer/crates/ide-db/src/syntax_helpers/suggest_name.rs b/src/tools/rust-analyzer/crates/ide-db/src/syntax_helpers/suggest_name.rs
index b8b9a7a76816..5d1e876ea298 100644
--- a/src/tools/rust-analyzer/crates/ide-db/src/syntax_helpers/suggest_name.rs
+++ b/src/tools/rust-analyzer/crates/ide-db/src/syntax_helpers/suggest_name.rs
@@ -206,17 +206,18 @@ impl NameGenerator {
         expr: &ast::Expr,
         sema: &Semantics<'_, RootDatabase>,
     ) -> Option {
+        let edition = sema.scope(expr.syntax())?.krate().edition(sema.db);
         // `from_param` does not benefit from stripping it need the largest
         // context possible so we check firstmost
-        if let Some(name) = from_param(expr, sema) {
+        if let Some(name) = from_param(expr, sema, edition) {
             return Some(self.suggest_name(&name));
         }
 
         let mut next_expr = Some(expr.clone());
         while let Some(expr) = next_expr {
-            let name = from_call(&expr)
-                .or_else(|| from_type(&expr, sema))
-                .or_else(|| from_field_name(&expr));
+            let name = from_call(&expr, edition)
+                .or_else(|| from_type(&expr, sema, edition))
+                .or_else(|| from_field_name(&expr, edition));
             if let Some(name) = name {
                 return Some(self.suggest_name(&name));
             }
@@ -270,7 +271,7 @@ impl NameGenerator {
     }
 }
 
-fn normalize(name: &str) -> Option {
+fn normalize(name: &str, edition: syntax::Edition) -> Option {
     let name = to_lower_snake_case(name).to_smolstr();
 
     if USELESS_NAMES.contains(&name.as_str()) {
@@ -281,16 +282,16 @@ fn normalize(name: &str) -> Option {
         return None;
     }
 
-    if !is_valid_name(&name) {
+    if !is_valid_name(&name, edition) {
         return None;
     }
 
     Some(name)
 }
 
-fn is_valid_name(name: &str) -> bool {
+fn is_valid_name(name: &str, edition: syntax::Edition) -> bool {
     matches!(
-        super::LexedStr::single_token(syntax::Edition::CURRENT_FIXME, name),
+        super::LexedStr::single_token(edition, name),
         Some((syntax::SyntaxKind::IDENT, _error))
     )
 }
@@ -304,11 +305,11 @@ fn is_useless_method(method: &ast::MethodCallExpr) -> bool {
     }
 }
 
-fn from_call(expr: &ast::Expr) -> Option {
-    from_func_call(expr).or_else(|| from_method_call(expr))
+fn from_call(expr: &ast::Expr, edition: syntax::Edition) -> Option {
+    from_func_call(expr, edition).or_else(|| from_method_call(expr, edition))
 }
 
-fn from_func_call(expr: &ast::Expr) -> Option {
+fn from_func_call(expr: &ast::Expr, edition: syntax::Edition) -> Option {
     let call = match expr {
         ast::Expr::CallExpr(call) => call,
         _ => return None,
@@ -318,10 +319,10 @@ fn from_func_call(expr: &ast::Expr) -> Option {
         _ => return None,
     };
     let ident = func.path()?.segment()?.name_ref()?.ident_token()?;
-    normalize(ident.text())
+    normalize(ident.text(), edition)
 }
 
-fn from_method_call(expr: &ast::Expr) -> Option {
+fn from_method_call(expr: &ast::Expr, edition: syntax::Edition) -> Option {
     let method = match expr {
         ast::Expr::MethodCallExpr(call) => call,
         _ => return None,
@@ -340,10 +341,14 @@ fn from_method_call(expr: &ast::Expr) -> Option {
         }
     }
 
-    normalize(name)
+    normalize(name, edition)
 }
 
-fn from_param(expr: &ast::Expr, sema: &Semantics<'_, RootDatabase>) -> Option {
+fn from_param(
+    expr: &ast::Expr,
+    sema: &Semantics<'_, RootDatabase>,
+    edition: Edition,
+) -> Option {
     let arg_list = expr.syntax().parent().and_then(ast::ArgList::cast)?;
     let args_parent = arg_list.syntax().parent()?;
     let func = match_ast! {
@@ -362,7 +367,7 @@ fn from_param(expr: &ast::Expr, sema: &Semantics<'_, RootDatabase>) -> Option Option {
@@ -374,10 +379,13 @@ fn var_name_from_pat(pat: &ast::Pat) -> Option {
     }
 }
 
-fn from_type(expr: &ast::Expr, sema: &Semantics<'_, RootDatabase>) -> Option {
+fn from_type(
+    expr: &ast::Expr,
+    sema: &Semantics<'_, RootDatabase>,
+    edition: Edition,
+) -> Option {
     let ty = sema.type_of_expr(expr)?.adjusted();
     let ty = ty.remove_ref().unwrap_or(ty);
-    let edition = sema.scope(expr.syntax())?.krate().edition(sema.db);
 
     name_of_type(&ty, sema.db, edition)
 }
@@ -417,7 +425,7 @@ fn name_of_type<'db>(
     } else {
         return None;
     };
-    normalize(&name)
+    normalize(&name, edition)
 }
 
 fn sequence_name<'db>(
@@ -450,13 +458,13 @@ fn trait_name(trait_: &hir::Trait, db: &RootDatabase, edition: Edition) -> Optio
     Some(name)
 }
 
-fn from_field_name(expr: &ast::Expr) -> Option {
+fn from_field_name(expr: &ast::Expr, edition: syntax::Edition) -> Option {
     let field = match expr {
         ast::Expr::FieldExpr(field) => field,
         _ => return None,
     };
     let ident = field.name_ref()?.ident_token()?;
-    normalize(ident.text())
+    normalize(ident.text(), edition)
 }
 
 #[cfg(test)]
diff --git a/src/tools/rust-analyzer/crates/ide/src/typing.rs b/src/tools/rust-analyzer/crates/ide/src/typing.rs
index 0381865fed45..f8b0dbfe6282 100644
--- a/src/tools/rust-analyzer/crates/ide/src/typing.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/typing.rs
@@ -17,7 +17,10 @@ mod on_enter;
 
 use either::Either;
 use hir::EditionedFileId;
-use ide_db::{FilePosition, RootDatabase, base_db::RootQueryDb};
+use ide_db::{
+    FilePosition, RootDatabase,
+    base_db::{RootQueryDb, SourceDatabase},
+};
 use span::Edition;
 use std::iter;
 
@@ -70,11 +73,12 @@ pub(crate) fn on_char_typed(
     if !TRIGGER_CHARS.contains(&char_typed) {
         return None;
     }
-    // FIXME: We need to figure out the edition of the file here, but that means hitting the
-    // database for more than just parsing the file which is bad.
+    let edition = db
+        .source_root_crates(db.file_source_root(position.file_id).source_root_id(db))
+        .first()
+        .map_or(Edition::CURRENT, |crates| crates.data(db).edition);
     // FIXME: We are hitting the database here, if we are unlucky this call might block momentarily
-    // causing the editor to feel sluggish!
-    let edition = Edition::CURRENT_FIXME;
+    // causing the editor to feel sluggish! We need to make this bail if it would block too long?
     let editioned_file_id_wrapper = EditionedFileId::from_span_guess_origin(
         db,
         span::EditionedFileId::new(position.file_id, edition),
@@ -457,8 +461,8 @@ mod tests {
         let (offset, mut before) = extract_offset(before);
         let edit = TextEdit::insert(offset, char_typed.to_string());
         edit.apply(&mut before);
-        let parse = SourceFile::parse(&before, span::Edition::CURRENT_FIXME);
-        on_char_typed_(&parse, offset, char_typed, span::Edition::CURRENT_FIXME).map(|it| {
+        let parse = SourceFile::parse(&before, span::Edition::CURRENT);
+        on_char_typed_(&parse, offset, char_typed, span::Edition::CURRENT).map(|it| {
             it.apply(&mut before);
             before.to_string()
         })
diff --git a/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs b/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs
index f244c9736c7c..546a1e05a063 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs
@@ -275,7 +275,10 @@ impl Sysroot {
             }
             tracing::debug!("Stitching sysroot library: {src_root}");
 
-            let mut stitched = stitched::Stitched { crates: Default::default() };
+            let mut stitched = stitched::Stitched {
+                crates: Default::default(),
+                edition: span::Edition::Edition2024,
+            };
 
             for path in stitched::SYSROOT_CRATES.trim().lines() {
                 let name = path.split('/').next_back().unwrap();
@@ -511,6 +514,7 @@ pub(crate) mod stitched {
     #[derive(Debug, Clone, Eq, PartialEq)]
     pub struct Stitched {
         pub(super) crates: Arena,
+        pub(crate) edition: span::Edition,
     }
 
     impl ops::Index for Stitched {
diff --git a/src/tools/rust-analyzer/crates/project-model/src/workspace.rs b/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
index 8f15f7e1507c..581b5fa51446 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
@@ -1831,7 +1831,7 @@ fn sysroot_to_crate_graph(
                     let display_name = CrateDisplayName::from_canonical_name(&stitched[krate].name);
                     let crate_id = crate_graph.add_crate_root(
                         file_id,
-                        Edition::CURRENT_FIXME,
+                        stitched.edition,
                         Some(display_name),
                         None,
                         cfg_options.clone(),

From 5768fb7d93414e830c6ca0fbf0f4f8502d2de4f4 Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 05:49:02 -0600
Subject: [PATCH 1400/1843] symcheck: Check for core symbols with the new
 mangling

The recent switch in default mangling meant that the check was no longer
working correctly. Resolve this by checking for both legacy- and
v0-mangled core symbols to the extent that this is possible.
---
 .../compiler-builtins/crates/symbol-check/Cargo.toml |  1 +
 .../crates/symbol-check/src/main.rs                  | 12 +++++++++++-
 .../crates/symbol-check/tests/all.rs                 |  6 ++++--
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/library/compiler-builtins/crates/symbol-check/Cargo.toml b/library/compiler-builtins/crates/symbol-check/Cargo.toml
index 774d1c31a4b0..6291a0ca7f45 100644
--- a/library/compiler-builtins/crates/symbol-check/Cargo.toml
+++ b/library/compiler-builtins/crates/symbol-check/Cargo.toml
@@ -6,6 +6,7 @@ publish = false
 
 [dependencies]
 object = { version = "0.37.3", features = ["wasm"] }
+regex = "1.12.3"
 serde_json = "1.0.149"
 
 [dev-dependencies]
diff --git a/library/compiler-builtins/crates/symbol-check/src/main.rs b/library/compiler-builtins/crates/symbol-check/src/main.rs
index 1413042e6c70..733d9f4e8bef 100644
--- a/library/compiler-builtins/crates/symbol-check/src/main.rs
+++ b/library/compiler-builtins/crates/symbol-check/src/main.rs
@@ -9,12 +9,14 @@ use std::fs;
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
+use std::sync::LazyLock;
 
 use object::read::archive::ArchiveFile;
 use object::{
     File as ObjFile, Object, ObjectSection, ObjectSymbol, Result as ObjResult, Symbol, SymbolKind,
     SymbolScope,
 };
+use regex::Regex;
 use serde_json::Value;
 
 const CHECK_LIBRARIES: &[&str] = &["compiler_builtins", "builtins_test_intrinsics"];
@@ -255,6 +257,14 @@ fn verify_no_duplicates(archive: &BinFile) {
 
 /// Ensure that there are no references to symbols from `core` that aren't also (somehow) defined.
 fn verify_core_symbols(archive: &BinFile) {
+    // Match both mangling styles:
+    //
+    // * `_ZN4core3str8converts9from_utf817hd4454ac14cbbb790E` (old)
+    // * `_RNvNtNtCscK9O3IwVk7N_4core3str8converts9from_utf8` (v0)
+    //
+    // Also account for the Apple leading `_`.
+    static RE: LazyLock = LazyLock::new(|| Regex::new(r"^_?_[RZ].*4core").unwrap());
+
     let mut defined = BTreeSet::new();
     let mut undefined = Vec::new();
     let mut has_symbols = false;
@@ -263,7 +273,7 @@ fn verify_core_symbols(archive: &BinFile) {
         has_symbols = true;
 
         // Find only symbols from `core`
-        if !symbol.name().unwrap().contains("_ZN4core") {
+        if !RE.is_match(symbol.name().unwrap()) {
             return;
         }
 
diff --git a/library/compiler-builtins/crates/symbol-check/tests/all.rs b/library/compiler-builtins/crates/symbol-check/tests/all.rs
index 4ad9509726fb..400469a49e2a 100644
--- a/library/compiler-builtins/crates/symbol-check/tests/all.rs
+++ b/library/compiler-builtins/crates/symbol-check/tests/all.rs
@@ -66,8 +66,10 @@ fn test_core_symbols() {
     let lib_out = dir.path().join("libfoo.rlib");
     rustc_build(&input_dir().join("core_symbols.rs"), &lib_out, |cmd| cmd);
     let assert = cargo_bin_cmd!().arg("check").arg(&lib_out).assert();
-    // FIXME(symcheck): this should fail but we don't detect the new mangling.
-    assert.success();
+    assert
+        .failure()
+        .stderr_contains("found 1 undefined symbols from core")
+        .stderr_contains("from_utf8");
 }
 
 #[test]

From bad1a450c01c285b727e82b116c5baf226ed4f86 Mon Sep 17 00:00:00 2001
From: Folkert de Vries 
Date: Sat, 7 Feb 2026 13:33:25 +0100
Subject: [PATCH 1401/1843] use `test` instead of `unix` to be
 platform-independent

---
 tests/ui/macros/cfg_select.rs     |  6 ++++--
 tests/ui/macros/cfg_select.stderr | 32 ++++++++++++++++++-------------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/tests/ui/macros/cfg_select.rs b/tests/ui/macros/cfg_select.rs
index 09d2c116a86d..0f009756e211 100644
--- a/tests/ui/macros/cfg_select.rs
+++ b/tests/ui/macros/cfg_select.rs
@@ -151,8 +151,10 @@ cfg_select! {
 }
 
 cfg_select! {
-    unix => {}
-    unix => {}
+    test => {}
+    test => {}
+    //~^ WARN unreachable configuration predicate
+    _ => {}
     //~^ WARN unreachable configuration predicate
 }
 
diff --git a/tests/ui/macros/cfg_select.stderr b/tests/ui/macros/cfg_select.stderr
index be238e0a65e5..5878708668d3 100644
--- a/tests/ui/macros/cfg_select.stderr
+++ b/tests/ui/macros/cfg_select.stderr
@@ -1,5 +1,5 @@
 error: none of the predicates in this `cfg_select` evaluated to true
-  --> $DIR/cfg_select.rs:159:1
+  --> $DIR/cfg_select.rs:161:1
    |
 LL | / cfg_select! {
 LL | |
@@ -8,49 +8,49 @@ LL | | }
    | |_^
 
 error: none of the predicates in this `cfg_select` evaluated to true
-  --> $DIR/cfg_select.rs:164:1
+  --> $DIR/cfg_select.rs:166:1
    |
 LL | cfg_select! {}
    | ^^^^^^^^^^^^^^
 
 error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `=>`
-  --> $DIR/cfg_select.rs:168:5
+  --> $DIR/cfg_select.rs:170:5
    |
 LL |     => {}
    |     ^^
 
 error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
-  --> $DIR/cfg_select.rs:173:5
+  --> $DIR/cfg_select.rs:175:5
    |
 LL |     () => {}
    |     ^^ expressions are not allowed here
 
 error[E0539]: malformed `cfg_select` macro input
-  --> $DIR/cfg_select.rs:178:5
+  --> $DIR/cfg_select.rs:180:5
    |
 LL |     "str" => {}
    |     ^^^^^ expected a valid identifier here
 
 error[E0539]: malformed `cfg_select` macro input
-  --> $DIR/cfg_select.rs:183:5
+  --> $DIR/cfg_select.rs:185:5
    |
 LL |     a::b => {}
    |     ^^^^ expected a valid identifier here
 
 error[E0537]: invalid predicate `a`
-  --> $DIR/cfg_select.rs:188:5
+  --> $DIR/cfg_select.rs:190:5
    |
 LL |     a() => {}
    |     ^^^
 
 error: expected one of `(`, `::`, `=>`, or `=`, found `+`
-  --> $DIR/cfg_select.rs:193:7
+  --> $DIR/cfg_select.rs:195:7
    |
 LL |     a + 1 => {}
    |       ^ expected one of `(`, `::`, `=>`, or `=`
 
 error: expected one of `(`, `::`, `=>`, or `=`, found `!`
-  --> $DIR/cfg_select.rs:199:8
+  --> $DIR/cfg_select.rs:201:8
    |
 LL |     cfg!() => {}
    |        ^ expected one of `(`, `::`, `=>`, or `=`
@@ -86,11 +86,17 @@ LL |     _ => {}
 warning: unreachable configuration predicate
   --> $DIR/cfg_select.rs:155:5
    |
-LL |     unix => {}
+LL |     test => {}
    |     ^^^^ this configuration predicate is never reached
 
+warning: unreachable configuration predicate
+  --> $DIR/cfg_select.rs:157:5
+   |
+LL |     _ => {}
+   |     ^ this configuration predicate is never reached
+
 warning: unexpected `cfg` condition name: `a`
-  --> $DIR/cfg_select.rs:193:5
+  --> $DIR/cfg_select.rs:195:5
    |
 LL |     a + 1 => {}
    |     ^ help: found config with similar value: `target_feature = "a"`
@@ -101,7 +107,7 @@ LL |     a + 1 => {}
    = note: `#[warn(unexpected_cfgs)]` on by default
 
 warning: unexpected `cfg` condition name: `cfg`
-  --> $DIR/cfg_select.rs:199:5
+  --> $DIR/cfg_select.rs:201:5
    |
 LL |     cfg!() => {}
    |     ^^^
@@ -109,7 +115,7 @@ LL |     cfg!() => {}
    = help: to expect this configuration use `--check-cfg=cfg(cfg)`
    = note: see  for more information about checking conditional configuration
 
-error: aborting due to 9 previous errors; 6 warnings emitted
+error: aborting due to 9 previous errors; 7 warnings emitted
 
 Some errors have detailed explanations: E0537, E0539.
 For more information about an error, try `rustc --explain E0537`.

From 63eefa9ee992ea62c60d2bf5ba1bdb6fa6d086b8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 7 Feb 2026 12:33:55 +0000
Subject: [PATCH 1402/1843] Bump time from 0.3.44 to 0.3.47

Bumps [time](https://github.com/time-rs/time) from 0.3.44 to 0.3.47.
- [Release notes](https://github.com/time-rs/time/releases)
- [Changelog](https://github.com/time-rs/time/blob/main/CHANGELOG.md)
- [Commits](https://github.com/time-rs/time/compare/v0.3.44...v0.3.47)

---
updated-dependencies:
- dependency-name: time
  dependency-version: 0.3.47
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
---
 src/tools/rust-analyzer/Cargo.lock | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock
index 2cf3e37a43bc..11d41a17b165 100644
--- a/src/tools/rust-analyzer/Cargo.lock
+++ b/src/tools/rust-analyzer/Cargo.lock
@@ -1614,9 +1614,9 @@ dependencies = [
 
 [[package]]
 name = "num-conv"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
 
 [[package]]
 name = "num-traits"
@@ -2914,9 +2914,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.44"
+version = "0.3.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d"
+checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
 dependencies = [
  "deranged",
  "itoa",
@@ -2924,22 +2924,22 @@ dependencies = [
  "num-conv",
  "num_threads",
  "powerfmt",
- "serde",
+ "serde_core",
  "time-core",
  "time-macros",
 ]
 
 [[package]]
 name = "time-core"
-version = "0.1.6"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
+checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
 
 [[package]]
 name = "time-macros"
-version = "0.2.24"
+version = "0.2.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
+checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
 dependencies = [
  "num-conv",
  "time-core",

From fb5a4dca328798ec30fa001aed16f19ceb32e000 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sat, 7 Feb 2026 23:16:12 +1100
Subject: [PATCH 1403/1843] Regression test for `let x = offset_of!(..) else {
 .. }` span

---
 tests/ui/span/let-offset-of.rs     | 19 +++++++++++++++++++
 tests/ui/span/let-offset-of.stderr | 17 +++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 tests/ui/span/let-offset-of.rs
 create mode 100644 tests/ui/span/let-offset-of.stderr

diff --git a/tests/ui/span/let-offset-of.rs b/tests/ui/span/let-offset-of.rs
new file mode 100644
index 000000000000..99b34a192847
--- /dev/null
+++ b/tests/ui/span/let-offset-of.rs
@@ -0,0 +1,19 @@
+#![crate_type = "rlib"]
+//@ edition: 2024
+//@ check-pass
+
+// Using `offset_of` in the RHS of a let-else statement should not produce
+// malformed spans or a blank diagnostic snippet.
+//
+// Regression test for .
+
+fn init_to_offset_of() {
+    use std::mem::offset_of;
+    struct Foo { field: u32 }
+
+    if let x = offset_of!(Foo, field) {}
+    //~^ WARN irrefutable `if let` pattern
+
+    let x = offset_of!(Foo, field) else { return; };
+    //~^ WARN irrefutable `let...else` pattern
+}
diff --git a/tests/ui/span/let-offset-of.stderr b/tests/ui/span/let-offset-of.stderr
new file mode 100644
index 000000000000..a0ef3fcc5ef5
--- /dev/null
+++ b/tests/ui/span/let-offset-of.stderr
@@ -0,0 +1,17 @@
+warning: irrefutable `if let` pattern
+  --> $DIR/let-offset-of.rs:14:8
+   |
+LL |     if let x = offset_of!(Foo, field) {}
+   |        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: this pattern will always match, so the `if let` is useless
+   = help: consider replacing the `if let` with a `let`
+   = note: `#[warn(irrefutable_let_patterns)]` on by default
+
+warning: irrefutable `let...else` pattern
+   |
+   = note: this pattern will always match, so the `else` clause is useless
+   = help: consider removing the `else` clause
+
+warning: 2 warnings emitted
+

From cc3fdc637a6314f69bcc5be0a4c281f285a04bbc Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sat, 7 Feb 2026 22:27:23 +1100
Subject: [PATCH 1404/1843] Avoid a bogus THIR span for `let x =
 offset_of!(..)`

---
 compiler/rustc_mir_build/src/thir/cx/block.rs |  9 +++++++--
 tests/ui/span/let-offset-of.stderr            |  4 ++++
 tests/ui/thir-print/offset_of.stdout          | 18 +++++++++---------
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index fad73a7115b0..b108dff5555f 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -75,8 +75,13 @@ impl<'tcx> ThirBuildCx<'tcx> {
                         debug!(?pattern);
 
                         let span = match local.init {
-                            Some(init) => local.span.with_hi(init.span.hi()),
-                            None => local.span,
+                            Some(init)
+                                if let Some(init_span) =
+                                    init.span.find_ancestor_inside_same_ctxt(local.span) =>
+                            {
+                                local.span.with_hi(init_span.hi())
+                            }
+                            Some(_) | None => local.span,
                         };
                         let stmt = Stmt {
                             kind: StmtKind::Let {
diff --git a/tests/ui/span/let-offset-of.stderr b/tests/ui/span/let-offset-of.stderr
index a0ef3fcc5ef5..6ea9b2194a36 100644
--- a/tests/ui/span/let-offset-of.stderr
+++ b/tests/ui/span/let-offset-of.stderr
@@ -9,6 +9,10 @@ LL |     if let x = offset_of!(Foo, field) {}
    = note: `#[warn(irrefutable_let_patterns)]` on by default
 
 warning: irrefutable `let...else` pattern
+  --> $DIR/let-offset-of.rs:17:5
+   |
+LL |     let x = offset_of!(Foo, field) else { return; };
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: this pattern will always match, so the `else` clause is useless
    = help: consider removing the `else` clause
diff --git a/tests/ui/thir-print/offset_of.stdout b/tests/ui/thir-print/offset_of.stdout
index 5666a5972f37..25c099438754 100644
--- a/tests/ui/thir-print/offset_of.stdout
+++ b/tests/ui/thir-print/offset_of.stdout
@@ -68,7 +68,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::concrete).10)
-                                            span: $DIR/offset_of.rs:37:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:37:5: 37:33 (#0)
                                         }
                                     }
                                     Stmt {
@@ -117,7 +117,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::concrete).20)
-                                            span: $DIR/offset_of.rs:38:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:38:5: 38:33 (#0)
                                         }
                                     }
                                     Stmt {
@@ -166,7 +166,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::concrete).30)
-                                            span: $DIR/offset_of.rs:39:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:39:5: 39:34 (#0)
                                         }
                                     }
                                     Stmt {
@@ -215,7 +215,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::concrete).40)
-                                            span: $DIR/offset_of.rs:40:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:40:5: 40:36 (#0)
                                         }
                                     }
                                     Stmt {
@@ -264,7 +264,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::concrete).50)
-                                            span: $DIR/offset_of.rs:41:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:41:5: 41:36 (#0)
                                         }
                                     }
                                 ]
@@ -864,7 +864,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::generic).12)
-                                            span: $DIR/offset_of.rs:45:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:45:5: 45:37 (#0)
                                         }
                                     }
                                     Stmt {
@@ -913,7 +913,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::generic).24)
-                                            span: $DIR/offset_of.rs:46:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:46:5: 46:37 (#0)
                                         }
                                     }
                                     Stmt {
@@ -962,7 +962,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::generic).36)
-                                            span: $DIR/offset_of.rs:47:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:47:5: 47:37 (#0)
                                         }
                                     }
                                     Stmt {
@@ -1011,7 +1011,7 @@ body:
                                             )
                                             else_block: None
                                             hir_id: HirId(DefId(offset_of::generic).48)
-                                            span: $DIR/offset_of.rs:48:5: 1445:57 (#0)
+                                            span: $DIR/offset_of.rs:48:5: 48:37 (#0)
                                         }
                                     }
                                 ]

From 729f384e486a424555c4ebc186b879146b074fcb Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Sat, 7 Feb 2026 13:55:11 +0100
Subject: [PATCH 1405/1843] Bump salsa

---
 src/tools/rust-analyzer/Cargo.lock            | 12 ++---
 src/tools/rust-analyzer/Cargo.toml            |  4 +-
 .../crates/base-db/src/editioned_file_id.rs   |  2 +-
 .../crates/hir-ty/src/tests/regression.rs     |  1 +
 .../rust-analyzer/crates/span/src/hygiene.rs  | 51 +++++++++----------
 5 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock
index 2cf3e37a43bc..9c98e0502070 100644
--- a/src/tools/rust-analyzer/Cargo.lock
+++ b/src/tools/rust-analyzer/Cargo.lock
@@ -2453,9 +2453,9 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 
 [[package]]
 name = "salsa"
-version = "0.25.2"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e2aa2fca57727371eeafc975acc8e6f4c52f8166a78035543f6ee1c74c2dcc"
+checksum = "f77debccd43ba198e9cee23efd7f10330ff445e46a98a2b107fed9094a1ee676"
 dependencies = [
  "boxcar",
  "crossbeam-queue",
@@ -2478,15 +2478,15 @@ dependencies = [
 
 [[package]]
 name = "salsa-macro-rules"
-version = "0.25.2"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfc2a1e7bf06964105515451d728f2422dedc3a112383324a00b191a5c397a3"
+checksum = "ea07adbf42d91cc076b7daf3b38bc8168c19eb362c665964118a89bc55ef19a5"
 
 [[package]]
 name = "salsa-macros"
-version = "0.25.2"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d844c1aa34946da46af683b5c27ec1088a3d9d84a2b837a108223fd830220e1"
+checksum = "d16d4d8b66451b9c75ddf740b7fc8399bc7b8ba33e854a5d7526d18708f67b05"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/src/tools/rust-analyzer/Cargo.toml b/src/tools/rust-analyzer/Cargo.toml
index 2288933a96cc..694f616d1b0b 100644
--- a/src/tools/rust-analyzer/Cargo.toml
+++ b/src/tools/rust-analyzer/Cargo.toml
@@ -135,13 +135,13 @@ rayon = "1.10.0"
 rowan = "=0.15.17"
 # Ideally we'd not enable the macros feature but unfortunately the `tracked` attribute does not work
 # on impls without it
-salsa = { version = "0.25.2", default-features = false, features = [
+salsa = { version = "0.26", default-features = false, features = [
     "rayon",
     "salsa_unstable",
     "macros",
     "inventory",
 ] }
-salsa-macros = "0.25.2"
+salsa-macros = "0.26"
 semver = "1.0.26"
 serde = { version = "1.0.219" }
 serde_derive = { version = "1.0.219" }
diff --git a/src/tools/rust-analyzer/crates/base-db/src/editioned_file_id.rs b/src/tools/rust-analyzer/crates/base-db/src/editioned_file_id.rs
index 13fb05d56547..dd419f48fc7e 100644
--- a/src/tools/rust-analyzer/crates/base-db/src/editioned_file_id.rs
+++ b/src/tools/rust-analyzer/crates/base-db/src/editioned_file_id.rs
@@ -60,7 +60,7 @@ const _: () = {
         }
     }
 
-    impl zalsa_struct_::HashEqLike for EditionedFileIdData {
+    impl zalsa_::HashEqLike for EditionedFileIdData {
         #[inline]
         fn hash(&self, state: &mut H) {
             Hash::hash(self, state);
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
index 4f1480c39366..716409e16027 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
@@ -2363,6 +2363,7 @@ fn test() {
 }
 "#,
         expect![[r#"
+            46..49 'Foo': Foo
             93..97 'self': Foo
             108..125 '{     ...     }': usize
             118..119 'N': usize
diff --git a/src/tools/rust-analyzer/crates/span/src/hygiene.rs b/src/tools/rust-analyzer/crates/span/src/hygiene.rs
index fe05ef946518..0a81cef52ec5 100644
--- a/src/tools/rust-analyzer/crates/span/src/hygiene.rs
+++ b/src/tools/rust-analyzer/crates/span/src/hygiene.rs
@@ -81,25 +81,24 @@ const _: () = {
     #[derive(Hash)]
     struct StructKey<'db, T0, T1, T2, T3>(T0, T1, T2, T3, std::marker::PhantomData<&'db ()>);
 
-    impl<'db, T0, T1, T2, T3> zalsa_::interned::HashEqLike>
-        for SyntaxContextData
+    impl<'db, T0, T1, T2, T3> zalsa_::HashEqLike> for SyntaxContextData
     where
-        Option: zalsa_::interned::HashEqLike,
-        Transparency: zalsa_::interned::HashEqLike,
-        Edition: zalsa_::interned::HashEqLike,
-        SyntaxContext: zalsa_::interned::HashEqLike,
+        Option: zalsa_::HashEqLike,
+        Transparency: zalsa_::HashEqLike,
+        Edition: zalsa_::HashEqLike,
+        SyntaxContext: zalsa_::HashEqLike,
     {
         fn hash(&self, h: &mut H) {
-            zalsa_::interned::HashEqLike::::hash(&self.outer_expn, &mut *h);
-            zalsa_::interned::HashEqLike::::hash(&self.outer_transparency, &mut *h);
-            zalsa_::interned::HashEqLike::::hash(&self.edition, &mut *h);
-            zalsa_::interned::HashEqLike::::hash(&self.parent, &mut *h);
+            zalsa_::HashEqLike::::hash(&self.outer_expn, &mut *h);
+            zalsa_::HashEqLike::::hash(&self.outer_transparency, &mut *h);
+            zalsa_::HashEqLike::::hash(&self.edition, &mut *h);
+            zalsa_::HashEqLike::::hash(&self.parent, &mut *h);
         }
         fn eq(&self, data: &StructKey<'db, T0, T1, T2, T3>) -> bool {
-            zalsa_::interned::HashEqLike::::eq(&self.outer_expn, &data.0)
-                && zalsa_::interned::HashEqLike::::eq(&self.outer_transparency, &data.1)
-                && zalsa_::interned::HashEqLike::::eq(&self.edition, &data.2)
-                && zalsa_::interned::HashEqLike::::eq(&self.parent, &data.3)
+            zalsa_::HashEqLike::::eq(&self.outer_expn, &data.0)
+                && zalsa_::HashEqLike::::eq(&self.outer_transparency, &data.1)
+                && zalsa_::HashEqLike::::eq(&self.edition, &data.2)
+                && zalsa_::HashEqLike::::eq(&self.parent, &data.3)
         }
     }
     impl zalsa_struct_::Configuration for SyntaxContext {
@@ -203,10 +202,10 @@ const _: () = {
     impl<'db> SyntaxContext {
         pub fn new<
             Db,
-            T0: zalsa_::interned::Lookup> + std::hash::Hash,
-            T1: zalsa_::interned::Lookup + std::hash::Hash,
-            T2: zalsa_::interned::Lookup + std::hash::Hash,
-            T3: zalsa_::interned::Lookup + std::hash::Hash,
+            T0: zalsa_::Lookup> + std::hash::Hash,
+            T1: zalsa_::Lookup + std::hash::Hash,
+            T2: zalsa_::Lookup + std::hash::Hash,
+            T3: zalsa_::Lookup + std::hash::Hash,
         >(
             db: &'db Db,
             outer_expn: T0,
@@ -218,10 +217,10 @@ const _: () = {
         ) -> Self
         where
             Db: ?Sized + salsa::Database,
-            Option: zalsa_::interned::HashEqLike,
-            Transparency: zalsa_::interned::HashEqLike,
-            Edition: zalsa_::interned::HashEqLike,
-            SyntaxContext: zalsa_::interned::HashEqLike,
+            Option: zalsa_::HashEqLike,
+            Transparency: zalsa_::HashEqLike,
+            Edition: zalsa_::HashEqLike,
+            SyntaxContext: zalsa_::HashEqLike,
         {
             let (zalsa, zalsa_local) = db.zalsas();
 
@@ -236,10 +235,10 @@ const _: () = {
                     std::marker::PhantomData,
                 ),
                 |id, data| SyntaxContextData {
-                    outer_expn: zalsa_::interned::Lookup::into_owned(data.0),
-                    outer_transparency: zalsa_::interned::Lookup::into_owned(data.1),
-                    edition: zalsa_::interned::Lookup::into_owned(data.2),
-                    parent: zalsa_::interned::Lookup::into_owned(data.3),
+                    outer_expn: zalsa_::Lookup::into_owned(data.0),
+                    outer_transparency: zalsa_::Lookup::into_owned(data.1),
+                    edition: zalsa_::Lookup::into_owned(data.2),
+                    parent: zalsa_::Lookup::into_owned(data.3),
                     opaque: opaque(zalsa_::FromId::from_id(id)),
                     opaque_and_semiopaque: opaque_and_semiopaque(zalsa_::FromId::from_id(id)),
                 },

From 84c2a531ac6db87db84e0f422a353a79af912694 Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Thu, 1 Jan 2026 11:46:49 +0100
Subject: [PATCH 1406/1843] Correctly implement client side request
 cancellation support

---
 src/tools/rust-analyzer/Cargo.lock              | 12 ++++++++++++
 src/tools/rust-analyzer/crates/ide/src/lib.rs   |  6 +++++-
 .../crates/rust-analyzer/src/global_state.rs    |  8 +++++++-
 .../rust-analyzer/src/handlers/dispatch.rs      | 17 ++++++++++++++++-
 4 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock
index 9c98e0502070..12a6c09b8dfb 100644
--- a/src/tools/rust-analyzer/Cargo.lock
+++ b/src/tools/rust-analyzer/Cargo.lock
@@ -3754,3 +3754,15 @@ dependencies = [
  "memchr",
  "time",
 ]
+
+[[patch.unused]]
+name = "salsa"
+version = "0.25.2"
+
+[[patch.unused]]
+name = "salsa-macro-rules"
+version = "0.25.2"
+
+[[patch.unused]]
+name = "salsa-macros"
+version = "0.25.2"
diff --git a/src/tools/rust-analyzer/crates/ide/src/lib.rs b/src/tools/rust-analyzer/crates/ide/src/lib.rs
index 930eaf2262d9..2e618550f92f 100644
--- a/src/tools/rust-analyzer/crates/ide/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/lib.rs
@@ -67,7 +67,7 @@ use ide_db::{
     FxHashMap, FxIndexSet, LineIndexDatabase,
     base_db::{
         CrateOrigin, CrateWorkspaceData, Env, FileSet, RootQueryDb, SourceDatabase, VfsPath,
-        salsa::{Cancelled, Database},
+        salsa::{CancellationToken, Cancelled, Database},
     },
     prime_caches, symbol_index,
 };
@@ -947,6 +947,10 @@ impl Analysis {
         // We use `attach_db_allow_change()` and not `attach_db()` because fixture injection can change the database.
         hir::attach_db_allow_change(&self.db, || Cancelled::catch(|| f(&self.db)))
     }
+
+    pub fn cancellation_token(&self) -> CancellationToken {
+        self.db.cancellation_token()
+    }
 }
 
 #[test]
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs
index afd4162de622..1462727df4e1 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs
@@ -14,7 +14,7 @@ use hir::ChangeWithProcMacros;
 use ide::{Analysis, AnalysisHost, Cancellable, FileId, SourceRootId};
 use ide_db::{
     MiniCore,
-    base_db::{Crate, ProcMacroPaths, SourceDatabase, salsa::Revision},
+    base_db::{Crate, ProcMacroPaths, SourceDatabase, salsa::CancellationToken, salsa::Revision},
 };
 use itertools::Itertools;
 use load_cargo::SourceRootConfig;
@@ -88,6 +88,7 @@ pub(crate) struct GlobalState {
     pub(crate) task_pool: Handle, Receiver>,
     pub(crate) fmt_pool: Handle, Receiver>,
     pub(crate) cancellation_pool: thread::Pool,
+    pub(crate) cancellation_tokens: FxHashMap,
 
     pub(crate) config: Arc,
     pub(crate) config_errors: Option,
@@ -265,6 +266,7 @@ impl GlobalState {
             task_pool,
             fmt_pool,
             cancellation_pool,
+            cancellation_tokens: Default::default(),
             loader,
             config: Arc::new(config.clone()),
             analysis_host,
@@ -617,6 +619,7 @@ impl GlobalState {
     }
 
     pub(crate) fn respond(&mut self, response: lsp_server::Response) {
+        self.cancellation_tokens.remove(&response.id);
         if let Some((method, start)) = self.req_queue.incoming.complete(&response.id) {
             if let Some(err) = &response.error
                 && err.message.starts_with("server panicked")
@@ -631,6 +634,9 @@ impl GlobalState {
     }
 
     pub(crate) fn cancel(&mut self, request_id: lsp_server::RequestId) {
+        if let Some(token) = self.cancellation_tokens.remove(&request_id) {
+            token.cancel();
+        }
         if let Some(response) = self.req_queue.incoming.cancel(request_id) {
             self.send(response.into());
         }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs
index 90deae2d902e..63b4e6430c2d 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs
@@ -253,6 +253,9 @@ impl RequestDispatcher<'_> {
         tracing::debug!(?params);
 
         let world = self.global_state.snapshot();
+        self.global_state
+            .cancellation_tokens
+            .insert(req.id.clone(), world.analysis.cancellation_token());
         if RUSTFMT {
             &mut self.global_state.fmt_pool.handle
         } else {
@@ -265,7 +268,19 @@ impl RequestDispatcher<'_> {
             });
             match thread_result_to_response::(req.id.clone(), result) {
                 Ok(response) => Task::Response(response),
-                Err(_cancelled) if ALLOW_RETRYING => Task::Retry(req),
+                Err(HandlerCancelledError::Inner(
+                    Cancelled::PendingWrite | Cancelled::PropagatedPanic,
+                )) if ALLOW_RETRYING => Task::Retry(req),
+                // Note: Technically the return value here does not matter as we have already responded to the client with this error.
+                Err(HandlerCancelledError::Inner(Cancelled::Local)) => Task::Response(Response {
+                    id: req.id,
+                    result: None,
+                    error: Some(ResponseError {
+                        code: lsp_server::ErrorCode::RequestCanceled as i32,
+                        message: "canceled by client".to_owned(),
+                        data: None,
+                    }),
+                }),
                 Err(_cancelled) => {
                     let error = on_cancelled();
                     Task::Response(Response { id: req.id, result: None, error: Some(error) })

From cee9cc6113b1c5ae3f47a87b73f546099d4b4296 Mon Sep 17 00:00:00 2001
From: Jacob Adam 
Date: Sat, 7 Feb 2026 12:37:00 +0100
Subject: [PATCH 1407/1843] Fix an ICE in the vtable iteration for a trait
 reference in const eval when a supertrait not implemented

compiler/rustc_trait_selection/src/traits/vtable.rs@`vtable_entries`:
The impossible predicates check in `vtable_entries` used
`instantiate_own` which only includes the method's own where-clauses,
not the parent trait's bounds. Replace it with
`instantiate_and_check_impossible_predicates` which also checks the
trait ref itself, so unsatisfied supertrait bounds are caught
and the method is marked `Vacant` instead of ICEing.
---
 compiler/rustc_trait_selection/src/traits/vtable.rs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_trait_selection/src/traits/vtable.rs b/compiler/rustc_trait_selection/src/traits/vtable.rs
index 584c8e2a27c8..bd48068eb579 100644
--- a/compiler/rustc_trait_selection/src/traits/vtable.rs
+++ b/compiler/rustc_trait_selection/src/traits/vtable.rs
@@ -12,7 +12,7 @@ use rustc_span::DUMMY_SP;
 use smallvec::{SmallVec, smallvec};
 use tracing::debug;
 
-use crate::traits::{impossible_predicates, is_vtable_safe_method};
+use crate::traits::is_vtable_safe_method;
 
 #[derive(Clone, Debug)]
 pub enum VtblSegment<'tcx> {
@@ -271,11 +271,7 @@ fn vtable_entries<'tcx>(
                     // do not hold for this particular set of type parameters.
                     // Note that this method could then never be called, so we
                     // do not want to try and codegen it, in that case (see #23435).
-                    let predicates = tcx.predicates_of(def_id).instantiate_own(tcx, args);
-                    if impossible_predicates(
-                        tcx,
-                        predicates.map(|(predicate, _)| predicate).collect(),
-                    ) {
+                    if tcx.instantiate_and_check_impossible_predicates((def_id, args)) {
                         debug!("vtable_entries: predicates do not hold");
                         return VtblEntry::Vacant;
                     }

From 6af91c56d199851abfebbbab4745f34f229649f1 Mon Sep 17 00:00:00 2001
From: Jacob Adam 
Date: Sat, 7 Feb 2026 12:42:59 +0100
Subject: [PATCH 1408/1843] Move tests from tests/crashes to tests/ui now that
 they don't ICE

---
 tests/crashes/137190-2.rs                     | 18 -------------
 tests/crashes/137190-3.rs                     | 10 --------
 .../vtable-impossible-predicates-async.rs}    |  6 ++++-
 .../vtable-unsatisfied-supertrait-generics.rs | 25 +++++++++++++++++++
 ...ble-unsatisfied-supertrait-generics.stderr | 20 +++++++++++++++
 .../coercion/vtable-unsatisfied-supertrait.rs | 16 ++++++++++++
 .../vtable-unsatisfied-supertrait.stderr      | 20 +++++++++++++++
 7 files changed, 86 insertions(+), 29 deletions(-)
 delete mode 100644 tests/crashes/137190-2.rs
 delete mode 100644 tests/crashes/137190-3.rs
 rename tests/{crashes/135470.rs => ui/coercion/vtable-impossible-predicates-async.rs} (79%)
 create mode 100644 tests/ui/coercion/vtable-unsatisfied-supertrait-generics.rs
 create mode 100644 tests/ui/coercion/vtable-unsatisfied-supertrait-generics.stderr
 create mode 100644 tests/ui/coercion/vtable-unsatisfied-supertrait.rs
 create mode 100644 tests/ui/coercion/vtable-unsatisfied-supertrait.stderr

diff --git a/tests/crashes/137190-2.rs b/tests/crashes/137190-2.rs
deleted file mode 100644
index 0c68b5aa4a51..000000000000
--- a/tests/crashes/137190-2.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-//@ known-bug: #137190
-trait Supertrait {
-    fn method(&self) {}
-}
-
-trait Trait
: Supertrait<()> {}
-
-impl
 Trait
 for () {}
-
-const fn upcast
(x: &dyn Trait
) -> &dyn Supertrait<()> {
-    x
-}
-
-const fn foo() -> &'static dyn Supertrait<()> {
-    upcast::<()>(&())
-}
-
-const _: &'static dyn Supertrait<()> = foo();
diff --git a/tests/crashes/137190-3.rs b/tests/crashes/137190-3.rs
deleted file mode 100644
index 88ae88e11bcd..000000000000
--- a/tests/crashes/137190-3.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-//@ known-bug: #137190
-trait Supertrait {
-    fn method(&self) {}
-}
-
-trait Trait: Supertrait {}
-
-impl Trait for () {}
-
-const _: &dyn Supertrait = &() as &dyn Trait as &dyn Supertrait;
diff --git a/tests/crashes/135470.rs b/tests/ui/coercion/vtable-impossible-predicates-async.rs
similarity index 79%
rename from tests/crashes/135470.rs
rename to tests/ui/coercion/vtable-impossible-predicates-async.rs
index efa017b5457c..fe6aa9843fc6 100644
--- a/tests/crashes/135470.rs
+++ b/tests/ui/coercion/vtable-impossible-predicates-async.rs
@@ -1,4 +1,8 @@
-//@ known-bug: #135470
+// Regression test for #135470.
+// Verify that we don't ICE when building vtable entries
+// for a blanket impl involving async and impossible predicates.
+
+//@ check-pass
 //@ compile-flags: -Copt-level=0
 //@ edition: 2021
 
diff --git a/tests/ui/coercion/vtable-unsatisfied-supertrait-generics.rs b/tests/ui/coercion/vtable-unsatisfied-supertrait-generics.rs
new file mode 100644
index 000000000000..41e603fc0a0a
--- /dev/null
+++ b/tests/ui/coercion/vtable-unsatisfied-supertrait-generics.rs
@@ -0,0 +1,25 @@
+// Regression test for #137190.
+// Variant of vtable-unsatisfied-supertrait.rs with generic traits.
+// Verify that we don't ICE when building vtable entries
+// for a generic trait whose supertrait is not implemented.
+
+//@ compile-flags: --crate-type lib
+
+trait Supertrait {
+    fn method(&self) {}
+}
+
+trait Trait
: Supertrait<()> {}
+
+impl
 Trait
 for () {}
+//~^ ERROR the trait bound `(): Supertrait<()>` is not satisfied
+
+const fn upcast
(x: &dyn Trait
) -> &dyn Supertrait<()> {
+    x
+}
+
+const fn foo() -> &'static dyn Supertrait<()> {
+    upcast::<()>(&())
+}
+
+const _: &'static dyn Supertrait<()> = foo();
diff --git a/tests/ui/coercion/vtable-unsatisfied-supertrait-generics.stderr b/tests/ui/coercion/vtable-unsatisfied-supertrait-generics.stderr
new file mode 100644
index 000000000000..a485d2d53953
--- /dev/null
+++ b/tests/ui/coercion/vtable-unsatisfied-supertrait-generics.stderr
@@ -0,0 +1,20 @@
+error[E0277]: the trait bound `(): Supertrait<()>` is not satisfied
+  --> $DIR/vtable-unsatisfied-supertrait-generics.rs:14:22
+   |
+LL | impl
 Trait
 for () {}
+   |                      ^^ the trait `Supertrait<()>` is not implemented for `()`
+   |
+help: this trait has no implementations, consider adding one
+  --> $DIR/vtable-unsatisfied-supertrait-generics.rs:8:1
+   |
+LL | trait Supertrait {
+   | ^^^^^^^^^^^^^^^^^^^
+note: required by a bound in `Trait`
+  --> $DIR/vtable-unsatisfied-supertrait-generics.rs:12:17
+   |
+LL | trait Trait
: Supertrait<()> {}
+   |                 ^^^^^^^^^^^^^^ required by this bound in `Trait`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/coercion/vtable-unsatisfied-supertrait.rs b/tests/ui/coercion/vtable-unsatisfied-supertrait.rs
new file mode 100644
index 000000000000..26a83d2fb066
--- /dev/null
+++ b/tests/ui/coercion/vtable-unsatisfied-supertrait.rs
@@ -0,0 +1,16 @@
+// Regression test for #137190.
+// Verify that we don't ICE when building vtable entries
+// for a trait whose supertrait is not implemented.
+
+//@ compile-flags: --crate-type lib
+
+trait Supertrait {
+    fn method(&self) {}
+}
+
+trait Trait: Supertrait {}
+
+impl Trait for () {}
+//~^ ERROR the trait bound `(): Supertrait` is not satisfied
+
+const _: &dyn Supertrait = &() as &dyn Trait as &dyn Supertrait;
diff --git a/tests/ui/coercion/vtable-unsatisfied-supertrait.stderr b/tests/ui/coercion/vtable-unsatisfied-supertrait.stderr
new file mode 100644
index 000000000000..7df2c95c7fac
--- /dev/null
+++ b/tests/ui/coercion/vtable-unsatisfied-supertrait.stderr
@@ -0,0 +1,20 @@
+error[E0277]: the trait bound `(): Supertrait` is not satisfied
+  --> $DIR/vtable-unsatisfied-supertrait.rs:13:16
+   |
+LL | impl Trait for () {}
+   |                ^^ the trait `Supertrait` is not implemented for `()`
+   |
+help: this trait has no implementations, consider adding one
+  --> $DIR/vtable-unsatisfied-supertrait.rs:7:1
+   |
+LL | trait Supertrait {
+   | ^^^^^^^^^^^^^^^^
+note: required by a bound in `Trait`
+  --> $DIR/vtable-unsatisfied-supertrait.rs:11:14
+   |
+LL | trait Trait: Supertrait {}
+   |              ^^^^^^^^^^ required by this bound in `Trait`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.

From 9249e9f78a7c12f1e797e6519375df698fd96970 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Thu, 5 Feb 2026 14:02:06 +0100
Subject: [PATCH 1409/1843] Port rustc_intrinsic to the new attribute parser

---
 compiler/rustc_ast_lowering/src/item.rs                  | 2 +-
 .../rustc_attr_parsing/src/attributes/rustc_internal.rs  | 9 +++++++++
 compiler/rustc_attr_parsing/src/context.rs               | 1 +
 compiler/rustc_hir/src/attrs/data_structures.rs          | 3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs       | 1 +
 compiler/rustc_middle/src/ty/util.rs                     | 6 ++++--
 compiler/rustc_mir_transform/src/check_inline.rs         | 5 +++--
 compiler/rustc_mir_transform/src/cross_crate_inline.rs   | 7 ++++---
 compiler/rustc_passes/src/check_attr.rs                  | 2 +-
 src/tools/clippy/clippy_lints/src/loops/empty_loop.rs    | 7 ++++---
 10 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index cde8b8f9fe2e..12eaa99143e9 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -1397,7 +1397,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             // create a fake body so that the entire rest of the compiler doesn't have to deal with
             // this as a special case.
             return self.lower_fn_body(decl, contract, |this| {
-                if attrs.iter().any(|a| a.has_name(sym::rustc_intrinsic))
+                if find_attr!(attrs, AttributeKind::RustcIntrinsic)
                     || this.tcx.is_sdylib_interface_build()
                 {
                     let span = this.lower_span(span);
diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 3b7d25b43ba2..637912210638 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -810,3 +810,12 @@ impl SingleAttributeParser for RustcDefPath {
         Some(AttributeKind::RustcDefPath(cx.attr_span))
     }
 }
+
+pub(crate) struct RustcIntrinsicParser;
+
+impl NoArgsAttributeParser for RustcIntrinsicParser {
+    const PATH: &[Symbol] = &[sym::rustc_intrinsic];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcIntrinsic;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 5b489d306499..68c9f232e1b1 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -264,6 +264,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index fa781a74c9e2..1bcf0bf1541e 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1120,6 +1120,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_if_this_changed]`
     RustcIfThisChanged(Span, Option),
 
+    /// Represents `#[rustc_intrinsic]`
+    RustcIntrinsic,
+
     /// Represents `#[rustc_layout]`
     RustcLayout(ThinVec),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 14b54c129d3e..92066c9cfad4 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -118,6 +118,7 @@ impl AttributeKind {
             RustcHasIncoherentInherentImpls => Yes,
             RustcHiddenTypeOfOpaques => No,
             RustcIfThisChanged(..) => No,
+            RustcIntrinsic => Yes,
             RustcLayout(..) => No,
             RustcLayoutScalarValidRangeEnd(..) => Yes,
             RustcLayoutScalarValidRangeStart(..) => Yes,
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index d6e6ffae7300..47b9bb95f830 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -10,11 +10,11 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hashes::Hash128;
-use rustc_hir as hir;
 use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
 use rustc_hir::limit::Limit;
+use rustc_hir::{self as hir, find_attr};
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_macros::{HashStable, TyDecodable, TyEncodable, extension};
 use rustc_span::sym;
@@ -1679,7 +1679,9 @@ pub fn is_doc_notable_trait(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 /// the compiler to make some assumptions about its shape; if the user doesn't use a feature gate, they may
 /// cause an ICE that we otherwise may want to prevent.
 pub fn intrinsic_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option {
-    if tcx.features().intrinsics() && tcx.has_attr(def_id, sym::rustc_intrinsic) {
+    if tcx.features().intrinsics()
+        && find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcIntrinsic)
+    {
         let must_be_overridden = match tcx.hir_node_by_def_id(def_id) {
             hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn { has_body, .. }, .. }) => {
                 !has_body
diff --git a/compiler/rustc_mir_transform/src/check_inline.rs b/compiler/rustc_mir_transform/src/check_inline.rs
index 8d28cb3ca003..2d1876426727 100644
--- a/compiler/rustc_mir_transform/src/check_inline.rs
+++ b/compiler/rustc_mir_transform/src/check_inline.rs
@@ -1,8 +1,9 @@
 //! Check that a body annotated with `#[rustc_force_inline]` will not fail to inline based on its
 //! definition alone (irrespective of any specific caller).
 
-use rustc_hir::attrs::InlineAttr;
+use rustc_hir::attrs::{AttributeKind, InlineAttr};
 use rustc_hir::def_id::DefId;
+use rustc_hir::find_attr;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::{Body, TerminatorKind};
 use rustc_middle::ty;
@@ -62,7 +63,7 @@ pub(super) fn is_inline_valid_on_fn<'tcx>(
     // but at this stage we don't know whether codegen knows the intrinsic,
     // so just conservatively don't inline it. This also ensures that we do not
     // accidentally inline the body of an intrinsic that *must* be overridden.
-    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
+    if find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcIntrinsic) {
         return Err("callee is an intrinsic");
     }
 
diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
index 71bdafa958ca..da133861617f 100644
--- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs
+++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -1,6 +1,7 @@
-use rustc_hir::attrs::InlineAttr;
+use rustc_hir::attrs::{AttributeKind, InlineAttr};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
+use rustc_hir::find_attr;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::*;
@@ -43,7 +44,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         return true;
     }
 
-    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
+    if find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcIntrinsic) {
         // Intrinsic fallback bodies are always cross-crate inlineable.
         // To ensure that the MIR inliner doesn't cluelessly try to inline fallback
         // bodies even when the backend would implement something better, we stop
@@ -157,7 +158,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                 // But intrinsics don't have a body that gets assigned to a CGU, so they are
                 // ignored.
                 if let Some((fn_def_id, _)) = func.const_fn_def()
-                    && self.tcx.has_attr(fn_def_id, sym::rustc_intrinsic)
+                    && find_attr!(tcx.get_all_attrs(fn_def_id), AttributeKind::RustcIntrinsic)
                 {
                     return;
                 }
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 680a93908354..8dcc0eb63c2f 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -313,6 +313,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcHasIncoherentInherentImpls
                     | AttributeKind::RustcHiddenTypeOfOpaques
                     | AttributeKind::RustcIfThisChanged(..)
+                    | AttributeKind::RustcIntrinsic
                     | AttributeKind::RustcLayout(..)
                     | AttributeKind::RustcLayoutScalarValidRangeEnd(..)
                     | AttributeKind::RustcLayoutScalarValidRangeStart(..)
@@ -385,7 +386,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_no_mir_inline
                             | sym::rustc_insignificant_dtor
                             | sym::rustc_nonnull_optimization_guaranteed
-                            | sym::rustc_intrinsic
                             | sym::rustc_inherit_overflow_checks
                             | sym::rustc_intrinsic_const_stable_indirect
                             | sym::rustc_trivial_field_reads
diff --git a/src/tools/clippy/clippy_lints/src/loops/empty_loop.rs b/src/tools/clippy/clippy_lints/src/loops/empty_loop.rs
index e809987d75a0..0a5b26bfa689 100644
--- a/src/tools/clippy/clippy_lints/src/loops/empty_loop.rs
+++ b/src/tools/clippy/clippy_lints/src/loops/empty_loop.rs
@@ -1,8 +1,9 @@
 use super::EMPTY_LOOP;
 use clippy_utils::diagnostics::span_lint_and_help;
-use clippy_utils::{is_in_panic_handler, is_no_std_crate, sym};
+use clippy_utils::{is_in_panic_handler, is_no_std_crate};
 
-use rustc_hir::{Block, Expr, ItemKind, Node};
+use rustc_hir::attrs::AttributeKind;
+use rustc_hir::{Block, Expr, ItemKind, Node, find_attr};
 use rustc_lint::LateContext;
 
 pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, loop_block: &Block<'_>) {
@@ -10,7 +11,7 @@ pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, loop_block: &Block<'_
     if let Node::Item(parent_node) = cx.tcx.hir_node(parent_hir_id)
         && matches!(parent_node.kind, ItemKind::Fn { .. })
         && let attrs = cx.tcx.hir_attrs(parent_hir_id)
-        && attrs.iter().any(|attr| attr.has_name(sym::rustc_intrinsic))
+        && find_attr!(attrs, AttributeKind::RustcIntrinsic)
     {
         // Intrinsic functions are expanded into an empty loop when lowering the AST
         // to simplify the job of later passes which might expect any function to have a body.

From c42a581d2f02da46b77f0c2ff00f6a7e6ff4b524 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Thu, 5 Feb 2026 14:09:40 +0100
Subject: [PATCH 1410/1843] Port rustc_intrinsic_const_stable_indirect to the
 new attribute parser

---
 .../src/attributes/rustc_internal.rs                   | 10 ++++++++++
 compiler/rustc_attr_parsing/src/context.rs             |  1 +
 compiler/rustc_hir/src/attrs/data_structures.rs        |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_middle/src/ty/util.rs                   |  5 ++++-
 compiler/rustc_passes/src/check_attr.rs                |  2 +-
 6 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 637912210638..7e4feedd2548 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -490,6 +490,7 @@ impl CombineAttributeParser for RustcMirParser {
             .collect()
     }
 }
+
 pub(crate) struct RustcNonConstTraitMethodParser;
 
 impl NoArgsAttributeParser for RustcNonConstTraitMethodParser {
@@ -819,3 +820,12 @@ impl NoArgsAttributeParser for RustcIntrinsicParser {
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcIntrinsic;
 }
+
+pub(crate) struct RustcIntrinsicConstStableIndirectParser;
+
+impl NoArgsAttributeParser for RustcIntrinsicConstStableIndirectParser {
+    const PATH: &'static [Symbol] = &[sym::rustc_intrinsic_const_stable_indirect];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcIntrinsicConstStableIndirect;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 68c9f232e1b1..0d594270ba68 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -264,6 +264,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 1bcf0bf1541e..0e6208831190 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1123,6 +1123,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_intrinsic]`
     RustcIntrinsic,
 
+    /// Represents `#[rustc_intrinsic_const_stable_indirect]`
+    RustcIntrinsicConstStableIndirect,
+
     /// Represents `#[rustc_layout]`
     RustcLayout(ThinVec),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 92066c9cfad4..1f322d84fad4 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -119,6 +119,7 @@ impl AttributeKind {
             RustcHiddenTypeOfOpaques => No,
             RustcIfThisChanged(..) => No,
             RustcIntrinsic => Yes,
+            RustcIntrinsicConstStableIndirect => No,
             RustcLayout(..) => No,
             RustcLayoutScalarValidRangeEnd(..) => Yes,
             RustcLayoutScalarValidRangeStart(..) => Yes,
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 47b9bb95f830..e01a5721a5e0 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -1691,7 +1691,10 @@ pub fn intrinsic_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcHiddenTypeOfOpaques
                     | AttributeKind::RustcIfThisChanged(..)
                     | AttributeKind::RustcIntrinsic
+                    | AttributeKind::RustcIntrinsicConstStableIndirect
                     | AttributeKind::RustcLayout(..)
                     | AttributeKind::RustcLayoutScalarValidRangeEnd(..)
                     | AttributeKind::RustcLayoutScalarValidRangeStart(..)
@@ -387,7 +388,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_insignificant_dtor
                             | sym::rustc_nonnull_optimization_guaranteed
                             | sym::rustc_inherit_overflow_checks
-                            | sym::rustc_intrinsic_const_stable_indirect
                             | sym::rustc_trivial_field_reads
                             | sym::rustc_on_unimplemented
                             | sym::rustc_do_not_const_check

From f090e9ce83e0873591e5faca204cfb68da93ff67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Thu, 5 Feb 2026 14:02:06 +0100
Subject: [PATCH 1411/1843] Port rustc_intrinsic to the new attribute parser

---
 clippy_lints/src/loops/empty_loop.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/clippy_lints/src/loops/empty_loop.rs b/clippy_lints/src/loops/empty_loop.rs
index e809987d75a0..0a5b26bfa689 100644
--- a/clippy_lints/src/loops/empty_loop.rs
+++ b/clippy_lints/src/loops/empty_loop.rs
@@ -1,8 +1,9 @@
 use super::EMPTY_LOOP;
 use clippy_utils::diagnostics::span_lint_and_help;
-use clippy_utils::{is_in_panic_handler, is_no_std_crate, sym};
+use clippy_utils::{is_in_panic_handler, is_no_std_crate};
 
-use rustc_hir::{Block, Expr, ItemKind, Node};
+use rustc_hir::attrs::AttributeKind;
+use rustc_hir::{Block, Expr, ItemKind, Node, find_attr};
 use rustc_lint::LateContext;
 
 pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, loop_block: &Block<'_>) {
@@ -10,7 +11,7 @@ pub(super) fn check(cx: &LateContext<'_>, expr: &Expr<'_>, loop_block: &Block<'_
     if let Node::Item(parent_node) = cx.tcx.hir_node(parent_hir_id)
         && matches!(parent_node.kind, ItemKind::Fn { .. })
         && let attrs = cx.tcx.hir_attrs(parent_hir_id)
-        && attrs.iter().any(|attr| attr.has_name(sym::rustc_intrinsic))
+        && find_attr!(attrs, AttributeKind::RustcIntrinsic)
     {
         // Intrinsic functions are expanded into an empty loop when lowering the AST
         // to simplify the job of later passes which might expect any function to have a body.

From fd7ff90f7029a79f2b19f696471c0e675932c283 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sat, 7 Feb 2026 14:28:12 +0100
Subject: [PATCH 1412/1843] Allow only a single accepter per attribute

---
 compiler/rustc_attr_parsing/src/context.rs   | 36 ++++++++++--------
 compiler/rustc_attr_parsing/src/interface.rs | 40 ++++++++++----------
 2 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 5b489d306499..8db561575562 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -1,5 +1,6 @@
 use std::cell::RefCell;
 use std::collections::BTreeMap;
+use std::collections::btree_map::Entry;
 use std::ops::{Deref, DerefMut};
 use std::sync::LazyLock;
 
@@ -61,7 +62,7 @@ use crate::target_checking::AllowedTargets;
 type GroupType = LazyLock>;
 
 pub(super) struct GroupTypeInner {
-    pub(super) accepters: BTreeMap<&'static [Symbol], Vec>>,
+    pub(super) accepters: BTreeMap<&'static [Symbol], GroupTypeInnerAccept>,
 }
 
 pub(super) struct GroupTypeInnerAccept {
@@ -101,7 +102,7 @@ macro_rules! attribute_parsers {
         @[$stage: ty] pub(crate) static $name: ident = [$($names: ty),* $(,)?];
     ) => {
         pub(crate) static $name: GroupType<$stage> = LazyLock::new(|| {
-            let mut accepters = BTreeMap::<_, Vec>>::new();
+            let mut accepters = BTreeMap::<_, GroupTypeInnerAccept<$stage>>::new();
             $(
                 {
                     thread_local! {
@@ -109,19 +110,24 @@ macro_rules! attribute_parsers {
                     };
 
                     for (path, template, accept_fn) in <$names>::ATTRIBUTES {
-                        accepters.entry(*path).or_default().push(GroupTypeInnerAccept {
-                            template: *template,
-                            accept_fn: Box::new(|cx, args| {
-                                STATE_OBJECT.with_borrow_mut(|s| {
-                                    accept_fn(s, cx, args)
-                                })
-                            }),
-                            allowed_targets: <$names as crate::attributes::AttributeParser<$stage>>::ALLOWED_TARGETS,
-                            finalizer: Box::new(|cx| {
-                                let state = STATE_OBJECT.take();
-                                state.finalize(cx)
-                            }),
-                        });
+                        match accepters.entry(*path) {
+                            Entry::Vacant(e) => {
+                                e.insert(GroupTypeInnerAccept {
+                                    template: *template,
+                                    accept_fn: Box::new(|cx, args| {
+                                        STATE_OBJECT.with_borrow_mut(|s| {
+                                            accept_fn(s, cx, args)
+                                        })
+                                    }),
+                                    allowed_targets: <$names as crate::attributes::AttributeParser<$stage>>::ALLOWED_TARGETS,
+                                    finalizer: Box::new(|cx| {
+                                        let state = STATE_OBJECT.take();
+                                        state.finalize(cx)
+                                    })
+                                });
+                            }
+                            Entry::Occupied(_) => panic!("Attribute {path:?} has multiple accepters"),
+                        }
                     }
                 }
             )*
diff --git a/compiler/rustc_attr_parsing/src/interface.rs b/compiler/rustc_attr_parsing/src/interface.rs
index bac4936c20d2..f75f63a0e811 100644
--- a/compiler/rustc_attr_parsing/src/interface.rs
+++ b/compiler/rustc_attr_parsing/src/interface.rs
@@ -327,7 +327,7 @@ impl<'sess, S: Stage> AttributeParser<'sess, S> {
                     let parts =
                         n.item.path.segments.iter().map(|seg| seg.ident.name).collect::>();
 
-                    if let Some(accepts) = S::parsers().accepters.get(parts.as_slice()) {
+                    if let Some(accept) = S::parsers().accepters.get(parts.as_slice()) {
                         let Some(args) = ArgParser::from_attr_args(
                             args,
                             &parts,
@@ -368,28 +368,26 @@ impl<'sess, S: Stage> AttributeParser<'sess, S> {
                             continue;
                         }
 
-                        for accept in accepts {
-                            let mut cx: AcceptContext<'_, 'sess, S> = AcceptContext {
-                                shared: SharedContext {
-                                    cx: self,
-                                    target_span,
-                                    target,
-                                    emit_lint: &mut emit_lint,
-                                },
-                                attr_span,
-                                inner_span: lower_span(n.item.span()),
-                                attr_style: attr.style,
-                                parsed_description: ParsedDescription::Attribute,
-                                template: &accept.template,
-                                attr_path: attr_path.clone(),
-                            };
+                        let mut cx: AcceptContext<'_, 'sess, S> = AcceptContext {
+                            shared: SharedContext {
+                                cx: self,
+                                target_span,
+                                target,
+                                emit_lint: &mut emit_lint,
+                            },
+                            attr_span,
+                            inner_span: lower_span(n.item.span()),
+                            attr_style: attr.style,
+                            parsed_description: ParsedDescription::Attribute,
+                            template: &accept.template,
+                            attr_path: attr_path.clone(),
+                        };
 
-                            (accept.accept_fn)(&mut cx, &args);
-                            finalizers.push(&accept.finalizer);
+                        (accept.accept_fn)(&mut cx, &args);
+                        finalizers.push(&accept.finalizer);
 
-                            if !matches!(cx.stage.should_emit(), ShouldEmit::Nothing) {
-                                Self::check_target(&accept.allowed_targets, target, &mut cx);
-                            }
+                        if !matches!(cx.stage.should_emit(), ShouldEmit::Nothing) {
+                            Self::check_target(&accept.allowed_targets, target, &mut cx);
                         }
                     } else {
                         // If we're here, we must be compiling a tool attribute... Or someone

From 2407f47903869fab5ff713126037c932caf77e3a Mon Sep 17 00:00:00 2001
From: Urgau 
Date: Sat, 7 Feb 2026 14:29:40 +0100
Subject: [PATCH 1413/1843] Also duplicate `#[expect]` attribute in
 `#[derive]`-ed code

---
 .../src/deriving/generic/mod.rs               |  1 +
 .../derive-expect-issue-150553-2.rs           | 14 ++++++++++++
 .../derive-expect-issue-150553-3.rs           | 14 ++++++++++++
 .../derive-expect-issue-150553-3.stderr       | 18 +++++++++++++++
 .../derive-expect-issue-150553.rs             | 22 +++++++++++++++++++
 5 files changed, 69 insertions(+)
 create mode 100644 tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-2.rs
 create mode 100644 tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.rs
 create mode 100644 tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.stderr
 create mode 100644 tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553.rs

diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
index bc86cdbc9f7e..3fe5e89ef06d 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
@@ -540,6 +540,7 @@ impl<'a> TraitDef<'a> {
                         .filter(|a| {
                             a.has_any_name(&[
                                 sym::allow,
+                                sym::expect,
                                 sym::warn,
                                 sym::deny,
                                 sym::forbid,
diff --git a/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-2.rs b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-2.rs
new file mode 100644
index 000000000000..43ee6bf26a69
--- /dev/null
+++ b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-2.rs
@@ -0,0 +1,14 @@
+// Make sure that the copied `#[expect]` attr in the derived code does not trigger an unfulfilled
+// expectation as it's linked to the original one which is fulfilled.
+//
+// See  for rational.
+
+//@ check-pass
+
+#[expect(non_camel_case_types)]
+#[derive(Debug)]
+pub struct SCREAMING_CASE {
+    pub t_ref: i64,
+}
+
+fn main() {}
diff --git a/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.rs b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.rs
new file mode 100644
index 000000000000..904366e6532e
--- /dev/null
+++ b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.rs
@@ -0,0 +1,14 @@
+// Make sure we produce the unfulfilled expectation lint if neither the struct or the
+// derived code fulfilled it.
+
+//@ check-pass
+
+#[expect(unexpected_cfgs)]
+//~^ WARN this lint expectation is unfulfilled
+//~^^ WARN this lint expectation is unfulfilled
+#[derive(Debug)]
+pub struct MyStruct {
+    pub t_ref: i64,
+}
+
+fn main() {}
diff --git a/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.stderr b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.stderr
new file mode 100644
index 000000000000..6478ec435db0
--- /dev/null
+++ b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553-3.stderr
@@ -0,0 +1,18 @@
+warning: this lint expectation is unfulfilled
+  --> $DIR/derive-expect-issue-150553-3.rs:6:10
+   |
+LL | #[expect(unexpected_cfgs)]
+   |          ^^^^^^^^^^^^^^^
+   |
+   = note: `#[warn(unfulfilled_lint_expectations)]` on by default
+
+warning: this lint expectation is unfulfilled
+  --> $DIR/derive-expect-issue-150553-3.rs:6:10
+   |
+LL | #[expect(unexpected_cfgs)]
+   |          ^^^^^^^^^^^^^^^
+   |
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+warning: 2 warnings emitted
+
diff --git a/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553.rs b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553.rs
new file mode 100644
index 000000000000..1752835c8bb8
--- /dev/null
+++ b/tests/ui/lint/rfc-2383-lint-reason/derive-expect-issue-150553.rs
@@ -0,0 +1,22 @@
+// Make sure we properly copy the `#[expect]` attr to the derived code and that no
+// unfulfilled expectations are trigerred.
+//
+// See  for rational.
+
+//@ check-pass
+
+#![deny(redundant_lifetimes)]
+
+use std::fmt::Debug;
+
+#[derive(Debug)]
+#[expect(redundant_lifetimes)]
+pub struct RefWrapper<'a, T>
+where
+    'a: 'static,
+    T: Debug,
+{
+    pub t_ref: &'a T,
+}
+
+fn main() {}

From 0538f7b2ab82f5ea828dec77eeded64f0df24c8f Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 06:33:33 -0600
Subject: [PATCH 1414/1843] meta: Switch to workspace dependencies

We have a handful of repeated dependencies that can be cleaned up, so
change here.
---
 library/compiler-builtins/Cargo.toml          | 33 +++++++++++++++++++
 .../builtins-shim/Cargo.toml                  |  5 ++-
 .../builtins-test/Cargo.toml                  | 21 ++++++------
 .../compiler-builtins/Cargo.toml              |  2 +-
 .../crates/libm-macros/Cargo.toml             |  8 ++---
 .../crates/musl-math-sys/Cargo.toml           |  6 ++--
 .../crates/symbol-check/Cargo.toml            | 12 +++----
 .../compiler-builtins/crates/util/Cargo.toml  | 10 +++---
 .../compiler-builtins/libm-test/Cargo.toml    | 32 +++++++++---------
 library/compiler-builtins/libm/Cargo.toml     |  1 +
 10 files changed, 82 insertions(+), 48 deletions(-)

diff --git a/library/compiler-builtins/Cargo.toml b/library/compiler-builtins/Cargo.toml
index d0eaa16393cd..26f67e02fc52 100644
--- a/library/compiler-builtins/Cargo.toml
+++ b/library/compiler-builtins/Cargo.toml
@@ -31,6 +31,39 @@ exclude = [
     "compiler-builtins",
 ]
 
+[workspace.dependencies]
+anyhow = "1.0.101"
+assert_cmd = "2.1.2"
+cc = "1.2.55"
+compiler_builtins = { path = "builtins-shim", default-features = false }
+criterion = { version = "0.6.0", default-features = false, features = ["cargo_bench_support"] }
+getrandom = "0.3.4"
+gmp-mpfr-sys = { version = "1.6.8", default-features = false }
+gungraun = "0.17.0"
+heck = "0.5.0"
+indicatif = { version = "0.18.3", default-features = false }
+libm = { path = "libm", default-features = false }
+libm-macros = { path = "crates/libm-macros" }
+libm-test = { path = "libm-test", default-features = false }
+libtest-mimic = "0.8.1"
+musl-math-sys = { path = "crates/musl-math-sys" }
+no-panic = "0.1.35"
+object = { version = "0.37.3", features = ["wasm"] }
+panic-handler = { path = "crates/panic-handler" }
+paste = "1.0.15"
+proc-macro2 = "1.0.106"
+quote = "1.0.44"
+rand = "0.9.2"
+rand_chacha = "0.9.0"
+rand_xoshiro = "0.7"
+rayon = "1.11.0"
+regex = "1.12.3"
+rug = { version = "1.28.1", default-features = false, features = ["float", "integer", "std"] }
+rustc_apfloat = "0.2.3"
+serde_json = "1.0.149"
+syn = "2.0.114"
+tempfile = "3.24.0"
+
 [profile.release]
 panic = "abort"
 
diff --git a/library/compiler-builtins/builtins-shim/Cargo.toml b/library/compiler-builtins/builtins-shim/Cargo.toml
index 746d5b21dc3f..37d3407e9f66 100644
--- a/library/compiler-builtins/builtins-shim/Cargo.toml
+++ b/library/compiler-builtins/builtins-shim/Cargo.toml
@@ -7,6 +7,9 @@
 # manifest that is identical except for the `core` dependency and forwards
 # to the same sources, which acts as the `compiler-builtins` Cargo entrypoint
 # for out of tree testing
+#
+# Ideally we can eventually replace this with a patch in the workspace
+# manifest .
 
 [package]
 name = "compiler_builtins"
@@ -33,7 +36,7 @@ doctest = false
 test = false
 
 [build-dependencies]
-cc = { optional = true, version = "1.2" }
+cc = { version = "1.2", optional = true }
 
 [features]
 default = ["compiler-builtins"]
diff --git a/library/compiler-builtins/builtins-test/Cargo.toml b/library/compiler-builtins/builtins-test/Cargo.toml
index 550f736a76db..9395ab1a985e 100644
--- a/library/compiler-builtins/builtins-test/Cargo.toml
+++ b/library/compiler-builtins/builtins-test/Cargo.toml
@@ -6,23 +6,22 @@ publish = false
 license = "MIT AND Apache-2.0 WITH LLVM-exception AND (MIT OR Apache-2.0)"
 
 [dependencies]
+compiler_builtins = { workspace = true, features = ["unstable-public-internals"] }
+
 # For fuzzing tests we want a deterministic seedable RNG. We also eliminate potential
 # problems with system RNGs on the variety of platforms this crate is tested on.
 # `xoshiro128**` is used for its quality, size, and speed at generating `u32` shift amounts.
-rand_xoshiro = "0.7"
-# To compare float builtins against
-rustc_apfloat = "0.2.3"
-# Really a dev dependency, but dev dependencies can't be optional
-gungraun = { version = "0.17.0", optional = true }
+rand_xoshiro.workspace = true
 
-[dependencies.compiler_builtins]
-path = "../builtins-shim"
-default-features = false
-features = ["unstable-public-internals"]
+# To compare float builtins against
+rustc_apfloat.workspace = true
+
+# Really a dev dependency, but dev dependencies can't be optional
+gungraun = { workspace = true, optional = true }
 
 [dev-dependencies]
-criterion = { version = "0.6.0", default-features = false, features = ["cargo_bench_support"] }
-paste = "1.0.15"
+criterion.workspace = true
+paste.workspace = true
 
 [target.'cfg(all(target_arch = "arm", not(any(target_env = "gnu", target_env = "musl")), target_os = "linux"))'.dev-dependencies]
 test = { git = "https://github.com/japaric/utest" }
diff --git a/library/compiler-builtins/compiler-builtins/Cargo.toml b/library/compiler-builtins/compiler-builtins/Cargo.toml
index 496dde2d4cf2..a8b8920421b3 100644
--- a/library/compiler-builtins/compiler-builtins/Cargo.toml
+++ b/library/compiler-builtins/compiler-builtins/Cargo.toml
@@ -31,7 +31,7 @@ doc = false
 core = { path = "../../core", optional = true }
 
 [build-dependencies]
-cc = { optional = true, version = "1.2" }
+cc = { version = "1.2", optional = true }
 
 [features]
 default = ["compiler-builtins"]
diff --git a/library/compiler-builtins/crates/libm-macros/Cargo.toml b/library/compiler-builtins/crates/libm-macros/Cargo.toml
index f6697b783457..f99a92e21c70 100644
--- a/library/compiler-builtins/crates/libm-macros/Cargo.toml
+++ b/library/compiler-builtins/crates/libm-macros/Cargo.toml
@@ -9,10 +9,10 @@ license = "MIT OR Apache-2.0"
 proc-macro = true
 
 [dependencies]
-heck = "0.5.0"
-proc-macro2 = "1.0.106"
-quote = "1.0.44"
-syn = { version = "2.0.114", features = ["full", "extra-traits", "visit-mut"] }
+heck.workspace = true
+proc-macro2.workspace = true
+quote.workspace = true
+syn = { workspace = true, features = ["full", "extra-traits", "visit-mut"] }
 
 [lints.rust]
 # Values used during testing
diff --git a/library/compiler-builtins/crates/musl-math-sys/Cargo.toml b/library/compiler-builtins/crates/musl-math-sys/Cargo.toml
index 60b0647b6dc6..eb97ffbc8669 100644
--- a/library/compiler-builtins/crates/musl-math-sys/Cargo.toml
+++ b/library/compiler-builtins/crates/musl-math-sys/Cargo.toml
@@ -5,10 +5,8 @@ edition = "2024"
 publish = false
 license = "MIT OR Apache-2.0"
 
-[dependencies]
-
 [dev-dependencies]
-libm = { path = "../../libm" }
+libm.workspace = true
 
 [build-dependencies]
-cc = "1.2.55"
+cc.workspace = true
diff --git a/library/compiler-builtins/crates/symbol-check/Cargo.toml b/library/compiler-builtins/crates/symbol-check/Cargo.toml
index 6291a0ca7f45..5bc13d337c27 100644
--- a/library/compiler-builtins/crates/symbol-check/Cargo.toml
+++ b/library/compiler-builtins/crates/symbol-check/Cargo.toml
@@ -5,11 +5,11 @@ edition = "2024"
 publish = false
 
 [dependencies]
-object = { version = "0.37.3", features = ["wasm"] }
-regex = "1.12.3"
-serde_json = "1.0.149"
+object.workspace = true
+regex.workspace = true
+serde_json.workspace = true
 
 [dev-dependencies]
-assert_cmd = "2.1.2"
-cc = "1.2.55"
-tempfile = "3.24.0"
+assert_cmd.workspace = true
+cc.workspace = true
+tempfile.workspace = true
diff --git a/library/compiler-builtins/crates/util/Cargo.toml b/library/compiler-builtins/crates/util/Cargo.toml
index b1ccd8a9e63c..88e0b332065d 100644
--- a/library/compiler-builtins/crates/util/Cargo.toml
+++ b/library/compiler-builtins/crates/util/Cargo.toml
@@ -12,8 +12,8 @@ build-mpfr = ["libm-test/build-mpfr", "dep:rug"]
 unstable-float = ["libm/unstable-float", "libm-test/unstable-float", "rug?/nightly-float"]
 
 [dependencies]
-libm = { path = "../../libm", default-features = false }
-libm-macros = { path = "../libm-macros" }
-libm-test = { path = "../../libm-test", default-features = false }
-musl-math-sys = { path = "../musl-math-sys", optional = true }
-rug = { version = "1.28.1", optional = true, default-features = false, features = ["float", "std"] }
+libm.workspace = true
+libm-macros.workspace = true
+libm-test.workspace = true
+musl-math-sys = { workspace = true, optional = true }
+rug = { workspace = true, optional = true }
diff --git a/library/compiler-builtins/libm-test/Cargo.toml b/library/compiler-builtins/libm-test/Cargo.toml
index b813331a8552..c395d6a21bd5 100644
--- a/library/compiler-builtins/libm-test/Cargo.toml
+++ b/library/compiler-builtins/libm-test/Cargo.toml
@@ -28,29 +28,29 @@ icount = ["dep:gungraun"]
 short-benchmarks = []
 
 [dependencies]
-anyhow = "1.0.101"
+anyhow.workspace = true
 # This is not directly used but is required so we can enable `gmp-mpfr-sys/force-cross`.
-gmp-mpfr-sys = { version = "1.6.8", optional = true, default-features = false }
-gungraun = { version = "0.17.0", optional = true }
-indicatif = { version = "0.18.3", default-features = false }
-libm = { path = "../libm", features = ["unstable-public-internals"] }
-libm-macros = { path = "../crates/libm-macros" }
-musl-math-sys = { path = "../crates/musl-math-sys", optional = true }
-paste = "1.0.15"
-rand = "0.9.2"
-rand_chacha = "0.9.0"
-rayon = "1.11.0"
-rug = { version = "1.28.1", optional = true, default-features = false, features = ["float", "integer", "std"] }
+gmp-mpfr-sys = { workspace = true, optional = true }
+gungraun = { workspace = true, optional = true }
+indicatif.workspace = true
+libm = { workspace = true, default-features = true, features = ["unstable-public-internals"] }
+libm-macros.workspace = true
+musl-math-sys = { workspace = true, optional = true }
+paste.workspace = true
+rand.workspace = true
+rand_chacha.workspace = true
+rayon.workspace = true
+rug = { workspace = true, optional = true }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
-getrandom = { version = "0.3.4", features = ["wasm_js"] }
+getrandom = { workspace = true, features = ["wasm_js"] }
 
 [build-dependencies]
-rand = { version = "0.9.2", optional = true }
+rand = { workspace = true, optional = true }
 
 [dev-dependencies]
-criterion = { version = "0.6.0", default-features = false, features = ["cargo_bench_support"] }
-libtest-mimic = "0.8.1"
+criterion.workspace = true
+libtest-mimic.workspace = true
 
 [[bench]]
 name = "icount"
diff --git a/library/compiler-builtins/libm/Cargo.toml b/library/compiler-builtins/libm/Cargo.toml
index 4d8b9bf827ad..617914a4dfae 100644
--- a/library/compiler-builtins/libm/Cargo.toml
+++ b/library/compiler-builtins/libm/Cargo.toml
@@ -43,6 +43,7 @@ unstable-float = []
 force-soft-floats = []
 
 [dev-dependencies]
+# FIXME(msrv): switch to `no-panic.workspace` when possible
 no-panic = "0.1.35"
 
 [lints.rust]

From 7c9ae5b0213393660abe57add8d4b34c90c3430f Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 06:48:37 -0600
Subject: [PATCH 1415/1843] meta: Sort Cargo.toml `[features]` table after
 `[dependencies]`

---
 .../compiler-builtins/crates/util/Cargo.toml  | 12 ++---
 .../compiler-builtins/libm-test/Cargo.toml    | 44 +++++++++----------
 library/compiler-builtins/libm/Cargo.toml     |  8 ++--
 3 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/library/compiler-builtins/crates/util/Cargo.toml b/library/compiler-builtins/crates/util/Cargo.toml
index 88e0b332065d..c56e2cc12ea5 100644
--- a/library/compiler-builtins/crates/util/Cargo.toml
+++ b/library/compiler-builtins/crates/util/Cargo.toml
@@ -5,15 +5,15 @@ edition = "2024"
 publish = false
 license = "MIT OR Apache-2.0"
 
-[features]
-default = ["build-musl", "build-mpfr", "unstable-float"]
-build-musl = ["libm-test/build-musl", "dep:musl-math-sys"]
-build-mpfr = ["libm-test/build-mpfr", "dep:rug"]
-unstable-float = ["libm/unstable-float", "libm-test/unstable-float", "rug?/nightly-float"]
-
 [dependencies]
 libm.workspace = true
 libm-macros.workspace = true
 libm-test.workspace = true
 musl-math-sys = { workspace = true, optional = true }
 rug = { workspace = true, optional = true }
+
+[features]
+default = ["build-musl", "build-mpfr", "unstable-float"]
+build-musl = ["libm-test/build-musl", "dep:musl-math-sys"]
+build-mpfr = ["libm-test/build-mpfr", "dep:rug"]
+unstable-float = ["libm/unstable-float", "libm-test/unstable-float", "rug?/nightly-float"]
diff --git a/library/compiler-builtins/libm-test/Cargo.toml b/library/compiler-builtins/libm-test/Cargo.toml
index c395d6a21bd5..4f65504bd584 100644
--- a/library/compiler-builtins/libm-test/Cargo.toml
+++ b/library/compiler-builtins/libm-test/Cargo.toml
@@ -5,28 +5,6 @@ edition = "2024"
 publish = false
 license = "MIT OR Apache-2.0"
 
-[features]
-default = ["build-mpfr", "unstable-float"]
-
-# Propagated from libm because this affects which functions we test.
-unstable-float = ["libm/unstable-float", "rug?/nightly-float"]
-
-# Generate tests which are random inputs and the outputs are calculated with
-# musl libc.
-build-mpfr = ["dep:rug", "dep:gmp-mpfr-sys"]
-
-# Build our own musl for testing and benchmarks
-build-musl = ["dep:musl-math-sys"]
-
-# Enable report generation without bringing in more dependencies by default
-benchmarking-reports = ["criterion/plotters", "criterion/html_reports"]
-
-# Enable icount benchmarks (requires gungraun-runner and valgrind locally)
-icount = ["dep:gungraun"]
-
-# Run with a reduced set of benchmarks, such as for CI
-short-benchmarks = []
-
 [dependencies]
 anyhow.workspace = true
 # This is not directly used but is required so we can enable `gmp-mpfr-sys/force-cross`.
@@ -52,6 +30,28 @@ rand = { workspace = true, optional = true }
 criterion.workspace = true
 libtest-mimic.workspace = true
 
+[features]
+default = ["build-mpfr", "unstable-float"]
+
+# Propagated from libm because this affects which functions we test.
+unstable-float = ["libm/unstable-float", "rug?/nightly-float"]
+
+# Generate tests which are random inputs and the outputs are calculated with
+# musl libc.
+build-mpfr = ["dep:rug", "dep:gmp-mpfr-sys"]
+
+# Build our own musl for testing and benchmarks
+build-musl = ["dep:musl-math-sys"]
+
+# Enable report generation without bringing in more dependencies by default
+benchmarking-reports = ["criterion/plotters", "criterion/html_reports"]
+
+# Enable icount benchmarks (requires gungraun-runner and valgrind locally)
+icount = ["dep:gungraun"]
+
+# Run with a reduced set of benchmarks, such as for CI
+short-benchmarks = []
+
 [[bench]]
 name = "icount"
 harness = false
diff --git a/library/compiler-builtins/libm/Cargo.toml b/library/compiler-builtins/libm/Cargo.toml
index 617914a4dfae..d80ddab0ab7a 100644
--- a/library/compiler-builtins/libm/Cargo.toml
+++ b/library/compiler-builtins/libm/Cargo.toml
@@ -15,6 +15,10 @@ license = "MIT"
 edition = "2021"
 rust-version = "1.63"
 
+[dev-dependencies]
+# FIXME(msrv): switch to `no-panic.workspace` when possible
+no-panic = "0.1.35"
+
 [features]
 default = ["arch"]
 
@@ -42,10 +46,6 @@ unstable-float = []
 # hard float operations.
 force-soft-floats = []
 
-[dev-dependencies]
-# FIXME(msrv): switch to `no-panic.workspace` when possible
-no-panic = "0.1.35"
-
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = [
   # compiler-builtins sets this feature, but we use it in `libm`

From da421f55850b5f7ef7e560ec8e52b41c39ff03be Mon Sep 17 00:00:00 2001
From: Pavel Grigorenko 
Date: Tue, 27 Jan 2026 23:54:24 +0300
Subject: [PATCH 1416/1843] =?UTF-8?q?`const=20{=20'=CE=A3'.len=5Futf8()=20?=
 =?UTF-8?q?}`?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 library/alloc/src/str.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/library/alloc/src/str.rs b/library/alloc/src/str.rs
index e772ac25a95c..8a3326c7d76a 100644
--- a/library/alloc/src/str.rs
+++ b/library/alloc/src/str.rs
@@ -411,9 +411,8 @@ impl str {
         fn map_uppercase_sigma(from: &str, i: usize) -> char {
             // See https://www.unicode.org/versions/Unicode7.0.0/ch03.pdf#G33992
             // for the definition of `Final_Sigma`.
-            debug_assert!('Σ'.len_utf8() == 2);
             let is_word_final = case_ignorable_then_cased(from[..i].chars().rev())
-                && !case_ignorable_then_cased(from[i + 2..].chars());
+                && !case_ignorable_then_cased(from[i + const { 'Σ'.len_utf8() }..].chars());
             if is_word_final { 'ς' } else { 'σ' }
         }
 

From 03dcd99aa322a0d5c9b5f203b1222798bbabf941 Mon Sep 17 00:00:00 2001
From: ia0 
Date: Sat, 7 Feb 2026 15:39:42 +0100
Subject: [PATCH 1417/1843] Format heterogeneous try blocks

---
 src/tools/rustfmt/src/expr.rs                 | 29 +++++++++++++--
 .../tests/source/try_blocks_heterogeneous.rs  | 34 ++++++++++++++++++
 .../tests/target/try_blocks_heterogeneous.rs  | 36 +++++++++++++++++++
 3 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100644 src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs
 create mode 100644 src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs

diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs
index de96f004dc87..70bce1920047 100644
--- a/src/tools/rustfmt/src/expr.rs
+++ b/src/tools/rustfmt/src/expr.rs
@@ -385,8 +385,33 @@ pub(crate) fn format_expr(
                 ))
             }
         }
-        // FIXME: heterogeneous try blocks, which include a type so are harder to format
-        ast::ExprKind::TryBlock(_, Some(_)) => Err(RewriteError::Unknown),
+        ast::ExprKind::TryBlock(ref block, Some(ref ty)) => {
+            let keyword = "try bikeshed ";
+            // 2 = " {".len()
+            let ty_shape = shape
+                .shrink_left(keyword.len())
+                .and_then(|shape| shape.sub_width(2))
+                .max_width_error(shape.width, expr.span)?;
+            let ty_str = ty.rewrite_result(context, ty_shape)?;
+            let prefix = format!("{keyword}{ty_str} ");
+            if let rw @ Ok(_) =
+                rewrite_single_line_block(context, &prefix, block, Some(&expr.attrs), None, shape)
+            {
+                rw
+            } else {
+                let budget = shape.width.saturating_sub(prefix.len());
+                Ok(format!(
+                    "{prefix}{}",
+                    rewrite_block(
+                        block,
+                        Some(&expr.attrs),
+                        None,
+                        context,
+                        Shape::legacy(budget, shape.indent)
+                    )?
+                ))
+            }
+        }
         ast::ExprKind::Gen(capture_by, ref block, ref kind, _) => {
             let mover = if matches!(capture_by, ast::CaptureBy::Value { .. }) {
                 "move "
diff --git a/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs b/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs
new file mode 100644
index 000000000000..bae0060374d8
--- /dev/null
+++ b/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs
@@ -0,0 +1,34 @@
+// rustfmt-edition: 2018
+
+fn main() -> Result<(), !> {
+    let _x = try bikeshed Option<_> {
+        4
+    };
+
+    try bikeshed Result<_, _> {}
+}
+
+fn baz() -> Option {
+    if (1 == 1) {
+        return try bikeshed Option {
+            5
+        };
+    }
+
+    // test
+    let x = try bikeshed Option<()> {
+        // try blocks are great
+    };
+
+    let y = try bikeshed Option {
+        6
+    }; // comment
+
+    let x = try bikeshed Option { baz()?; baz()?; baz()?; 7 };
+
+    let x = try bikeshed Foo { 1 + 1 + 1 };
+
+    let x = try bikeshed Foo {};
+
+    return None;
+}
diff --git a/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs b/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs
new file mode 100644
index 000000000000..6f910ed5d051
--- /dev/null
+++ b/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs
@@ -0,0 +1,36 @@
+// rustfmt-edition: 2018
+
+fn main() -> Result<(), !> {
+    let _x = try bikeshed Option<_> { 4 };
+
+    try bikeshed Result<_, _> {}
+}
+
+fn baz() -> Option {
+    if (1 == 1) {
+        return try bikeshed Option { 5 };
+    }
+
+    // test
+    let x = try bikeshed Option<()> {
+        // try blocks are great
+    };
+
+    let y = try bikeshed Option { 6 }; // comment
+
+    let x = try bikeshed Option {
+        baz()?;
+        baz()?;
+        baz()?;
+        7
+    };
+
+    let x = try bikeshed Foo {
+        1 + 1 + 1
+    };
+
+    let x =
+        try bikeshed Foo {};
+
+    return None;
+}

From 1ec5101f20cfffb67557692a5ec647133e2bcd19 Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 07:06:05 -0600
Subject: [PATCH 1418/1843] Bump the libm MSRV to 1.67

This gets us:

* `saturating_sub_unsigned`
* `::ilog2`
* Correct lexing of float literals with the `f16` or `f128` suffix

Link: https://github.com/rust-lang/compiler-builtins/issues/1017
---
 library/compiler-builtins/libm/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/libm/Cargo.toml b/library/compiler-builtins/libm/Cargo.toml
index d80ddab0ab7a..98202d1977dc 100644
--- a/library/compiler-builtins/libm/Cargo.toml
+++ b/library/compiler-builtins/libm/Cargo.toml
@@ -13,7 +13,7 @@ keywords = ["libm", "math"]
 repository = "https://github.com/rust-lang/compiler-builtins"
 license = "MIT"
 edition = "2021"
-rust-version = "1.63"
+rust-version = "1.67"
 
 [dev-dependencies]
 # FIXME(msrv): switch to `no-panic.workspace` when possible

From 3d0989266de7d312a17ffac4e8251eb7a710358f Mon Sep 17 00:00:00 2001
From: Trevor Gross 
Date: Sat, 7 Feb 2026 07:06:05 -0600
Subject: [PATCH 1419/1843] cleanup: Perform some simplifications possible with
 the MSRV bump

---
 .../libm/src/math/generic/sqrt.rs             |  4 +--
 .../libm/src/math/support/float_traits.rs     |  7 +----
 .../libm/src/math/support/hex_float.rs        | 28 +------------------
 3 files changed, 3 insertions(+), 36 deletions(-)

diff --git a/library/compiler-builtins/libm/src/math/generic/sqrt.rs b/library/compiler-builtins/libm/src/math/generic/sqrt.rs
index 9481c4cdb7bb..e97a43d34956 100644
--- a/library/compiler-builtins/libm/src/math/generic/sqrt.rs
+++ b/library/compiler-builtins/libm/src/math/generic/sqrt.rs
@@ -419,9 +419,7 @@ mod tests {
     fn conformance_tests_f16() {
         let cases = [
             (f16::PI, 0x3f17_u16),
-            // 10_000.0, using a hex literal for MSRV hack (Rust < 1.67 checks literal widths as
-            // part of the AST, so the `cfg` is irrelevant here).
-            (f16::from_bits(0x70e2), 0x5640_u16),
+            (10000.0_f16, 0x5640_u16),
             (f16::from_bits(0x0000000f), 0x13bf_u16),
             (f16::INFINITY, f16::INFINITY.to_bits()),
         ];
diff --git a/library/compiler-builtins/libm/src/math/support/float_traits.rs b/library/compiler-builtins/libm/src/math/support/float_traits.rs
index 4e5011f62e0f..60c8bfca5165 100644
--- a/library/compiler-builtins/libm/src/math/support/float_traits.rs
+++ b/library/compiler-builtins/libm/src/math/support/float_traits.rs
@@ -1,5 +1,3 @@
-#![allow(unknown_lints)] // FIXME(msrv) we shouldn't need this
-
 use core::{fmt, mem, ops};
 
 use super::int_traits::{CastFrom, Int, MinInt};
@@ -289,10 +287,7 @@ macro_rules! float_impl {
                 cfg_if! {
                     // fma is not yet available in `core`
                     if #[cfg(intrinsics_enabled)] {
-                        // FIXME(msrv,bench): once our benchmark rustc version is above the
-                        // 2022-09-23 nightly, this can be removed.
-                        #[allow(unused_unsafe)]
-                        unsafe { core::intrinsics::$fma_intrinsic(self, y, z) }
+                        core::intrinsics::$fma_intrinsic(self, y, z)
                     } else {
                         super::super::$fma_fn(self, y, z)
                     }
diff --git a/library/compiler-builtins/libm/src/math/support/hex_float.rs b/library/compiler-builtins/libm/src/math/support/hex_float.rs
index c8558b90053d..2f9369e50441 100644
--- a/library/compiler-builtins/libm/src/math/support/hex_float.rs
+++ b/library/compiler-builtins/libm/src/math/support/hex_float.rs
@@ -121,7 +121,7 @@ const fn parse_finite(
         Ok(Parsed { sig, exp }) => (sig, exp),
     };
 
-    let mut round_bits = u128_ilog2(sig) as i32 - sig_bits as i32;
+    let mut round_bits = sig.ilog2() as i32 - sig_bits as i32;
 
     // Round at least up to min_lsb
     if exp < min_lsb - round_bits {
@@ -299,29 +299,11 @@ const fn parse_hex(mut b: &[u8]) -> Result {
         ));
     };
 
-    {
-        let e;
-        if negate_exp {
-            e = (exp as i64) - (pexp as i64);
-        } else {
-            e = (exp as i64) + (pexp as i64);
-        };
-
-        exp = if e < i32::MIN as i64 {
-            i32::MIN
-        } else if e > i32::MAX as i64 {
-            i32::MAX
-        } else {
-            e as i32
-        };
-    }
-    /* FIXME(msrv): once MSRV >= 1.66, replace the above workaround block with:
     if negate_exp {
         exp = exp.saturating_sub_unsigned(pexp);
     } else {
         exp = exp.saturating_add_unsigned(pexp);
     };
-    */
 
     Ok(Parsed { sig, exp })
 }
@@ -342,14 +324,6 @@ const fn hex_digit(c: u8) -> Option {
     }
 }
 
-/* FIXME(msrv): vendor some things that are not const stable at our MSRV */
-
-/// `u128::ilog2`
-const fn u128_ilog2(v: u128) -> u32 {
-    assert!(v != 0);
-    u128::BITS - 1 - v.leading_zeros()
-}
-
 #[cfg(any(test, feature = "unstable-public-internals"))]
 mod hex_fmt {
     use core::fmt;

From cf1a784f3f2643bb5afb0fa7e1944c8748a0fefa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:21:25 +0100
Subject: [PATCH 1420/1843] Port rustc_regions to the new attribute parser

---
 .../src/attributes/rustc_internal.rs             | 16 ++++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs       |  1 +
 compiler/rustc_borrowck/src/nll.rs               |  5 +++--
 compiler/rustc_hir/src/attrs/data_structures.rs  |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs    |  1 +
 compiler/rustc_passes/src/check_attr.rs          |  2 +-
 6 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 3b7d25b43ba2..526d5ec1e1dc 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -219,6 +219,22 @@ impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintQueryInstability;
 }
 
+pub(crate) struct RustcRegionsParser;
+
+impl NoArgsAttributeParser for RustcRegionsParser {
+    const PATH: &[Symbol] = &[sym::rustc_regions];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::Trait { body: false })),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+    ]);
+
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcRegions;
+}
+
 pub(crate) struct RustcLintUntrackedQueryInformationParser;
 
 impl NoArgsAttributeParser for RustcLintUntrackedQueryInformationParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 5b489d306499..3c49c272c121 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -276,6 +276,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 64e3b59acfff..ee32d64e9958 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -7,6 +7,8 @@ use std::str::FromStr;
 
 use polonius_engine::{Algorithm, AllFacts, Output};
 use rustc_data_structures::frozen::Frozen;
+use rustc_hir::attrs::AttributeKind;
+use rustc_hir::find_attr;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::pretty::PrettyPrintMirOptions;
 use rustc_middle::mir::{Body, MirDumper, PassWhere, Promoted};
@@ -15,7 +17,6 @@ use rustc_middle::ty::{self, TyCtxt};
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::points::DenseLocationMap;
 use rustc_session::config::MirIncludeSpans;
-use rustc_span::sym;
 use tracing::{debug, instrument};
 
 use crate::borrow_set::BorrowSet;
@@ -295,7 +296,7 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
 ) {
     let tcx = infcx.tcx;
     let base_def_id = tcx.typeck_root_def_id(body.source.def_id());
-    if !tcx.has_attr(base_def_id, sym::rustc_regions) {
+    if !find_attr!(tcx.get_all_attrs(base_def_id), AttributeKind::RustcRegions) {
         return;
     }
 
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index fa781a74c9e2..2bbb19e0d10f 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1198,6 +1198,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_reallocator]`
     RustcReallocator,
 
+    /// Represents `#[rustc_regions]`
+    RustcRegions,
+
     /// Represents `#[rustc_scalable_vector(N)]`
     RustcScalableVector {
         /// The base multiple of lanes that are in a scalable vector, if provided. `element_count`
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 14b54c129d3e..ee4915eec649 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -144,6 +144,7 @@ impl AttributeKind {
             RustcPreserveUbChecks => No,
             RustcPubTransparent(..) => Yes,
             RustcReallocator => No,
+            RustcRegions => No,
             RustcScalableVector { .. } => Yes,
             RustcShouldNotBeCalledOnConstItems(..) => Yes,
             RustcSimdMonomorphizeLaneLimit(..) => Yes, // Affects layout computation, which needs to work cross-crate
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 680a93908354..292db8e1b684 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -335,6 +335,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcPassIndirectlyInNonRusticAbis(..)
                     | AttributeKind::RustcPreserveUbChecks
                     | AttributeKind::RustcReallocator
+                    | AttributeKind::RustcRegions
                     | AttributeKind::RustcScalableVector { .. }
                     | AttributeKind::RustcShouldNotBeCalledOnConstItems(..)
                     | AttributeKind::RustcSimdMonomorphizeLaneLimit(..)
@@ -401,7 +402,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_never_type_options
                             | sym::rustc_autodiff
                             | sym::rustc_capture_analysis
-                            | sym::rustc_regions
                             | sym::rustc_strict_coherence
                             | sym::rustc_mir
                             | sym::rustc_outlives

From 85ca098f55cade17d59dd625d2e9af724a73a33b Mon Sep 17 00:00:00 2001
From: Usman Akinyemi 
Date: Sun, 1 Feb 2026 22:34:22 +0530
Subject: [PATCH 1421/1843] rustc_parse: improve the error diagnostic for
 "missing let"

Signed-off-by: Usman Akinyemi 
---
 compiler/rustc_parse/src/errors.rs      | 18 ++++++++++
 compiler/rustc_parse/src/parser/expr.rs | 47 ++++++++++++++++++++++++-
 tests/ui/missing/missing-let.rs         | 12 +++++++
 tests/ui/missing/missing-let.stderr     | 28 +++++++++++++--
 4 files changed, 102 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 418d56f632bd..aafe585a4649 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -573,6 +573,24 @@ pub(crate) struct ExpectedExpressionFoundLet {
     pub comparison: Option,
 }
 
+#[derive(Diagnostic)]
+#[diag("let-chain with missing `let`")]
+pub(crate) struct LetChainMissingLet {
+    #[primary_span]
+    pub span: Span,
+    #[label("expected `let` expression, found assignment")]
+    pub label_span: Span,
+    #[label("let expression later in the condition")]
+    pub rhs_span: Span,
+    #[suggestion(
+        "add `let` before the expression",
+        applicability = "maybe-incorrect",
+        code = "let ",
+        style = "verbose"
+    )]
+    pub sug_span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag("`||` operators are not supported in let chain conditions")]
 pub(crate) struct OrInLetChain {
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 8bb22c2a831b..05216be06ff5 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -4282,7 +4282,52 @@ impl MutVisitor for CondChecker<'_> {
                 mut_visit::walk_expr(self, e);
                 self.forbid_let_reason = forbid_let_reason;
             }
-            ExprKind::Assign(ref lhs, _, span) => {
+            ExprKind::Assign(ref lhs, ref rhs, span) => {
+                if let ExprKind::Call(_, _) = &lhs.kind {
+                    fn get_path_from_rhs(e: &Expr) -> Option<(u32, &Path)> {
+                        fn inner(e: &Expr, depth: u32) -> Option<(u32, &Path)> {
+                            match &e.kind {
+                                ExprKind::Binary(_, lhs, _) => inner(lhs, depth + 1),
+                                ExprKind::Path(_, path) => Some((depth, path)),
+                                _ => None,
+                            }
+                        }
+
+                        inner(e, 0)
+                    }
+
+                    if let Some((depth, path)) = get_path_from_rhs(rhs) {
+                        // For cases like if Some(_) = x && let Some(_) = y && let Some(_) = z
+                        // This return let Some(_) = y expression
+                        fn find_let_some(expr: &Expr) -> Option<&Expr> {
+                            match &expr.kind {
+                                ExprKind::Let(..) => Some(expr),
+
+                                ExprKind::Binary(op, lhs, rhs) if op.node == BinOpKind::And => {
+                                    find_let_some(lhs).or_else(|| find_let_some(rhs))
+                                }
+
+                                _ => None,
+                            }
+                        }
+
+                        let expr_span = lhs.span.to(path.span);
+
+                        if let Some(later_rhs) = find_let_some(rhs)
+                            && depth > 0
+                        {
+                            let guar = self.parser.dcx().emit_err(errors::LetChainMissingLet {
+                                span: lhs.span,
+                                label_span: expr_span,
+                                rhs_span: later_rhs.span,
+                                sug_span: lhs.span.shrink_to_lo(),
+                            });
+
+                            self.found_incorrect_let_chain = Some(guar);
+                        }
+                    }
+                }
+
                 let forbid_let_reason = self.forbid_let_reason;
                 self.forbid_let_reason = Some(errors::ForbiddenLetReason::OtherForbidden);
                 let missing_let = self.missing_let;
diff --git a/tests/ui/missing/missing-let.rs b/tests/ui/missing/missing-let.rs
index 36db7bc95826..595de640470b 100644
--- a/tests/ui/missing/missing-let.rs
+++ b/tests/ui/missing/missing-let.rs
@@ -1,6 +1,18 @@
 fn main() {
     let x = Some(42);
+    let y = Some(42);
+    let z = Some(42);
     if let Some(_) = x
         && Some(x) = x //~^ ERROR expected expression, found `let` statement
+        //~| NOTE: only supported directly in conditions of `if` and `while` expressions
+    {}
+
+    if Some(_) = y &&
+    //~^ NOTE expected `let` expression, found assignment
+    //~| ERROR let-chain with missing `let`
+        let Some(_) = z
+        //~^ ERROR: expected expression, found `let` statement
+        //~| NOTE: let expression later in the condition
+        //~| NOTE: only supported directly in conditions of `if` and `while` expressions
     {}
 }
diff --git a/tests/ui/missing/missing-let.stderr b/tests/ui/missing/missing-let.stderr
index 897ff6329d59..0a6e76b154f7 100644
--- a/tests/ui/missing/missing-let.stderr
+++ b/tests/ui/missing/missing-let.stderr
@@ -1,5 +1,5 @@
 error: expected expression, found `let` statement
-  --> $DIR/missing-let.rs:3:8
+  --> $DIR/missing-let.rs:5:8
    |
 LL |     if let Some(_) = x
    |        ^^^^^^^^^^^^^^^
@@ -14,5 +14,29 @@ help: you might have meant to compare for equality
 LL |         && Some(x) == x
    |                     +
 
-error: aborting due to 1 previous error
+error: expected expression, found `let` statement
+  --> $DIR/missing-let.rs:13:9
+   |
+LL |         let Some(_) = z
+   |         ^^^
+   |
+   = note: only supported directly in conditions of `if` and `while` expressions
+
+error: let-chain with missing `let`
+  --> $DIR/missing-let.rs:10:8
+   |
+LL |     if Some(_) = y &&
+   |        ^^^^^^^----
+   |        |
+   |        expected `let` expression, found assignment
+...
+LL |         let Some(_) = z
+   |         --------------- let expression later in the condition
+   |
+help: add `let` before the expression
+   |
+LL |     if let Some(_) = y &&
+   |        +++
+
+error: aborting due to 3 previous errors
 

From c814f76c0680cb073941c19e8750d859be7e6393 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 14:58:56 +0100
Subject: [PATCH 1422/1843] Convert to inline diagnostics in `rustc_lint`

---
 Cargo.lock                                    |    1 -
 compiler/rustc_driver_impl/src/lib.rs         |    6 +-
 compiler/rustc_lint/Cargo.toml                |    1 -
 compiler/rustc_lint/messages.ftl              | 1088 -----------
 compiler/rustc_lint/src/async_closures.rs     |    8 +-
 compiler/rustc_lint/src/builtin.rs            |   15 +-
 compiler/rustc_lint/src/errors.rs             |   41 +-
 compiler/rustc_lint/src/if_let_rescope.rs     |   23 +-
 .../rustc_lint/src/impl_trait_overcaptures.rs |   26 +-
 compiler/rustc_lint/src/levels.rs             |    7 +-
 compiler/rustc_lint/src/lib.rs                |    2 -
 compiler/rustc_lint/src/lints.rs              | 1740 +++++++++++------
 compiler/rustc_lint/src/non_fmt_panic.rs      |   23 +-
 compiler/rustc_lint/src/non_local_def.rs      |   11 +-
 .../src/opaque_hidden_inferred_bound.rs       |    6 +-
 compiler/rustc_lint/src/transmute.rs          |   10 +-
 .../rustc_lint/src/types/improper_ctypes.rs   |  112 +-
 17 files changed, 1307 insertions(+), 1813 deletions(-)
 delete mode 100644 compiler/rustc_lint/messages.ftl

diff --git a/Cargo.lock b/Cargo.lock
index 7a8e06e5ba93..03cf3bcba894 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4143,7 +4143,6 @@ dependencies = [
  "rustc_data_structures",
  "rustc_errors",
  "rustc_feature",
- "rustc_fluent_macro",
  "rustc_hir",
  "rustc_index",
  "rustc_infer",
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 315b11056085..d72913764e9d 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -111,11 +111,7 @@ pub fn default_translator() -> Translator {
     Translator::with_fallback_bundle(DEFAULT_LOCALE_RESOURCES.to_vec(), false)
 }
 
-pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
-    // tidy-alphabetical-start
-    rustc_lint::DEFAULT_LOCALE_RESOURCE,
-    // tidy-alphabetical-end
-];
+pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[];
 
 /// Exit status code used for successful compilation and help output.
 pub const EXIT_SUCCESS: i32 = 0;
diff --git a/compiler/rustc_lint/Cargo.toml b/compiler/rustc_lint/Cargo.toml
index 035d8b4903fa..758d2762a6af 100644
--- a/compiler/rustc_lint/Cargo.toml
+++ b/compiler/rustc_lint/Cargo.toml
@@ -14,7 +14,6 @@ rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl
deleted file mode 100644
index ff771532ae49..000000000000
--- a/compiler/rustc_lint/messages.ftl
+++ /dev/null
@@ -1,1088 +0,0 @@
-lint_abs_path_with_module = absolute paths must start with `self`, `super`, `crate`, or an external crate name in the 2018 edition
-    .suggestion = use `crate`
-
-lint_ambiguous_glob_reexport = ambiguous glob re-exports
-    .label_first_reexport = the name `{$name}` in the {$namespace} namespace is first re-exported here
-    .label_duplicate_reexport = but the name `{$name}` in the {$namespace} namespace is also re-exported here
-
-lint_ambiguous_negative_literals = `-` has lower precedence than method calls, which might be unexpected
-    .example = e.g. `-4.abs()` equals `-4`; while `(-4).abs()` equals `4`
-    .negative_literal = add parentheses around the `-` and the literal to call the method on a negative literal
-    .current_behavior = add parentheses around the literal and the method call to keep the current behavior
-
-lint_ambiguous_wide_pointer_comparisons = ambiguous wide pointer comparison, the comparison includes metadata which may not be expected
-    .addr_metadata_suggestion = use explicit `std::ptr::eq` method to compare metadata and addresses
-    .addr_suggestion = use `std::ptr::addr_eq` or untyped pointers to only compare their addresses
-    .cast_suggestion = use untyped pointers to only compare their addresses
-    .expect_suggestion = or expect the lint to compare the pointers metadata and addresses
-
-lint_associated_const_elided_lifetime = {$elided ->
-        [true] `&` without an explicit lifetime name cannot be used here
-        *[false] `'_` cannot be used here
-    }
-    .suggestion = use the `'static` lifetime
-    .note = cannot automatically infer `'static` because of other lifetimes in scope
-
-lint_async_fn_in_trait = use of `async fn` in public traits is discouraged as auto trait bounds cannot be specified
-    .note = you can suppress this lint if you plan to use the trait only in your own code, or do not care about auto traits like `Send` on the `Future`
-    .suggestion = you can alternatively desugar to a normal `fn` that returns `impl Future` and add any desired bounds such as `Send`, but these cannot be relaxed without a breaking API change
-
-lint_atomic_ordering_fence = memory fences cannot have `Relaxed` ordering
-    .help = consider using ordering modes `Acquire`, `Release`, `AcqRel` or `SeqCst`
-
-lint_atomic_ordering_invalid = `{$method}`'s failure ordering may not be `Release` or `AcqRel`, since a failed `{$method}` does not result in a write
-    .label = invalid failure ordering
-    .help = consider using `Acquire` or `Relaxed` failure ordering instead
-
-lint_atomic_ordering_load = atomic loads cannot have `Release` or `AcqRel` ordering
-    .help = consider using ordering modes `Acquire`, `SeqCst` or `Relaxed`
-
-lint_atomic_ordering_store = atomic stores cannot have `Acquire` or `AcqRel` ordering
-    .help = consider using ordering modes `Release`, `SeqCst` or `Relaxed`
-
-lint_attr_crate_level =
-    this attribute can only be applied at the crate level
-    .suggestion = to apply to the crate, use an inner attribute
-    .note = read  for more information
-
-lint_bad_attribute_argument = bad attribute argument
-
-lint_bad_opt_access = {$msg}
-
-lint_break_with_label_and_loop = this labeled break expression is easy to confuse with an unlabeled break with a labeled value expression
-    .suggestion = wrap this expression in parentheses
-
-lint_builtin_allow_internal_unsafe =
-    `allow_internal_unsafe` allows defining macros using unsafe without triggering the `unsafe_code` lint at their call site
-
-lint_builtin_anonymous_params = anonymous parameters are deprecated and will be removed in the next edition
-    .suggestion = try naming the parameter or explicitly ignoring it
-
-lint_builtin_clashing_extern_diff_name = `{$this}` redeclares `{$orig}` with a different signature
-    .previous_decl_label = `{$orig}` previously declared here
-    .mismatch_label = this signature doesn't match the previous declaration
-
-lint_builtin_clashing_extern_same_name = `{$this}` redeclared with a different signature
-    .previous_decl_label = `{$orig}` previously declared here
-    .mismatch_label = this signature doesn't match the previous declaration
-lint_builtin_const_no_mangle = const items should never be `#[no_mangle]`
-    .suggestion = try a static value
-
-lint_builtin_decl_unsafe_fn = declaration of an `unsafe` function
-lint_builtin_decl_unsafe_method = declaration of an `unsafe` method
-
-lint_builtin_deref_nullptr = dereferencing a null pointer
-    .label = this code causes undefined behavior when executed
-
-lint_builtin_double_negations = use of a double negation
-    .note = the prefix `--` could be misinterpreted as a decrement operator which exists in other languages
-    .note_decrement = use `-= 1` if you meant to decrement the value
-    .add_parens_suggestion = add parentheses for clarity
-
-lint_builtin_ellipsis_inclusive_range_patterns = `...` range patterns are deprecated
-    .suggestion = use `..=` for an inclusive range
-
-lint_builtin_explicit_outlives = outlives requirements can be inferred
-    .suggestion = remove {$count ->
-        [one] this bound
-        *[other] these bounds
-    }
-
-lint_builtin_export_name_fn = declaration of a function with `export_name`
-lint_builtin_export_name_method = declaration of a method with `export_name`
-lint_builtin_export_name_static = declaration of a static with `export_name`
-
-lint_builtin_global_asm = usage of `core::arch::global_asm`
-lint_builtin_global_macro_unsafety = using this macro is unsafe even though it does not need an `unsafe` block
-
-lint_builtin_impl_unsafe_method = implementation of an `unsafe` method
-
-lint_builtin_incomplete_features = the feature `{$name}` is incomplete and may not be safe to use and/or cause compiler crashes
-    .note = see issue #{$n}  for more information
-    .help = consider using `min_{$name}` instead, which is more stable and complete
-
-lint_builtin_internal_features = the feature `{$name}` is internal to the compiler or standard library
-    .note = using it is strongly discouraged
-
-lint_builtin_keyword_idents = `{$kw}` is a keyword in the {$next} edition
-    .suggestion = you can use a raw identifier to stay compatible
-
-lint_builtin_link_section_fn = declaration of a function with `link_section`
-
-lint_builtin_link_section_static = declaration of a static with `link_section`
-
-lint_builtin_missing_copy_impl = type could implement `Copy`; consider adding `impl Copy`
-
-lint_builtin_missing_debug_impl =
-    type does not implement `{$debug}`; consider adding `#[derive(Debug)]` or a manual implementation
-
-lint_builtin_missing_doc = missing documentation for {$article} {$desc}
-
-lint_builtin_mutable_transmutes =
-    transmuting &T to &mut T is undefined behavior, even if the reference is unused, consider instead using an UnsafeCell
-
-lint_builtin_no_mangle_fn = declaration of a `no_mangle` function
-lint_builtin_no_mangle_generic = functions generic over types or consts must be mangled
-    .suggestion = remove this attribute
-
-lint_builtin_no_mangle_method = declaration of a `no_mangle` method
-lint_builtin_no_mangle_static = declaration of a `no_mangle` static
-lint_builtin_non_shorthand_field_patterns = the `{$ident}:` in this pattern is redundant
-    .suggestion = use shorthand field pattern
-
-lint_builtin_overridden_symbol_name =
-    the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them
-
-lint_builtin_overridden_symbol_section =
-    the program's behavior with overridden link sections on items is unpredictable and Rust cannot provide guarantees when you manually override them
-
-lint_builtin_special_module_name_used_lib = found module declaration for lib.rs
-    .note = lib.rs is the root of this crate's library target
-    .help = to refer to it from other targets, use the library's name as the path
-
-lint_builtin_special_module_name_used_main = found module declaration for main.rs
-    .note = a binary crate cannot be used as library
-
-lint_builtin_trivial_bounds = {$predicate_kind_name} bound {$predicate} does not depend on any type or lifetime parameters
-
-lint_builtin_type_alias_bounds_enable_feat_help = add `#![feature(lazy_type_alias)]` to the crate attributes to enable the desired semantics
-lint_builtin_type_alias_bounds_label = will not be checked at usage sites of the type alias
-lint_builtin_type_alias_bounds_limitation_note = this is a known limitation of the type checker that may be lifted in a future edition.
-    see issue #112792  for more information
-lint_builtin_type_alias_bounds_param_bounds = bounds on generic parameters in type aliases are not enforced
-    .suggestion = remove {$count ->
-        [one] this bound
-        *[other] these bounds
-    }
-lint_builtin_type_alias_bounds_qualify_assoc_tys_sugg = fully qualify this associated type
-lint_builtin_type_alias_bounds_where_clause = where clauses on type aliases are not enforced
-    .suggestion = remove this where clause
-
-lint_builtin_unpermitted_type_init_label = this code causes undefined behavior when executed
-lint_builtin_unpermitted_type_init_label_suggestion = help: use `MaybeUninit` instead, and only call `assume_init` after initialization is done
-
-lint_builtin_unpermitted_type_init_uninit = the type `{$ty}` does not permit being left uninitialized
-
-lint_builtin_unpermitted_type_init_zeroed = the type `{$ty}` does not permit zero-initialization
-lint_builtin_unreachable_pub = unreachable `pub` {$what}
-    .suggestion = consider restricting its visibility
-    .help = or consider exporting it for use by other crates
-
-lint_builtin_unsafe_block = usage of an `unsafe` block
-
-lint_builtin_unsafe_extern_block = usage of an `unsafe extern` block
-
-lint_builtin_unsafe_impl = implementation of an `unsafe` trait
-
-lint_builtin_unsafe_trait = declaration of an `unsafe` trait
-
-lint_builtin_unstable_features = use of an unstable feature
-
-lint_builtin_unused_doc_comment = unused doc comment
-    .label = rustdoc does not generate documentation for {$kind}
-    .plain_help = use `//` for a plain comment
-    .block_help = use `/* */` for a plain comment
-
-lint_builtin_while_true = denote infinite loops with `loop {"{"} ... {"}"}`
-    .suggestion = use `loop`
-
-lint_check_name_unknown_tool = unknown lint tool: `{$tool_name}`
-
-lint_closure_returning_async_block = closure returning async block can be made into an async closure
-    .label = this async block can be removed, and the closure can be turned into an async closure
-    .suggestion = turn this into an async closure
-
-lint_command_line_source = `forbid` lint level was set on command line
-
-lint_confusable_identifier_pair = found both `{$existing_sym}` and `{$sym}` as identifiers, which look alike
-    .current_use = this identifier can be confused with `{$existing_sym}`
-    .other_use = other identifier used here
-
-lint_const_item_interior_mutations =
-    mutation of an interior mutable `const` item with call to `{$method_name}`
-    .label = `{$const_name}` is a interior mutable `const` item of type `{$const_ty}`
-    .temporary = each usage of a `const` item creates a new temporary
-    .never_original = only the temporaries and never the original `const {$const_name}` will be modified
-    .suggestion_static = for a shared instance of `{$const_name}`, consider making it a `static` item instead
-    .help = for more details on interior mutability see 
-
-lint_dangling_pointers_from_locals = {$fn_kind} returns a dangling pointer to dropped local variable `{$local_var_name}`
-    .ret_ty = return type is `{$ret_ty}`
-    .local_var = local variable `{$local_var_name}` is dropped at the end of the {$fn_kind}
-    .created_at = dangling pointer created here
-    .note_safe = a dangling pointer is safe, but dereferencing one is undefined behavior
-    .note_more_info = for more information, see 
-
-lint_dangling_pointers_from_temporaries = this creates a dangling pointer because temporary `{$ty}` is dropped at end of statement
-    .label_ptr = pointer created here
-    .label_temporary = this `{$ty}` is dropped at end of statement
-    .help_bind = bind the `{$ty}` to a variable such that it outlives the pointer returned by `{$callee}`
-    .note_safe = a dangling pointer is safe, but dereferencing one is undefined behavior
-    .note_return = returning a pointer to a local variable will always result in a dangling pointer
-    .note_more_info = for more information, see 
-
-
-lint_default_hash_types = prefer `{$preferred}` over `{$used}`, it has better performance
-    .note = a `use rustc_data_structures::fx::{$preferred}` may be necessary
-
-lint_default_source = `forbid` lint level is the default for {$id}
-
-lint_deprecated_lint_name =
-    lint name `{$name}` is deprecated and may not have an effect in the future
-    .suggestion = change it to
-    .help = change it to {$replace}
-
-lint_deprecated_where_clause_location = where clause not allowed here
-    .note = see issue #89122  for more information
-    .suggestion_move_to_end = move it to the end of the type declaration
-    .suggestion_remove_where = remove this `where`
-
-lint_doc_alias_duplicated = doc alias is duplicated
-    .label = first defined here
-
-lint_doc_auto_cfg_expects_hide_or_show =
-    only `hide` or `show` are allowed in `#[doc(auto_cfg(...))]`
-
-lint_doc_auto_cfg_hide_show_expects_list =
-    `#![doc(auto_cfg({$attr_name}(...)))]` expects a list of items
-
-lint_doc_auto_cfg_hide_show_unexpected_item =
-    `#![doc(auto_cfg({$attr_name}(...)))]` only accepts identifiers or key/value items
-
-lint_doc_auto_cfg_wrong_literal =
-    expected boolean for `#[doc(auto_cfg = ...)]`
-
-lint_doc_invalid =
-    invalid `doc` attribute
-
-lint_doc_test_literal = `#![doc(test(...)]` does not take a literal
-
-lint_doc_test_takes_list =
-    `#[doc(test(...)]` takes a list of attributes
-
-lint_doc_test_unknown =
-    unknown `doc(test)` attribute `{$name}`
-
-lint_doc_unknown_any =
-    unknown `doc` attribute `{$name}`
-
-lint_doc_unknown_include =
-    unknown `doc` attribute `include`
-    .suggestion = use `doc = include_str!` instead
-
-lint_doc_unknown_passes =
-    unknown `doc` attribute `{$name}`
-    .note = `doc` attribute `{$name}` no longer functions; see issue #44136 
-    .label = no longer functions
-    .no_op_note = `doc({$name})` is now a no-op
-
-lint_doc_unknown_plugins =
-    unknown `doc` attribute `plugins`
-    .note = `doc` attribute `plugins` no longer functions; see issue #44136  and CVE-2018-1000622 
-    .label = no longer functions
-    .no_op_note = `doc(plugins)` is now a no-op
-
-lint_doc_unknown_spotlight =
-    unknown `doc` attribute `spotlight`
-    .note = `doc(spotlight)` was renamed to `doc(notable_trait)`
-    .suggestion = use `notable_trait` instead
-    .no_op_note = `doc(spotlight)` is now a no-op
-
-
-lint_drop_glue =
-    types that do not implement `Drop` can still have drop glue, consider instead using `{$needs_drop}` to detect whether a type is trivially dropped
-
-lint_drop_trait_constraints =
-    bounds on `{$predicate}` are most likely incorrect, consider instead using `{$needs_drop}` to detect whether a type can be trivially dropped
-
-lint_dropping_copy_types = calls to `std::mem::drop` with a value that implements `Copy` does nothing
-    .label = argument has type `{$arg_ty}`
-
-lint_dropping_references = calls to `std::mem::drop` with a reference instead of an owned value does nothing
-    .label = argument has type `{$arg_ty}`
-
-lint_empty_attribute =
-    unused attribute
-    .suggestion = {$valid_without_list ->
-        [true] remove these parentheses
-        *[other] remove this attribute
-    }
-    .note = {$valid_without_list ->
-        [true] using `{$attr_path}` with an empty list is equivalent to not using a list at all
-        *[other] using `{$attr_path}` with an empty list has no effect
-    }
-
--lint_previously_accepted =
-    this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
-lint_enum_intrinsics_mem_discriminant =
-    the return value of `mem::discriminant` is unspecified when called with a non-enum type
-    .note = the argument to `discriminant` should be a reference to an enum, but it was passed a reference to a `{$ty_param}`, which is not an enum
-
-lint_enum_intrinsics_mem_variant =
-    the return value of `mem::variant_count` is unspecified when called with a non-enum type
-    .note = the type parameter of `variant_count` should be an enum, but it was instantiated with the type `{$ty_param}`, which is not an enum
-
-lint_expectation = this lint expectation is unfulfilled
-    .note = the `unfulfilled_lint_expectations` lint can't be expected and will always produce this message
-    .rationale = {$rationale}
-
-lint_expected_name_value =
-    expected this to be of the form `... = "..."`
-    .warn = {-lint_previously_accepted}
-
-lint_expected_no_args =
-    didn't expect any arguments here
-    .warn = {-lint_previously_accepted}
-
-lint_for_loops_over_fallibles =
-    for loop over {$article} `{$ref_prefix}{$ty}`. This is more readably written as an `if let` statement
-    .suggestion = consider using `if let` to clear intent
-    .remove_next = to iterate over `{$recv_snip}` remove the call to `next`
-    .use_while_let = to check pattern in a loop use `while let`
-    .use_question_mark = consider unwrapping the `Result` with `?` to iterate over its contents
-
-lint_forgetting_copy_types = calls to `std::mem::forget` with a value that implements `Copy` does nothing
-    .label = argument has type `{$arg_ty}`
-
-lint_forgetting_references = calls to `std::mem::forget` with a reference instead of an owned value does nothing
-    .label = argument has type `{$arg_ty}`
-
-lint_function_casts_as_integer = direct cast of function item into an integer
-    .cast_as_fn = first cast to a pointer `as *const ()`
-
-lint_hidden_glob_reexport = private item shadows public glob re-export
-    .note_glob_reexport = the name `{$name}` in the {$namespace} namespace is supposed to be publicly re-exported here
-    .note_private_item = but the private item here shadows it
-
-lint_hidden_lifetime_parameters = hidden lifetime parameters in types are deprecated
-
-lint_identifier_non_ascii_char = identifier contains non-ASCII characters
-
-lint_identifier_uncommon_codepoints = identifier contains {$codepoints_len ->
-    [one] { $identifier_type ->
-        [Exclusion] a character from an archaic script
-        [Technical] a character that is for non-linguistic, specialized usage
-        [Limited_Use] a character from a script in limited use
-        [Not_NFKC] a non normalized (NFKC) character
-        *[other] an uncommon character
-    }
-    *[other] { $identifier_type ->
-        [Exclusion] {$codepoints_len} characters from archaic scripts
-        [Technical] {$codepoints_len} characters that are for non-linguistic, specialized usage
-        [Limited_Use] {$codepoints_len} characters from scripts in limited use
-        [Not_NFKC] {$codepoints_len} non normalized (NFKC) characters
-        *[other] uncommon characters
-    }
-}: {$codepoints}
-    .note = {$codepoints_len ->
-        [one] this character is
-        *[other] these characters are
-    } included in the{$identifier_type ->
-        [Restricted] {""}
-        *[other] {" "}{$identifier_type}
-    } Unicode general security profile
-
-lint_if_let_dtor = {$dtor_kind ->
-    [dyn] value may invoke a custom destructor because it contains a trait object
-    *[concrete] value invokes this custom destructor
-    }
-
-lint_if_let_rescope = `if let` assigns a shorter lifetime since Edition 2024
-    .label = this value has a significant drop implementation which may observe a major change in drop order and requires your discretion
-    .help = the value is now dropped here in Edition 2024
-    .suggestion = a `match` with a single arm can preserve the drop order up to Edition 2021
-
-lint_ignored_unless_crate_specified = {$level}({$name}) is ignored unless specified at crate level
-
-lint_ill_formed_attribute_input = {$num_suggestions ->
-        [1] attribute must be of the form {$suggestions}
-        *[other] valid forms for the attribute are {$suggestions}
-    }
-    .note = for more information, visit <{$docs}>
-
-lint_impl_trait_overcaptures = `{$self_ty}` will capture more lifetimes than possibly intended in edition 2024
-    .note = specifically, {$num_captured ->
-        [one] this lifetime is
-        *[other] these lifetimes are
-     } in scope but not mentioned in the type's bounds
-    .note2 = all lifetimes in scope will be captured by `impl Trait`s in edition 2024
-
-lint_impl_trait_redundant_captures = all possible in-scope parameters are already captured, so `use<...>` syntax is redundant
-    .suggestion = remove the `use<...>` syntax
-
-lint_implicit_sysroot_crate_import = dangerous use of `extern crate {$name}` which is not guaranteed to exist exactly once in the sysroot
-    .help = try using a cargo dependency or using a re-export of the dependency provided by a rustc_* crate
-
-lint_implicit_unsafe_autorefs = implicit autoref creates a reference to the dereference of a raw pointer
-    .note = creating a reference requires the pointer target to be valid and imposes aliasing requirements
-    .raw_ptr = this raw pointer has type `{$raw_ptr_ty}`
-    .autoref = autoref is being applied to this expression, resulting in: `{$autoref_ty}`
-    .overloaded_deref = references are created through calls to explicit `Deref(Mut)::deref(_mut)` implementations
-    .method_def = method calls to `{$method_name}` require a reference
-    .suggestion = try using a raw pointer method instead; or if this reference is intentional, make it explicit
-
-lint_improper_ctypes = `extern` {$desc} uses type `{$ty}`, which is not FFI-safe
-    .label = not FFI-safe
-    .note = the type is defined here
-
-lint_improper_ctypes_array_help = consider passing a pointer to the array
-
-lint_improper_ctypes_array_reason = passing raw arrays by value is not FFI-safe
-lint_improper_ctypes_box = box cannot be represented as a single pointer
-
-lint_improper_ctypes_char_help = consider using `u32` or `libc::wchar_t` instead
-
-lint_improper_ctypes_char_reason = the `char` type has no C equivalent
-
-lint_improper_ctypes_cstr_help =
-    consider passing a `*const std::ffi::c_char` instead, and use `CStr::as_ptr()`
-lint_improper_ctypes_cstr_reason = `CStr`/`CString` do not have a guaranteed layout
-
-lint_improper_ctypes_dyn = trait objects have no C equivalent
-
-lint_improper_ctypes_enum_repr_help =
-    consider adding a `#[repr(C)]`, `#[repr(transparent)]`, or integer `#[repr(...)]` attribute to this enum
-
-lint_improper_ctypes_enum_repr_reason = enum has no representation hint
-lint_improper_ctypes_fnptr_help = consider using an `extern fn(...) -> ...` function pointer instead
-
-lint_improper_ctypes_fnptr_reason = this function pointer has Rust-specific calling convention
-lint_improper_ctypes_non_exhaustive = this enum is non-exhaustive
-lint_improper_ctypes_non_exhaustive_variant = this enum has non-exhaustive variants
-
-lint_improper_ctypes_only_phantomdata = composed only of `PhantomData`
-
-lint_improper_ctypes_opaque = opaque types have no C equivalent
-
-lint_improper_ctypes_slice_help = consider using a raw pointer instead
-
-lint_improper_ctypes_slice_reason = slices have no C equivalent
-lint_improper_ctypes_str_help = consider using `*const u8` and a length instead
-
-lint_improper_ctypes_str_reason = string slices have no C equivalent
-lint_improper_ctypes_struct_fieldless_help = consider adding a member to this struct
-
-lint_improper_ctypes_struct_fieldless_reason = this struct has no fields
-lint_improper_ctypes_struct_layout_help = consider adding a `#[repr(C)]` or `#[repr(transparent)]` attribute to this struct
-
-lint_improper_ctypes_struct_layout_reason = this struct has unspecified layout
-lint_improper_ctypes_struct_non_exhaustive = this struct is non-exhaustive
-lint_improper_ctypes_struct_zst = this struct contains only zero-sized fields
-
-lint_improper_ctypes_tuple_help = consider using a struct instead
-
-lint_improper_ctypes_tuple_reason = tuples have unspecified layout
-lint_improper_ctypes_union_fieldless_help = consider adding a member to this union
-
-lint_improper_ctypes_union_fieldless_reason = this union has no fields
-lint_improper_ctypes_union_layout_help = consider adding a `#[repr(C)]` or `#[repr(transparent)]` attribute to this union
-
-lint_improper_ctypes_union_layout_reason = this union has unspecified layout
-lint_improper_ctypes_union_non_exhaustive = this union is non-exhaustive
-
-lint_improper_ctypes_unsafe_binder = unsafe binders are incompatible with foreign function interfaces
-
-lint_improper_gpu_kernel_arg = passing type `{$ty}` to a function with "gpu-kernel" ABI may have unexpected behavior
-    .help = use primitive types and raw pointers to get reliable behavior
-
-lint_incorrect_do_not_recommend_args =
-    `#[diagnostic::do_not_recommend]` does not expect any arguments
-
-lint_int_to_ptr_transmutes = transmuting an integer to a pointer creates a pointer without provenance
-    .note = this is dangerous because dereferencing the resulting pointer is undefined behavior
-    .note_exposed_provenance = exposed provenance semantics can be used to create a pointer based on some previously exposed provenance
-    .help_transmute = for more information about transmute, see 
-    .help_exposed_provenance = for more information about exposed provenance, see 
-    .suggestion_with_exposed_provenance = use `std::ptr::with_exposed_provenance{$suffix}` instead to use a previously exposed provenance
-    .suggestion_without_provenance_mut = if you truly mean to create a pointer without provenance, use `std::ptr::without_provenance_mut`
-
-lint_invalid_asm_label_binary = avoid using labels containing only the digits `0` and `1` in inline assembly
-    .label = use a different label that doesn't start with `0` or `1`
-    .help = start numbering with `2` instead
-    .note1 = an LLVM bug makes these labels ambiguous with a binary literal number on x86
-    .note2 = see  for more information
-
-lint_invalid_asm_label_format_arg = avoid using named labels in inline assembly
-    .help = only local labels of the form `:` should be used in inline asm
-    .note1 = format arguments may expand to a non-numeric value
-    .note2 = see the asm section of Rust By Example  for more information
-lint_invalid_asm_label_named = avoid using named labels in inline assembly
-    .help = only local labels of the form `:` should be used in inline asm
-    .note = see the asm section of Rust By Example  for more information
-lint_invalid_asm_label_no_span = the label may be declared in the expansion of a macro
-
-lint_invalid_crate_type_value = invalid `crate_type` value
-    .suggestion = did you mean
-
-# FIXME: we should ordinalize $valid_up_to when we add support for doing so
-lint_invalid_from_utf8_checked = calls to `{$method}` with an invalid literal always return an error
-    .label = the literal was valid UTF-8 up to the {$valid_up_to} bytes
-
-# FIXME: we should ordinalize $valid_up_to when we add support for doing so
-lint_invalid_from_utf8_unchecked = calls to `{$method}` with an invalid literal are undefined behavior
-    .label = the literal was valid UTF-8 up to the {$valid_up_to} bytes
-
-lint_invalid_nan_comparisons_eq_ne = incorrect NaN comparison, NaN cannot be directly compared to itself
-    .suggestion = use `f32::is_nan()` or `f64::is_nan()` instead
-
-lint_invalid_nan_comparisons_lt_le_gt_ge = incorrect NaN comparison, NaN is not orderable
-
-lint_invalid_null_arguments = calling this function with a null pointer is undefined behavior, even if the result of the function is unused
-    .origin = null pointer originates from here
-    .doc = for more information, visit  and 
-
-lint_invalid_reference_casting_assign_to_ref = assigning to `&T` is undefined behavior, consider using an `UnsafeCell`
-    .label = casting happened here
-
-lint_invalid_reference_casting_bigger_layout = casting references to a bigger memory layout than the backing allocation is undefined behavior, even if the reference is unused
-    .label = casting happened here
-    .alloc = backing allocation comes from here
-    .layout = casting from `{$from_ty}` ({$from_size} bytes) to `{$to_ty}` ({$to_size} bytes)
-
-lint_invalid_reference_casting_borrow_as_mut = casting `&T` to `&mut T` is undefined behavior, even if the reference is unused, consider instead using an `UnsafeCell`
-    .label = casting happened here
-
-lint_invalid_reference_casting_note_book = for more information, visit 
-
-lint_invalid_reference_casting_note_ty_has_interior_mutability = even for types with interior mutability, the only legal way to obtain a mutable pointer from a shared reference is through `UnsafeCell::get`
-
-lint_invalid_style = {$is_used_as_inner ->
-        [false] crate-level attribute should be an inner attribute: add an exclamation mark: `#![{$name}]`
-        *[other] the `#![{$name}]` attribute can only be used at the crate root
-    }
-    .note = this attribute does not have an `!`, which means it is applied to this {$target}
-
-lint_invalid_target = `#[{$name}]` attribute cannot be used on {$target}
-    .warn = {-lint_previously_accepted}
-    .help = `#[{$name}]` can {$only}be applied to {$applied}
-    .suggestion = remove the attribute
-
-lint_lintpass_by_hand = implementing `LintPass` by hand
-    .help = try using `declare_lint_pass!` or `impl_lint_pass!` instead
-
-lint_macro_expr_fragment_specifier_2024_migration =
-    the `expr` fragment specifier will accept more expressions in the 2024 edition
-    .suggestion = to keep the existing behavior, use the `expr_2021` fragment specifier
-
-lint_malformed_attribute = malformed lint attribute input
-
-lint_malformed_doc =
-    malformed `doc` attribute input
-    .warn = {-lint_previously_accepted}
-
-lint_map_unit_fn = `Iterator::map` call that discard the iterator's values
-    .note = `Iterator::map`, like many of the methods on `Iterator`, gets executed lazily, meaning that its effects won't be visible until it is iterated
-    .function_label = this function returns `()`, which is likely not what you wanted
-    .argument_label = called `Iterator::map` with callable that returns `()`
-    .map_label = after this call to map, the resulting iterator is `impl Iterator`, which means the only information carried by the iterator is the number of items
-    .suggestion = you might have meant to use `Iterator::for_each`
-
-lint_mismatched_lifetime_syntaxes_eliding_while_named =
-    eliding a lifetime that's named elsewhere is confusing
-
-lint_mismatched_lifetime_syntaxes_help =
-    the same lifetime is referred to in inconsistent ways, making the signature confusing
-
-lint_mismatched_lifetime_syntaxes_hiding_and_eliding_while_named =
-    hiding or eliding a lifetime that's named elsewhere is confusing
-
-lint_mismatched_lifetime_syntaxes_hiding_while_elided =
-    hiding a lifetime that's elided elsewhere is confusing
-
-lint_mismatched_lifetime_syntaxes_hiding_while_named =
-    hiding a lifetime that's named elsewhere is confusing
-
-lint_mismatched_lifetime_syntaxes_input_elided =
-    the lifetime is elided here
-
-lint_mismatched_lifetime_syntaxes_input_hidden =
-    the lifetime is hidden here
-
-lint_mismatched_lifetime_syntaxes_input_named =
-    the lifetime is named here
-
-lint_mismatched_lifetime_syntaxes_output_elided =
-    the same lifetime is elided here
-
-lint_mismatched_lifetime_syntaxes_output_hidden =
-    the same lifetime is hidden here
-
-lint_mismatched_lifetime_syntaxes_output_named =
-    the same lifetime is named here
-
-lint_mismatched_lifetime_syntaxes_suggestion_explicit =
-    consistently use `{$lifetime_name}`
-
-lint_mismatched_lifetime_syntaxes_suggestion_implicit =
-    remove the lifetime name from references
-
-lint_mismatched_lifetime_syntaxes_suggestion_mixed =
-    remove the lifetime name from references and use `'_` for type paths
-
-lint_mismatched_lifetime_syntaxes_suggestion_mixed_only_paths =
-    use `'_` for type paths
-
-lint_missing_gpu_kernel_export_name = function with the "gpu-kernel" ABI has a mangled name
-    .note = mangled names make it hard to find the kernel, this is usually not intended
-    .help = use `unsafe(no_mangle)` or `unsafe(export_name = "")`
-
-lint_mixed_script_confusables =
-    the usage of Script Group `{$set}` in this crate consists solely of mixed script confusables
-    .includes_note = the usage includes {$includes}
-    .note = please recheck to make sure their usages are indeed what you want
-
-lint_multiple_supertrait_upcastable = `{$ident}` is dyn-compatible and has multiple supertraits
-
-lint_named_argument_used_positionally = named argument `{$named_arg_name}` is not used by name
-    .label_named_arg = this named argument is referred to by position in formatting string
-    .label_position_arg = this formatting argument uses named argument `{$named_arg_name}` by position
-    .suggestion = use the named argument by name to avoid ambiguity
-
-lint_node_source = `forbid` level set here
-    .note = {$reason}
-
-lint_non_binding_let_multi_drop_fn =
-    consider immediately dropping the value using `drop(..)` after the `let` statement
-
-lint_non_binding_let_multi_suggestion =
-    consider immediately dropping the value
-
-lint_non_binding_let_on_drop_type =
-    non-binding let on a type that has a destructor
-
-lint_non_binding_let_on_sync_lock = non-binding let on a synchronization lock
-    .label = this lock is not assigned to a binding and is immediately dropped
-
-lint_non_binding_let_suggestion =
-    consider binding to an unused variable to avoid immediately dropping the value
-
-lint_non_camel_case_type = {$sort} `{$name}` should have an upper camel case name
-    .suggestion = convert the identifier to upper camel case
-    .label = should have an UpperCamelCase name
-
-lint_non_fmt_panic = panic message is not a string literal
-    .note = this usage of `{$name}!()` is deprecated; it will be a hard error in Rust 2021
-    .more_info_note = for more information, see 
-    .supports_fmt_note = the `{$name}!()` macro supports formatting, so there's no need for the `format!()` macro here
-    .supports_fmt_suggestion = remove the `format!(..)` macro call
-    .display_suggestion = add a "{"{"}{"}"}" format string to `Display` the message
-    .debug_suggestion =
-        add a "{"{"}:?{"}"}" format string to use the `Debug` implementation of `{$ty}`
-    .panic_suggestion = {$already_suggested ->
-        [true] or use
-        *[false] use
-    } std::panic::panic_any instead
-
-lint_non_fmt_panic_braces =
-    panic message contains {$count ->
-        [one] a brace
-        *[other] braces
-    }
-    .note = this message is not used as a format string, but will be in Rust 2021
-    .suggestion = add a "{"{"}{"}"}" format string to use the message literally
-
-lint_non_fmt_panic_unused =
-    panic message contains {$count ->
-        [one] an unused
-        *[other] unused
-    } formatting {$count ->
-        [one] placeholder
-        *[other] placeholders
-    }
-    .note = this message is not used as a format string when given without arguments, but will be in Rust 2021
-    .add_args_suggestion = add the missing {$count ->
-        [one] argument
-        *[other] arguments
-    }
-    .add_fmt_suggestion = or add a "{"{"}{"}"}" format string to use the message literally
-
-lint_non_glob_import_type_ir_inherent = non-glob import of `rustc_type_ir::inherent`
-    .suggestion = try using a glob import instead
-
-lint_non_local_definitions_cargo_update = the {$macro_kind} `{$macro_name}` may come from an old version of the `{$crate_name}` crate, try updating your dependency with `cargo update -p {$crate_name}`
-
-lint_non_local_definitions_impl = non-local `impl` definition, `impl` blocks should be written at the same level as their item
-    .non_local = an `impl` is never scoped, even when it is nested inside an item, as it may impact type checking outside of that item, which can be the case if neither the trait or the self type are at the same nesting level as the `impl`
-    .doctest = make this doc-test a standalone test with its own `fn main() {"{"} ... {"}"}`
-    .exception = items in an anonymous const item (`const _: () = {"{"} ... {"}"}`) are treated as in the same scope as the anonymous const's declaration for the purpose of this lint
-    .const_anon = use a const-anon item to suppress this lint
-    .macro_to_change = the {$macro_kind} `{$macro_to_change}` defines the non-local `impl`, and may need to be changed
-
-lint_non_local_definitions_impl_move_help =
-    move the `impl` block outside of this {$body_kind_descr} {$depth ->
-        [one] `{$body_name}`
-       *[other] `{$body_name}` and up {$depth} bodies
-    }
-
-lint_non_local_definitions_macro_rules = non-local `macro_rules!` definition, `#[macro_export]` macro should be written at top level module
-    .help =
-        remove the `#[macro_export]` or move this `macro_rules!` outside the of the current {$body_kind_descr} {$depth ->
-            [one] `{$body_name}`
-           *[other] `{$body_name}` and up {$depth} bodies
-        }
-    .help_doctest =
-        remove the `#[macro_export]` or make this doc-test a standalone test with its own `fn main() {"{"} ... {"}"}`
-    .non_local = a `macro_rules!` definition is non-local if it is nested inside an item and has a `#[macro_export]` attribute
-
-lint_non_snake_case = {$sort} `{$name}` should have a snake case name
-    .rename_or_convert_suggestion = rename the identifier or convert it to a snake case raw identifier
-    .cannot_convert_note = `{$sc}` cannot be used as a raw identifier
-    .rename_suggestion = rename the identifier
-    .convert_suggestion = convert the identifier to snake case
-    .help = convert the identifier to snake case: `{$sc}`
-    .label = should have a snake_case name
-
-lint_non_upper_case_global = {$sort} `{$name}` should have an upper case name
-    .suggestion = convert the identifier to upper case
-    .label = should have an UPPER_CASE name
-
-lint_noop_method_call = call to `.{$method}()` on a reference in this situation does nothing
-    .suggestion = remove this redundant call
-    .note = the type `{$orig_ty}` does not implement `{$trait_}`, so calling `{$method}` on `&{$orig_ty}` copies the reference, which does not do anything and can be removed
-    .derive_suggestion = if you meant to clone `{$orig_ty}`, implement `Clone` for it
-
-lint_only_cast_u8_to_char = only `u8` can be cast into `char`
-    .suggestion = use a `char` literal instead
-
-lint_opaque_hidden_inferred_bound = opaque type `{$ty}` does not satisfy its associated type bounds
-    .specifically = this associated type bound is unsatisfied for `{$proj_ty}`
-
-lint_opaque_hidden_inferred_bound_sugg = add this bound
-
-lint_overflowing_bin_hex = literal out of range for `{$ty}`
-    .negative_note = the literal `{$lit}` (decimal `{$dec}`) does not fit into the type `{$ty}`
-    .negative_becomes_note = and the value `-{$lit}` will become `{$actually}{$ty}`
-    .positive_note = the literal `{$lit}` (decimal `{$dec}`) does not fit into the type `{$ty}` and will become `{$actually}{$ty}`
-    .suggestion = consider using the type `{$suggestion_ty}` instead
-    .sign_bit_suggestion = to use as a negative number (decimal `{$negative_val}`), consider using the type `{$uint_ty}` for the literal and cast it to `{$int_ty}`
-    .help = consider using the type `{$suggestion_ty}` instead
-
-lint_overflowing_int = literal out of range for `{$ty}`
-    .note = the literal `{$lit}` does not fit into the type `{$ty}` whose range is `{$min}..={$max}`
-    .help = consider using the type `{$suggestion_ty}` instead
-
-lint_overflowing_literal = literal out of range for `{$ty}`
-    .note = the literal `{$lit}` does not fit into the type `{$ty}` and will be converted to `{$ty}::INFINITY`
-
-lint_overflowing_uint = literal out of range for `{$ty}`
-    .note = the literal `{$lit}` does not fit into the type `{$ty}` whose range is `{$min}..={$max}`
-
-lint_overruled_attribute = {$lint_level}({$lint_source}) incompatible with previous forbid
-    .label = overruled by previous forbid
-
-lint_pass_by_value = passing `{$ty}` by reference
-    .suggestion = try passing by value
-
-lint_path_statement_drop = path statement drops value
-    .suggestion = use `drop` to clarify the intent
-
-lint_path_statement_no_effect = path statement with no effect
-
-lint_pattern_in_bodiless = patterns aren't allowed in functions without bodies
-    .label = pattern not allowed in function without body
-
-lint_pattern_in_foreign = patterns aren't allowed in foreign function declarations
-    .label = pattern not allowed in foreign function
-
-lint_query_instability = using `{$query}` can result in unstable query results
-    .note = if you believe this case to be fine, allow this lint and add a comment explaining your rationale
-
-lint_query_untracked = `{$method}` accesses information that is not tracked by the query system
-    .note = if you believe this case to be fine, allow this lint and add a comment explaining your rationale
-
-lint_range_endpoint_out_of_range = range endpoint is out of range for `{$ty}`
-
-lint_range_use_inclusive_range = use an inclusive range instead
-
-
-lint_raw_prefix = prefix `'r` is reserved
-    .label = reserved prefix
-    .suggestion = insert whitespace here to avoid this being parsed as a prefix in Rust 2021
-
-lint_reason_must_be_string_literal = reason must be a string literal
-
-lint_reason_must_come_last = reason in lint attribute must come last
-
-lint_redundant_import = the item `{$ident}` is imported redundantly
-    .label_imported_here = the item `{$ident}` is already imported here
-    .label_defined_here = the item `{$ident}` is already defined here
-    .label_imported_prelude = the item `{$ident}` is already imported by the extern prelude
-    .label_defined_prelude = the item `{$ident}` is already defined by the extern prelude
-
-lint_redundant_semicolons =
-    unnecessary trailing {$multiple ->
-        [true] semicolons
-        *[false] semicolon
-    }
-
-lint_redundant_semicolons_suggestion = remove {$multiple_semicolons ->
-        [true] these semicolons
-        *[false] this semicolon
-    }
-
-lint_remove_mut_from_pattern = remove `mut` from the parameter
-
-lint_removed_lint = lint `{$name}` has been removed: {$reason}
-
-lint_renamed_lint = lint `{$name}` has been renamed to `{$replace}`
-    .suggestion = use the new name
-    .help = use the new name `{$replace}`
-
-lint_requested_level = requested on the command line with `{$level} {$lint_name}`
-
-lint_reserved_multihash = reserved token in Rust 2024
-    .suggestion = insert whitespace here to avoid this being parsed as a forbidden token in Rust 2024
-
-lint_reserved_prefix = prefix `{$prefix}` is unknown
-    .label = unknown prefix
-    .suggestion = insert whitespace here to avoid this being parsed as a prefix in Rust 2021
-
-lint_reserved_string = will be parsed as a guarded string in Rust 2024
-    .suggestion = insert whitespace here to avoid this being parsed as a guarded string in Rust 2024
-
-lint_shadowed_into_iter =
-    this method call resolves to `<&{$target} as IntoIterator>::into_iter` (due to backwards compatibility), but will resolve to `<{$target} as IntoIterator>::into_iter` in Rust {$edition}
-    .use_iter_suggestion = use `.iter()` instead of `.into_iter()` to avoid ambiguity
-    .remove_into_iter_suggestion = or remove `.into_iter()` to iterate by value
-    .use_explicit_into_iter_suggestion =
-        or use `IntoIterator::into_iter(..)` instead of `.into_iter()` to explicitly iterate by value
-
-lint_single_use_lifetime = lifetime parameter `{$ident}` only used once
-    .label_param = this lifetime...
-    .label_use = ...is used only here
-    .suggestion = elide the single-use lifetime
-
-lint_span_use_eq_ctxt = use `.eq_ctxt()` instead of `.ctxt() == .ctxt()`
-
-lint_static_mut_refs_lint = creating a {$shared_label}reference to mutable static
-    .label = {$shared_label}reference to mutable static
-    .suggestion = use `&raw const` instead to create a raw pointer
-    .suggestion_mut = use `&raw mut` instead to create a raw pointer
-    .shared_note = shared references to mutable statics are dangerous; it's undefined behavior if the static is mutated or if a mutable reference is created for it while the shared reference lives
-    .mut_note = mutable references to mutable statics are dangerous; it's undefined behavior if any other pointer to the static is used or if any other reference is created for the static while the mutable reference lives
-
-lint_supertrait_as_deref_target = this `Deref` implementation is covered by an implicit supertrait coercion
-    .label = `{$self_ty}` implements `Deref` which conflicts with supertrait `{$supertrait_principal}`
-    .label2 = target type is a supertrait of `{$self_ty}`
-    .help = consider removing this implementation or replacing it with a method instead
-
-lint_surrogate_char_cast = surrogate values are not valid for `char`
-    .note = `0xD800..=0xDFFF` are reserved for Unicode surrogates and are not valid `char` values
-
-lint_suspicious_double_ref_clone =
-    using `.clone()` on a double reference, which returns `{$ty}` instead of cloning the inner type
-
-lint_suspicious_double_ref_deref =
-    using `.deref()` on a double reference, which returns `{$ty}` instead of dereferencing the inner type
-
-lint_symbol_intern_string_literal = using `Symbol::intern` on a string literal
-    .help = consider adding the symbol to `compiler/rustc_span/src/symbol.rs`
-
-lint_too_large_char_cast = value exceeds maximum `char` value
-    .note = maximum valid `char` value is `0x10FFFF`
-
-lint_ty_qualified = usage of qualified `ty::{$ty}`
-    .suggestion = try importing it and using it unqualified
-
-lint_tykind = usage of `ty::TyKind`
-    .help = try using `Ty` instead
-
-lint_tykind_kind = usage of `ty::TyKind::`
-    .suggestion = try using `ty::` directly
-
-lint_type_ir_direct_use = do not use `rustc_type_ir` unless you are implementing type system internals
-    .note = use `rustc_middle::ty` instead
-
-lint_type_ir_inherent_usage = do not use `rustc_type_ir::inherent` unless you're inside of the trait solver
-    .note = the method or struct you're looking for is likely defined somewhere else downstream in the compiler
-
-lint_type_ir_trait_usage = do not use `rustc_type_ir::Interner` or `rustc_type_ir::InferCtxtLike` unless you're inside of the trait solver
-    .note = the method or struct you're looking for is likely defined somewhere else downstream in the compiler
-
-lint_undefined_transmute = pointers cannot be transmuted to integers during const eval
-    .note = at compile-time, pointers do not have an integer value
-    .note2 = avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
-    .help = for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
-
-lint_undropped_manually_drops = calls to `std::mem::drop` with `std::mem::ManuallyDrop` instead of the inner value does nothing
-    .label = argument has type `{$arg_ty}`
-    .suggestion = use `std::mem::ManuallyDrop::into_inner` to get the inner value
-
-lint_unexpected_cfg_add_build_rs_println = or consider adding `{$build_rs_println}` to the top of the `build.rs`
-lint_unexpected_cfg_add_cargo_feature = consider using a Cargo feature instead
-lint_unexpected_cfg_add_cargo_toml_lint_cfg = or consider adding in `Cargo.toml` the `check-cfg` lint config for the lint:{$cargo_toml_lint_cfg}
-lint_unexpected_cfg_add_cmdline_arg = to expect this configuration use `{$cmdline_arg}`
-lint_unexpected_cfg_boolean = you may have meant to use `{$literal}` (notice the capitalization). Doing so makes this predicate evaluate to `{$literal}` unconditionally
-lint_unexpected_cfg_cargo_update = the {$macro_kind} `{$macro_name}` may come from an old version of the `{$crate_name}` crate, try updating your dependency with `cargo update -p {$crate_name}`
-
-lint_unexpected_cfg_define_features = consider defining some features in `Cargo.toml`
-lint_unexpected_cfg_doc_cargo = see  for more information about checking conditional configuration
-lint_unexpected_cfg_doc_rustc = see  for more information about checking conditional configuration
-
-lint_unexpected_cfg_from_external_macro_origin = using a cfg inside a {$macro_kind} will use the cfgs from the destination crate and not the ones from the defining crate
-lint_unexpected_cfg_from_external_macro_refer = try referring to `{$macro_name}` crate for guidance on how handle this unexpected cfg
-lint_unexpected_cfg_name = unexpected `cfg` condition name: `{$name}`
-lint_unexpected_cfg_name_expected_names = expected names are: {$possibilities}{$and_more ->
-        [0] {""}
-        *[other] {" "}and {$and_more} more
-    }
-lint_unexpected_cfg_name_expected_values = expected values for `{$best_match}` are: {$possibilities}
-lint_unexpected_cfg_name_similar_name = there is a config with a similar name
-lint_unexpected_cfg_name_similar_name_different_values = there is a config with a similar name and different values
-lint_unexpected_cfg_name_similar_name_no_value = there is a config with a similar name and no value
-lint_unexpected_cfg_name_similar_name_value = there is a config with a similar name and value
-lint_unexpected_cfg_name_version_syntax = there is a similar config predicate: `version("..")`
-lint_unexpected_cfg_name_with_similar_value = found config with similar value
-
-lint_unexpected_cfg_value = unexpected `cfg` condition value: {$has_value ->
-        [true] `{$value}`
-        *[false] (none)
-    }
-lint_unexpected_cfg_value_add_feature = consider adding `{$value}` as a feature in `Cargo.toml`
-lint_unexpected_cfg_value_expected_values = expected values for `{$name}` are: {$have_none_possibility ->
-        [true] {"(none), "}
-        *[false] {""}
-    }{$possibilities}{$and_more ->
-        [0] {""}
-        *[other] {" "}and {$and_more} more
-    }
-lint_unexpected_cfg_value_no_expected_value = no expected value for `{$name}`
-lint_unexpected_cfg_value_no_expected_values = no expected values for `{$name}`
-lint_unexpected_cfg_value_remove_condition = remove the condition
-lint_unexpected_cfg_value_remove_value = remove the value
-lint_unexpected_cfg_value_similar_name = there is a expected value with a similar name
-lint_unexpected_cfg_value_specify_value = specify a config value
-
-lint_ungated_async_fn_track_caller = `#[track_caller]` on async functions is a no-op
-     .label = this function will not propagate the caller location
-
-lint_unicode_text_flow = unicode codepoint changing visible direction of text present in comment
-    .label = {$num_codepoints ->
-            [1] this comment contains an invisible unicode text flow control codepoint
-            *[other] this comment contains invisible unicode text flow control codepoints
-        }
-    .note = these kind of unicode codepoints change the way text flows on applications that support them, but can cause confusion because they change the order of characters on the screen
-    .suggestion = if their presence wasn't intentional, you can remove them
-    .label_comment_char = {$c_debug}
-
-
-lint_unit_bindings = binding has unit type `()`
-    .label = this pattern is inferred to be the unit type `()`
-
-lint_unknown_gated_lint =
-    unknown lint: `{$name}`
-    .note = the `{$name}` lint is unstable
-
-lint_unknown_lint =
-    unknown lint: `{$name}`
-    .suggestion = {$from_rustc ->
-        [true] a lint with a similar name exists in `rustc` lints
-        *[false] did you mean
-    }
-    .help = {$from_rustc ->
-        [true] a lint with a similar name exists in `rustc` lints: `{$replace}`
-        *[false] did you mean: `{$replace}`
-    }
-
-lint_unknown_tool_in_scoped_lint = unknown tool name `{$tool_name}` found in scoped lint: `{$tool_name}::{$lint_name}`
-    .help = add `#![register_tool({$tool_name})]` to the crate root
-
-lint_unnecessary_qualification = unnecessary qualification
-    .suggestion = remove the unnecessary path segments
-
-lint_unpredictable_fn_pointer_comparisons = function pointer comparisons do not produce meaningful results since their addresses are not guaranteed to be unique
-    .note_duplicated_fn = the address of the same function can vary between different codegen units
-    .note_deduplicated_fn = furthermore, different functions could have the same address after being merged together
-    .note_visit_fn_addr_eq = for more information visit 
-    .fn_addr_eq_suggestion = refactor your code, or use `std::ptr::fn_addr_eq` to suppress the lint
-
-lint_unqualified_local_imports = `use` of a local item without leading `self::`, `super::`, or `crate::`
-
-lint_unreachable_cfg_select_predicate = unreachable configuration predicate
-    .label = this configuration predicate is never reached
-
-lint_unreachable_cfg_select_predicate_wildcard = unreachable configuration predicate
-    .label = always matches
-    .label2 = this configuration predicate is never reached
-
-lint_unsafe_attr_outside_unsafe = unsafe attribute used without unsafe
-    .label = usage of unsafe attribute
-lint_unsafe_attr_outside_unsafe_suggestion = wrap the attribute in `unsafe(...)`
-
-lint_unsupported_group = `{$lint_group}` lint group is not supported with ´--force-warn´
-
-lint_unused_allocation = unnecessary allocation, use `&` instead
-lint_unused_allocation_mut = unnecessary allocation, use `&mut` instead
-
-lint_unused_closure =
-    unused {$pre}{$count ->
-        [one] closure
-        *[other] closures
-    }{$post} that must be used
-    .note = closures are lazy and do nothing unless called
-
-lint_unused_comparisons = comparison is useless due to type limits
-
-lint_unused_coroutine =
-    unused {$pre}{$count ->
-        [one] coroutine
-        *[other] coroutine
-    }{$post} that must be used
-    .note = coroutines are lazy and do nothing unless resumed
-
-lint_unused_crate_dependency = extern crate `{$extern_crate}` is unused in crate `{$local_crate}`
-    .help = remove the dependency or add `use {$extern_crate} as _;` to the crate root
-
-lint_unused_def = unused {$pre}`{$def}`{$post} that must be used
-    .suggestion = use `let _ = ...` to ignore the resulting value
-
-lint_unused_delim = unnecessary {$delim} around {$item}
-    .suggestion = remove these {$delim}
-
-lint_unused_duplicate =
-    unused attribute
-    .suggestion = remove this attribute
-    .note = attribute also specified here
-    .warn = {-lint_previously_accepted}
-
-lint_unused_import_braces = braces around {$node} is unnecessary
-
-lint_unused_imports = {$num_snippets ->
-        [one] unused import: {$span_snippets}
-        *[other] unused imports: {$span_snippets}
-    }
-    .suggestion_remove_whole_use = remove the whole `use` item
-    .suggestion_remove_imports = {$num_to_remove ->
-            [one] remove the unused import
-            *[other] remove the unused imports
-        }
-    .help = if this is a test module, consider adding a `#[cfg(test)]` to the containing module
-
-lint_unused_lifetime = lifetime parameter `{$ident}` never used
-    .suggestion = elide the unused lifetime
-
-lint_unused_op = unused {$op} that must be used
-    .label = the {$op} produces a value
-    .suggestion = use `let _ = ...` to ignore the resulting value
-
-lint_unused_result = unused result of type `{$ty}`
-
-lint_unused_visibilities = visibility qualifiers have no effect on `const _` declarations
-    .note = `const _` does not declare a name, so there is nothing for the qualifier to apply to
-    .suggestion = remove the qualifier
-
-lint_use_let_underscore_ignore_suggestion = use `let _ = ...` to ignore the expression or result
-
-lint_useless_ptr_null_checks_fn_ptr = function pointers are not nullable, so checking them for null will always return false
-    .help = wrap the function pointer inside an `Option` and use `Option::is_none` to check for null pointer value
-    .label = expression has type `{$orig_ty}`
-
-lint_useless_ptr_null_checks_fn_ret = returned pointer of `{$fn_name}` call is never null, so checking it for null will always return false
-
-lint_useless_ptr_null_checks_ref = references are not nullable, so checking them for null will always return false
-    .label = expression has type `{$orig_ty}`
-
-lint_uses_power_alignment = repr(C) does not follow the power alignment rule. This may affect platform C ABI compatibility for this type
-
-lint_variant_size_differences =
-    enum variant is more than three times larger ({$largest} bytes) than the next largest
diff --git a/compiler/rustc_lint/src/async_closures.rs b/compiler/rustc_lint/src/async_closures.rs
index 02fb22bf782e..0d3a954667f7 100644
--- a/compiler/rustc_lint/src/async_closures.rs
+++ b/compiler/rustc_lint/src/async_closures.rs
@@ -108,16 +108,18 @@ impl<'tcx> LateLintPass<'tcx> for AsyncClosureUsage {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_closure_returning_async_block)]
+#[diag("closure returning async block can be made into an async closure")]
 struct ClosureReturningAsyncBlock {
-    #[label]
+    #[label(
+        "this async block can be removed, and the closure can be turned into an async closure"
+    )]
     async_decl_span: Span,
     #[subdiagnostic]
     sugg: AsyncClosureSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion("turn this into an async closure", applicability = "maybe-incorrect")]
 struct AsyncClosureSugg {
     #[suggestion_part(code = "")]
     deletion_span: Span,
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index e2a061cab680..ef21b3f573be 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -22,7 +22,7 @@ use rustc_ast::visit::{FnCtxt, FnKind};
 use rustc_ast::{self as ast, *};
 use rustc_ast_pretty::pprust::expr_to_string;
 use rustc_attr_parsing::AttributeParser;
-use rustc_errors::{Applicability, LintDiagnostic};
+use rustc_errors::{Applicability, LintDiagnostic, inline_fluent};
 use rustc_feature::GateIssue;
 use rustc_hir as hir;
 use rustc_hir::attrs::{AttributeKind, DocAttribute};
@@ -61,10 +61,7 @@ use crate::lints::{
     BuiltinUnreachablePub, BuiltinUnsafe, BuiltinUnstableFeatures, BuiltinUnusedDocComment,
     BuiltinUnusedDocCommentSub, BuiltinWhileTrue, InvalidAsmLabel,
 };
-use crate::{
-    EarlyContext, EarlyLintPass, LateContext, LateLintPass, Level, LintContext,
-    fluent_generated as fluent,
-};
+use crate::{EarlyContext, EarlyLintPass, LateContext, LateLintPass, Level, LintContext};
 declare_lint! {
     /// The `while_true` lint detects `while true { }`.
     ///
@@ -2655,8 +2652,12 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue {
             let conjured_ty = cx.typeck_results().expr_ty(expr);
             if let Some(err) = with_no_trimmed_paths!(ty_find_init_error(cx, conjured_ty, init)) {
                 let msg = match init {
-                    InitKind::Zeroed => fluent::lint_builtin_unpermitted_type_init_zeroed,
-                    InitKind::Uninit => fluent::lint_builtin_unpermitted_type_init_uninit,
+                    InitKind::Zeroed => {
+                        inline_fluent!("the type `{$ty}` does not permit zero-initialization")
+                    }
+                    InitKind::Uninit => {
+                        inline_fluent!("the type `{$ty}` does not permit being left uninitialized")
+                    }
                 };
                 let sub = BuiltinUnpermittedTypeInitSub { err };
                 cx.emit_span_lint(
diff --git a/compiler/rustc_lint/src/errors.rs b/compiler/rustc_lint/src/errors.rs
index 8f2f3594161a..b0154bed1a51 100644
--- a/compiler/rustc_lint/src/errors.rs
+++ b/compiler/rustc_lint/src/errors.rs
@@ -1,17 +1,15 @@
 use rustc_errors::codes::*;
-use rustc_errors::{Diag, EmissionGuarantee, Subdiagnostic};
+use rustc_errors::{Diag, EmissionGuarantee, Subdiagnostic, inline_fluent};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_session::lint::Level;
 use rustc_span::{Span, Symbol};
 
-use crate::fluent_generated as fluent;
-
 #[derive(Diagnostic)]
-#[diag(lint_overruled_attribute, code = E0453)]
+#[diag("{$lint_level}({$lint_source}) incompatible with previous forbid", code = E0453)]
 pub(crate) struct OverruledAttribute<'a> {
     #[primary_span]
     pub span: Span,
-    #[label]
+    #[label("overruled by previous forbid")]
     pub overruled: Span,
     pub lint_level: &'a str,
     pub lint_source: Symbol,
@@ -29,24 +27,24 @@ impl Subdiagnostic for OverruledAttributeSub {
     fn add_to_diag(self, diag: &mut Diag<'_, G>) {
         match self {
             OverruledAttributeSub::DefaultSource { id } => {
-                diag.note(fluent::lint_default_source);
+                diag.note(inline_fluent!("`forbid` lint level is the default for {$id}"));
                 diag.arg("id", id);
             }
             OverruledAttributeSub::NodeSource { span, reason } => {
-                diag.span_label(span, fluent::lint_node_source);
+                diag.span_label(span, inline_fluent!("`forbid` level set here"));
                 if let Some(rationale) = reason {
                     diag.note(rationale.to_string());
                 }
             }
             OverruledAttributeSub::CommandLineSource => {
-                diag.note(fluent::lint_command_line_source);
+                diag.note(inline_fluent!("`forbid` lint level was set on command line"));
             }
         }
     }
 }
 
 #[derive(Diagnostic)]
-#[diag(lint_malformed_attribute, code = E0452)]
+#[diag("malformed lint attribute input", code = E0452)]
 pub(crate) struct MalformedAttribute {
     #[primary_span]
     pub span: Span,
@@ -56,50 +54,55 @@ pub(crate) struct MalformedAttribute {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum MalformedAttributeSub {
-    #[label(lint_bad_attribute_argument)]
+    #[label("bad attribute argument")]
     BadAttributeArgument(#[primary_span] Span),
-    #[label(lint_reason_must_be_string_literal)]
+    #[label("reason must be a string literal")]
     ReasonMustBeStringLiteral(#[primary_span] Span),
-    #[label(lint_reason_must_come_last)]
+    #[label("reason in lint attribute must come last")]
     ReasonMustComeLast(#[primary_span] Span),
 }
 
 #[derive(Diagnostic)]
-#[diag(lint_unknown_tool_in_scoped_lint, code = E0710)]
+#[diag("unknown tool name `{$tool_name}` found in scoped lint: `{$tool_name}::{$lint_name}`", code = E0710)]
 pub(crate) struct UnknownToolInScopedLint {
     #[primary_span]
     pub span: Option,
     pub tool_name: Symbol,
     pub lint_name: String,
-    #[help]
+    #[help("add `#![register_tool({$tool_name})]` to the crate root")]
     pub is_nightly_build: bool,
 }
 
 #[derive(Diagnostic)]
-#[diag(lint_builtin_ellipsis_inclusive_range_patterns, code = E0783)]
+#[diag("`...` range patterns are deprecated", code = E0783)]
 pub(crate) struct BuiltinEllipsisInclusiveRangePatterns {
     #[primary_span]
     pub span: Span,
-    #[suggestion(style = "short", code = "{replace}", applicability = "machine-applicable")]
+    #[suggestion(
+        "use `..=` for an inclusive range",
+        style = "short",
+        code = "{replace}",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
     pub replace: String,
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_requested_level)]
+#[note("requested on the command line with `{$level} {$lint_name}`")]
 pub(crate) struct RequestedLevel<'a> {
     pub level: Level,
     pub lint_name: &'a str,
 }
 
 #[derive(Diagnostic)]
-#[diag(lint_unsupported_group, code = E0602)]
+#[diag("`{$lint_group}` lint group is not supported with ´--force-warn´", code = E0602)]
 pub(crate) struct UnsupportedGroup {
     pub lint_group: String,
 }
 
 #[derive(Diagnostic)]
-#[diag(lint_check_name_unknown_tool, code = E0602)]
+#[diag("unknown lint tool: `{$tool_name}`", code = E0602)]
 pub(crate) struct CheckNameUnknownTool<'a> {
     pub tool_name: Symbol,
     #[subdiagnostic]
diff --git a/compiler/rustc_lint/src/if_let_rescope.rs b/compiler/rustc_lint/src/if_let_rescope.rs
index 612d542a27de..8569070a43ca 100644
--- a/compiler/rustc_lint/src/if_let_rescope.rs
+++ b/compiler/rustc_lint/src/if_let_rescope.rs
@@ -3,7 +3,9 @@ use std::ops::ControlFlow;
 
 use hir::intravisit::{self, Visitor};
 use rustc_ast::Recovered;
-use rustc_errors::{Applicability, Diag, EmissionGuarantee, Subdiagnostic, SuggestionStyle};
+use rustc_errors::{
+    Applicability, Diag, EmissionGuarantee, Subdiagnostic, SuggestionStyle, inline_fluent,
+};
 use rustc_hir::{self as hir, HirIdSet};
 use rustc_macros::{LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::adjustment::Adjust;
@@ -303,13 +305,15 @@ impl<'tcx> LateLintPass<'tcx> for IfLetRescope {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_if_let_rescope)]
+#[diag("`if let` assigns a shorter lifetime since Edition 2024")]
 struct IfLetRescopeLint {
     #[subdiagnostic]
     destructors: Vec,
-    #[label]
+    #[label(
+        "this value has a significant drop implementation which may observe a major change in drop order and requires your discretion"
+    )]
     significant_droppers: Vec,
-    #[help]
+    #[help("the value is now dropped here in Edition 2024")]
     lifetime_ends: Vec,
     #[subdiagnostic]
     rewrite: Option,
@@ -352,7 +356,9 @@ impl Subdiagnostic for IfLetRescopeRewrite {
                 .chain(repeat_n('}', closing_brackets.count))
                 .collect(),
         ));
-        let msg = diag.eagerly_translate(crate::fluent_generated::lint_suggestion);
+        let msg = diag.eagerly_translate(inline_fluent!(
+            "a `match` with a single arm can preserve the drop order up to Edition 2021"
+        ));
         diag.multipart_suggestion_with_style(
             msg,
             suggestions,
@@ -363,7 +369,12 @@ impl Subdiagnostic for IfLetRescopeRewrite {
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_if_let_dtor)]
+#[note(
+    "{$dtor_kind ->
+        [dyn] value may invoke a custom destructor because it contains a trait object
+        *[concrete] value invokes this custom destructor
+    }"
+)]
 struct DestructorLabel {
     #[primary_span]
     span: Span,
diff --git a/compiler/rustc_lint/src/impl_trait_overcaptures.rs b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
index 1cdc5e4a1b36..087403719968 100644
--- a/compiler/rustc_lint/src/impl_trait_overcaptures.rs
+++ b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
@@ -3,7 +3,7 @@ use std::cell::LazyCell;
 use rustc_data_structures::debug_assert_matches;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::unord::UnordSet;
-use rustc_errors::{LintDiagnostic, Subdiagnostic};
+use rustc_errors::{LintDiagnostic, Subdiagnostic, inline_fluent};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -28,7 +28,7 @@ use rustc_trait_selection::errors::{
 use rustc_trait_selection::regions::OutlivesEnvironmentBuildExt;
 use rustc_trait_selection::traits::ObligationCtxt;
 
-use crate::{LateContext, LateLintPass, fluent_generated as fluent};
+use crate::{LateContext, LateLintPass};
 
 declare_lint! {
     /// The `impl_trait_overcaptures` lint warns against cases where lifetime
@@ -435,11 +435,23 @@ struct ImplTraitOvercapturesLint<'tcx> {
 
 impl<'a> LintDiagnostic<'a, ()> for ImplTraitOvercapturesLint<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_impl_trait_overcaptures);
+        diag.primary_message(inline_fluent!(
+            "`{$self_ty}` will capture more lifetimes than possibly intended in edition 2024"
+        ));
         diag.arg("self_ty", self.self_ty.to_string())
             .arg("num_captured", self.num_captured)
-            .span_note(self.uncaptured_spans, fluent::lint_note)
-            .note(fluent::lint_note2);
+            .span_note(
+                self.uncaptured_spans,
+                inline_fluent!(
+                    "specifically, {$num_captured ->
+                        [one] this lifetime is
+                        *[other] these lifetimes are
+                     } in scope but not mentioned in the type's bounds"
+                ),
+            )
+            .note(inline_fluent!(
+                "all lifetimes in scope will be captured by `impl Trait`s in edition 2024"
+            ));
         if let Some(suggestion) = self.suggestion {
             suggestion.add_to_diag(diag);
         }
@@ -447,9 +459,9 @@ impl<'a> LintDiagnostic<'a, ()> for ImplTraitOvercapturesLint<'_> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_impl_trait_redundant_captures)]
+#[diag("all possible in-scope parameters are already captured, so `use<...>` syntax is redundant")]
 struct ImplTraitRedundantCapturesLint {
-    #[suggestion(lint_suggestion, code = "", applicability = "machine-applicable")]
+    #[suggestion("remove the `use<...>` syntax", code = "", applicability = "machine-applicable")]
     capturing_span: Span,
 }
 
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index 57dacbc11e05..cebe1a5dc373 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -2,7 +2,7 @@ use rustc_ast::attr::AttributeExt;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_data_structures::unord::UnordSet;
-use rustc_errors::{Diag, LintDiagnostic, MultiSpan};
+use rustc_errors::{Diag, LintDiagnostic, MultiSpan, inline_fluent};
 use rustc_feature::{Features, GateIssue};
 use rustc_hir::HirId;
 use rustc_hir::intravisit::{self, Visitor};
@@ -31,7 +31,6 @@ use crate::errors::{
     CheckNameUnknownTool, MalformedAttribute, MalformedAttributeSub, OverruledAttribute,
     OverruledAttributeSub, RequestedLevel, UnknownToolInScopedLint, UnsupportedGroup,
 };
-use crate::fluent_generated as fluent;
 use crate::late::unerased_lint_store;
 use crate::lints::{
     DeprecatedLintName, DeprecatedLintNameFromCommandLine, IgnoredUnlessCrateSpecified,
@@ -942,9 +941,9 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
             let lint = builtin::UNKNOWN_LINTS;
             let level = self.lint_level(builtin::UNKNOWN_LINTS);
             lint_level(self.sess, lint, level, Some(span.into()), |lint| {
-                lint.primary_message(fluent::lint_unknown_gated_lint);
+                lint.primary_message(inline_fluent!("unknown lint: `{$name}`"));
                 lint.arg("name", lint_id.lint.name_lower());
-                lint.note(fluent::lint_note);
+                lint.note(inline_fluent!("the `{$name}` lint is unstable"));
                 rustc_session::parse::add_feature_diagnostics_for_issue(
                     lint,
                     &self.sess,
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index ea31a6373cbc..93b6f1491041 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -139,8 +139,6 @@ pub use rustc_errors::BufferedEarlyLint;
 pub use rustc_session::lint::Level::{self, *};
 pub use rustc_session::lint::{FutureIncompatibleInfo, Lint, LintId, LintPass, LintVec};
 
-rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
-
 pub fn provide(providers: &mut Providers) {
     levels::provide(providers);
     expect::provide(providers);
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index bdc171f11e3b..0aa5199cffc6 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -5,7 +5,7 @@ use std::num::NonZero;
 use rustc_errors::codes::*;
 use rustc_errors::{
     Applicability, Diag, DiagArgValue, DiagMessage, DiagStyledString, ElidedLifetimeInPathSubdiag,
-    EmissionGuarantee, LintDiagnostic, MultiSpan, Subdiagnostic, SuggestionStyle,
+    EmissionGuarantee, LintDiagnostic, MultiSpan, Subdiagnostic, SuggestionStyle, inline_fluent,
 };
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -17,18 +17,24 @@ use rustc_session::Session;
 use rustc_span::edition::Edition;
 use rustc_span::{Ident, Span, Symbol, sym};
 
+use crate::LateContext;
 use crate::builtin::{InitError, ShorthandAssocTyCollector, TypeAliasBounds};
 use crate::errors::{OverruledAttributeSub, RequestedLevel};
 use crate::lifetime_syntax::LifetimeSyntaxCategories;
-use crate::{LateContext, fluent_generated as fluent};
 
 // array_into_iter.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_shadowed_into_iter)]
+#[diag(
+    "this method call resolves to `<&{$target} as IntoIterator>::into_iter` (due to backwards compatibility), but will resolve to `<{$target} as IntoIterator>::into_iter` in Rust {$edition}"
+)]
 pub(crate) struct ShadowedIntoIterDiag {
     pub target: &'static str,
     pub edition: &'static str,
-    #[suggestion(lint_use_iter_suggestion, code = "iter", applicability = "machine-applicable")]
+    #[suggestion(
+        "use `.iter()` instead of `.into_iter()` to avoid ambiguity",
+        code = "iter",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
     #[subdiagnostic]
     pub sub: Option,
@@ -36,13 +42,17 @@ pub(crate) struct ShadowedIntoIterDiag {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum ShadowedIntoIterDiagSub {
-    #[suggestion(lint_remove_into_iter_suggestion, code = "", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "or remove `.into_iter()` to iterate by value",
+        code = "",
+        applicability = "maybe-incorrect"
+    )]
     RemoveIntoIter {
         #[primary_span]
         span: Span,
     },
     #[multipart_suggestion(
-        lint_use_explicit_into_iter_suggestion,
+        "or use `IntoIterator::into_iter(..)` instead of `.into_iter()` to explicitly iterate by value",
         applicability = "maybe-incorrect"
     )]
     UseExplicitIntoIter {
@@ -55,10 +65,12 @@ pub(crate) enum ShadowedIntoIterDiagSub {
 
 // autorefs.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_implicit_unsafe_autorefs)]
-#[note]
+#[diag("implicit autoref creates a reference to the dereference of a raw pointer")]
+#[note(
+    "creating a reference requires the pointer target to be valid and imposes aliasing requirements"
+)]
 pub(crate) struct ImplicitUnsafeAutorefsDiag<'a> {
-    #[label(lint_raw_ptr)]
+    #[label("this raw pointer has type `{$raw_ptr_ty}`")]
     pub raw_ptr_span: Span,
     pub raw_ptr_ty: Ty<'a>,
     #[subdiagnostic]
@@ -71,18 +83,20 @@ pub(crate) struct ImplicitUnsafeAutorefsDiag<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum ImplicitUnsafeAutorefsOrigin<'a> {
-    #[note(lint_autoref)]
+    #[note("autoref is being applied to this expression, resulting in: `{$autoref_ty}`")]
     Autoref {
         #[primary_span]
         autoref_span: Span,
         autoref_ty: Ty<'a>,
     },
-    #[note(lint_overloaded_deref)]
+    #[note(
+        "references are created through calls to explicit `Deref(Mut)::deref(_mut)` implementations"
+    )]
     OverloadedDeref,
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_method_def)]
+#[note("method calls to `{$method_name}` require a reference")]
 pub(crate) struct ImplicitUnsafeAutorefsMethodNote {
     #[primary_span]
     pub def_span: Span,
@@ -90,7 +104,10 @@ pub(crate) struct ImplicitUnsafeAutorefsMethodNote {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "try using a raw pointer method instead; or if this reference is intentional, make it explicit",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct ImplicitUnsafeAutorefsSuggestion {
     pub mutbl: &'static str,
     pub deref: &'static str,
@@ -102,78 +119,105 @@ pub(crate) struct ImplicitUnsafeAutorefsSuggestion {
 
 // builtin.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_while_true)]
+#[diag("denote infinite loops with `loop {\"{\"} ... {\"}\"}`")]
 pub(crate) struct BuiltinWhileTrue {
-    #[suggestion(style = "short", code = "{replace}", applicability = "machine-applicable")]
+    #[suggestion(
+        "use `loop`",
+        style = "short",
+        code = "{replace}",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
     pub replace: String,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_non_shorthand_field_patterns)]
+#[diag("the `{$ident}:` in this pattern is redundant")]
 pub(crate) struct BuiltinNonShorthandFieldPatterns {
     pub ident: Ident,
-    #[suggestion(code = "{prefix}{ident}", applicability = "machine-applicable")]
+    #[suggestion(
+        "use shorthand field pattern",
+        code = "{prefix}{ident}",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
     pub prefix: &'static str,
 }
 
 #[derive(LintDiagnostic)]
 pub(crate) enum BuiltinUnsafe {
-    #[diag(lint_builtin_allow_internal_unsafe)]
+    #[diag(
+        "`allow_internal_unsafe` allows defining macros using unsafe without triggering the `unsafe_code` lint at their call site"
+    )]
     AllowInternalUnsafe,
-    #[diag(lint_builtin_unsafe_block)]
+    #[diag("usage of an `unsafe` block")]
     UnsafeBlock,
-    #[diag(lint_builtin_unsafe_extern_block)]
+    #[diag("usage of an `unsafe extern` block")]
     UnsafeExternBlock,
-    #[diag(lint_builtin_unsafe_trait)]
+    #[diag("declaration of an `unsafe` trait")]
     UnsafeTrait,
-    #[diag(lint_builtin_unsafe_impl)]
+    #[diag("implementation of an `unsafe` trait")]
     UnsafeImpl,
-    #[diag(lint_builtin_no_mangle_fn)]
-    #[note(lint_builtin_overridden_symbol_name)]
+    #[diag("declaration of a `no_mangle` function")]
+    #[note(
+        "the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them"
+    )]
     NoMangleFn,
-    #[diag(lint_builtin_export_name_fn)]
-    #[note(lint_builtin_overridden_symbol_name)]
+    #[diag("declaration of a function with `export_name`")]
+    #[note(
+        "the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them"
+    )]
     ExportNameFn,
-    #[diag(lint_builtin_link_section_fn)]
-    #[note(lint_builtin_overridden_symbol_section)]
+    #[diag("declaration of a function with `link_section`")]
+    #[note(
+        "the program's behavior with overridden link sections on items is unpredictable and Rust cannot provide guarantees when you manually override them"
+    )]
     LinkSectionFn,
-    #[diag(lint_builtin_no_mangle_static)]
-    #[note(lint_builtin_overridden_symbol_name)]
+    #[diag("declaration of a `no_mangle` static")]
+    #[note(
+        "the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them"
+    )]
     NoMangleStatic,
-    #[diag(lint_builtin_export_name_static)]
-    #[note(lint_builtin_overridden_symbol_name)]
+    #[diag("declaration of a static with `export_name`")]
+    #[note(
+        "the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them"
+    )]
     ExportNameStatic,
-    #[diag(lint_builtin_link_section_static)]
-    #[note(lint_builtin_overridden_symbol_section)]
+    #[diag("declaration of a static with `link_section`")]
+    #[note(
+        "the program's behavior with overridden link sections on items is unpredictable and Rust cannot provide guarantees when you manually override them"
+    )]
     LinkSectionStatic,
-    #[diag(lint_builtin_no_mangle_method)]
-    #[note(lint_builtin_overridden_symbol_name)]
+    #[diag("declaration of a `no_mangle` method")]
+    #[note(
+        "the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them"
+    )]
     NoMangleMethod,
-    #[diag(lint_builtin_export_name_method)]
-    #[note(lint_builtin_overridden_symbol_name)]
+    #[diag("declaration of a method with `export_name`")]
+    #[note(
+        "the linker's behavior with multiple libraries exporting duplicate symbol names is undefined and Rust cannot provide guarantees when you manually override them"
+    )]
     ExportNameMethod,
-    #[diag(lint_builtin_decl_unsafe_fn)]
+    #[diag("declaration of an `unsafe` function")]
     DeclUnsafeFn,
-    #[diag(lint_builtin_decl_unsafe_method)]
+    #[diag("declaration of an `unsafe` method")]
     DeclUnsafeMethod,
-    #[diag(lint_builtin_impl_unsafe_method)]
+    #[diag("implementation of an `unsafe` method")]
     ImplUnsafeMethod,
-    #[diag(lint_builtin_global_asm)]
-    #[note(lint_builtin_global_macro_unsafety)]
+    #[diag("usage of `core::arch::global_asm`")]
+    #[note("using this macro is unsafe even though it does not need an `unsafe` block")]
     GlobalAsm,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_missing_doc)]
+#[diag("missing documentation for {$article} {$desc}")]
 pub(crate) struct BuiltinMissingDoc<'a> {
     pub article: &'a str,
     pub desc: &'a str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_missing_copy_impl)]
+#[diag("type could implement `Copy`; consider adding `impl Copy`")]
 pub(crate) struct BuiltinMissingCopyImpl;
 
 pub(crate) struct BuiltinMissingDebugImpl<'a> {
@@ -184,24 +228,24 @@ pub(crate) struct BuiltinMissingDebugImpl<'a> {
 // Needed for def_path_str
 impl<'a> LintDiagnostic<'a, ()> for BuiltinMissingDebugImpl<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_builtin_missing_debug_impl);
+        diag.primary_message(inline_fluent!("type does not implement `{$debug}`; consider adding `#[derive(Debug)]` or a manual implementation"));
         diag.arg("debug", self.tcx.def_path_str(self.def_id));
     }
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_anonymous_params)]
+#[diag("anonymous parameters are deprecated and will be removed in the next edition")]
 pub(crate) struct BuiltinAnonymousParams<'a> {
-    #[suggestion(code = "_: {ty_snip}")]
+    #[suggestion("try naming the parameter or explicitly ignoring it", code = "_: {ty_snip}")]
     pub suggestion: (Span, Applicability),
     pub ty_snip: &'a str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_unused_doc_comment)]
+#[diag("unused doc comment")]
 pub(crate) struct BuiltinUnusedDocComment<'a> {
     pub kind: &'a str,
-    #[label]
+    #[label("rustdoc does not generate documentation for {$kind}")]
     pub label: Span,
     #[subdiagnostic]
     pub sub: BuiltinUnusedDocCommentSub,
@@ -209,34 +253,41 @@ pub(crate) struct BuiltinUnusedDocComment<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum BuiltinUnusedDocCommentSub {
-    #[help(lint_plain_help)]
+    #[help("use `//` for a plain comment")]
     PlainHelp,
-    #[help(lint_block_help)]
+    #[help("use `/* */` for a plain comment")]
     BlockHelp,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_no_mangle_generic)]
+#[diag("functions generic over types or consts must be mangled")]
 pub(crate) struct BuiltinNoMangleGeneric {
     // Use of `#[no_mangle]` suggests FFI intent; correct
     // fix may be to monomorphize source by hand
-    #[suggestion(style = "short", code = "", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "remove this attribute",
+        style = "short",
+        code = "",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_const_no_mangle)]
+#[diag("const items should never be `#[no_mangle]`")]
 pub(crate) struct BuiltinConstNoMangle {
-    #[suggestion(code = "pub static ", applicability = "machine-applicable")]
+    #[suggestion("try a static value", code = "pub static ", applicability = "machine-applicable")]
     pub suggestion: Option,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_mutable_transmutes)]
+#[diag(
+    "transmuting &T to &mut T is undefined behavior, even if the reference is unused, consider instead using an UnsafeCell"
+)]
 pub(crate) struct BuiltinMutablesTransmutes;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_unstable_features)]
+#[diag("use of an unstable feature")]
 pub(crate) struct BuiltinUnstableFeatures;
 
 // lint_ungated_async_fn_track_caller
@@ -247,8 +298,11 @@ pub(crate) struct BuiltinUngatedAsyncFnTrackCaller<'a> {
 
 impl<'a> LintDiagnostic<'a, ()> for BuiltinUngatedAsyncFnTrackCaller<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_ungated_async_fn_track_caller);
-        diag.span_label(self.label, fluent::lint_label);
+        diag.primary_message(inline_fluent!("`#[track_caller]` on async functions is a no-op"));
+        diag.span_label(
+            self.label,
+            inline_fluent!("this function will not propagate the caller location"),
+        );
         rustc_session::parse::add_feature_diagnostics(
             diag,
             self.session,
@@ -258,20 +312,24 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinUngatedAsyncFnTrackCaller<'_> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_unreachable_pub)]
+#[diag("unreachable `pub` {$what}")]
 pub(crate) struct BuiltinUnreachablePub<'a> {
     pub what: &'a str,
     pub new_vis: &'a str,
-    #[suggestion(code = "{new_vis}")]
+    #[suggestion("consider restricting its visibility", code = "{new_vis}")]
     pub suggestion: (Span, Applicability),
-    #[help]
+    #[help("or consider exporting it for use by other crates")]
     pub help: bool,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_macro_expr_fragment_specifier_2024_migration)]
+#[diag("the `expr` fragment specifier will accept more expressions in the 2024 edition")]
 pub(crate) struct MacroExprFragment2024 {
-    #[suggestion(code = "expr_2021", applicability = "machine-applicable")]
+    #[suggestion(
+        "to keep the existing behavior, use the `expr_2021` fragment specifier",
+        code = "expr_2021",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
 }
 
@@ -287,14 +345,18 @@ pub(crate) struct BuiltinTypeAliasBounds<'hir> {
 impl<'a> LintDiagnostic<'a, ()> for BuiltinTypeAliasBounds<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
         diag.primary_message(if self.in_where_clause {
-            fluent::lint_builtin_type_alias_bounds_where_clause
+            inline_fluent!("where clauses on type aliases are not enforced")
         } else {
-            fluent::lint_builtin_type_alias_bounds_param_bounds
+            inline_fluent!("bounds on generic parameters in type aliases are not enforced")
         });
-        diag.span_label(self.label, fluent::lint_builtin_type_alias_bounds_label);
-        diag.note(fluent::lint_builtin_type_alias_bounds_limitation_note);
+        diag.span_label(
+            self.label,
+            inline_fluent!("will not be checked at usage sites of the type alias"),
+        );
+        diag.note(inline_fluent!("this is a known limitation of the type checker that may be lifted in a future edition.
+            see issue #112792  for more information"));
         if self.enable_feat_help {
-            diag.help(fluent::lint_builtin_type_alias_bounds_enable_feat_help);
+            diag.help(inline_fluent!("add `#![feature(lazy_type_alias)]` to the crate attributes to enable the desired semantics"));
         }
 
         // We perform the walk in here instead of in `` to
@@ -319,7 +381,20 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinTypeAliasBounds<'_> {
         };
 
         diag.arg("count", self.suggestions.len());
-        diag.multipart_suggestion(fluent::lint_suggestion, self.suggestions, applicability);
+        diag.multipart_suggestion(
+            if self.in_where_clause {
+                inline_fluent!("remove this where clause")
+            } else {
+                inline_fluent!(
+                    "remove {$count ->
+                        [one] this bound
+                        *[other] these bounds
+                    }"
+                )
+            },
+            self.suggestions,
+            applicability,
+        );
 
         // Suggest fully qualifying paths of the form `T::Assoc` with `T` type param via
         // `::Assoc` to remove their reliance on any type param bounds.
@@ -333,7 +408,7 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinTypeAliasBounds<'_> {
         // (We could employ some simple heuristics but that's likely not worth it).
         for qself in collector.qselves {
             diag.multipart_suggestion(
-                fluent::lint_builtin_type_alias_bounds_qualify_assoc_tys_sugg,
+                inline_fluent!("fully qualify this associated type"),
                 vec![
                     (qself.shrink_to_lo(), "<".into()),
                     (qself.shrink_to_hi(), " as /* Trait */>".into()),
@@ -345,23 +420,27 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinTypeAliasBounds<'_> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_trivial_bounds)]
+#[diag(
+    "{$predicate_kind_name} bound {$predicate} does not depend on any type or lifetime parameters"
+)]
 pub(crate) struct BuiltinTrivialBounds<'a> {
     pub predicate_kind_name: &'a str,
     pub predicate: Clause<'a>,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_double_negations)]
-#[note(lint_note)]
-#[note(lint_note_decrement)]
+#[diag("use of a double negation")]
+#[note(
+    "the prefix `--` could be misinterpreted as a decrement operator which exists in other languages"
+)]
+#[note("use `-= 1` if you meant to decrement the value")]
 pub(crate) struct BuiltinDoubleNegations {
     #[subdiagnostic]
     pub add_parens: BuiltinDoubleNegationsAddParens,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_add_parens_suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion("add parentheses for clarity", applicability = "maybe-incorrect")]
 pub(crate) struct BuiltinDoubleNegationsAddParens {
     #[suggestion_part(code = "(")]
     pub start_span: Span,
@@ -371,31 +450,44 @@ pub(crate) struct BuiltinDoubleNegationsAddParens {
 
 #[derive(LintDiagnostic)]
 pub(crate) enum BuiltinEllipsisInclusiveRangePatternsLint {
-    #[diag(lint_builtin_ellipsis_inclusive_range_patterns)]
+    #[diag("`...` range patterns are deprecated")]
     Parenthesise {
-        #[suggestion(code = "{replace}", applicability = "machine-applicable")]
+        #[suggestion(
+            "use `..=` for an inclusive range",
+            code = "{replace}",
+            applicability = "machine-applicable"
+        )]
         suggestion: Span,
         replace: String,
     },
-    #[diag(lint_builtin_ellipsis_inclusive_range_patterns)]
+    #[diag("`...` range patterns are deprecated")]
     NonParenthesise {
-        #[suggestion(style = "short", code = "..=", applicability = "machine-applicable")]
+        #[suggestion(
+            "use `..=` for an inclusive range",
+            style = "short",
+            code = "..=",
+            applicability = "machine-applicable"
+        )]
         suggestion: Span,
     },
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_keyword_idents)]
+#[diag("`{$kw}` is a keyword in the {$next} edition")]
 pub(crate) struct BuiltinKeywordIdents {
     pub kw: Ident,
     pub next: Edition,
-    #[suggestion(code = "{prefix}r#{kw}", applicability = "machine-applicable")]
+    #[suggestion(
+        "you can use a raw identifier to stay compatible",
+        code = "{prefix}r#{kw}",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
     pub prefix: &'static str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_explicit_outlives)]
+#[diag("outlives requirements can be inferred")]
 pub(crate) struct BuiltinExplicitOutlives {
     pub count: usize,
     #[subdiagnostic]
@@ -403,7 +495,12 @@ pub(crate) struct BuiltinExplicitOutlives {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion)]
+#[multipart_suggestion(
+    "remove {$count ->
+        [one] this bound
+        *[other] these bounds
+    }"
+)]
 pub(crate) struct BuiltinExplicitOutlivesSuggestion {
     #[suggestion_part(code = "")]
     pub spans: Vec,
@@ -412,7 +509,9 @@ pub(crate) struct BuiltinExplicitOutlivesSuggestion {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_incomplete_features)]
+#[diag(
+    "the feature `{$name}` is incomplete and may not be safe to use and/or cause compiler crashes"
+)]
 pub(crate) struct BuiltinIncompleteFeatures {
     pub name: Symbol,
     #[subdiagnostic]
@@ -422,18 +521,18 @@ pub(crate) struct BuiltinIncompleteFeatures {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_internal_features)]
-#[note]
+#[diag("the feature `{$name}` is internal to the compiler or standard library")]
+#[note("using it is strongly discouraged")]
 pub(crate) struct BuiltinInternalFeatures {
     pub name: Symbol,
 }
 
 #[derive(Subdiagnostic)]
-#[help(lint_help)]
+#[help("consider using `min_{$name}` instead, which is more stable and complete")]
 pub(crate) struct BuiltinIncompleteFeaturesHelp;
 
 #[derive(Subdiagnostic)]
-#[note(lint_note)]
+#[note("see issue #{$n}  for more information")]
 pub(crate) struct BuiltinFeatureIssueNote {
     pub n: NonZero,
 }
@@ -450,12 +549,15 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinUnpermittedTypeInit<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
         diag.primary_message(self.msg);
         diag.arg("ty", self.ty);
-        diag.span_label(self.label, fluent::lint_builtin_unpermitted_type_init_label);
+        diag.span_label(
+            self.label,
+            inline_fluent!("this code causes undefined behavior when executed"),
+        );
         if let InhabitedPredicate::True = self.ty.inhabited_predicate(self.tcx) {
             // Only suggest late `MaybeUninit::assume_init` initialization if the type is inhabited.
             diag.span_label(
                 self.label,
-                fluent::lint_builtin_unpermitted_type_init_label_suggestion,
+                inline_fluent!("help: use `MaybeUninit` instead, and only call `assume_init` after initialization is done"),
             );
         }
         self.sub.add_to_diag(diag);
@@ -487,24 +589,24 @@ impl Subdiagnostic for BuiltinUnpermittedTypeInitSub {
 
 #[derive(LintDiagnostic)]
 pub(crate) enum BuiltinClashingExtern<'a> {
-    #[diag(lint_builtin_clashing_extern_same_name)]
+    #[diag("`{$this}` redeclared with a different signature")]
     SameName {
         this: Symbol,
         orig: Symbol,
-        #[label(lint_previous_decl_label)]
+        #[label("`{$orig}` previously declared here")]
         previous_decl_label: Span,
-        #[label(lint_mismatch_label)]
+        #[label("this signature doesn't match the previous declaration")]
         mismatch_label: Span,
         #[subdiagnostic]
         sub: BuiltinClashingExternSub<'a>,
     },
-    #[diag(lint_builtin_clashing_extern_diff_name)]
+    #[diag("`{$this}` redeclares `{$orig}` with a different signature")]
     DiffName {
         this: Symbol,
         orig: Symbol,
-        #[label(lint_previous_decl_label)]
+        #[label("`{$orig}` previously declared here")]
         previous_decl_label: Span,
-        #[label(lint_mismatch_label)]
+        #[label("this signature doesn't match the previous declaration")]
         mismatch_label: Span,
         #[subdiagnostic]
         sub: BuiltinClashingExternSub<'a>,
@@ -529,9 +631,9 @@ impl Subdiagnostic for BuiltinClashingExternSub<'_> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_builtin_deref_nullptr)]
+#[diag("dereferencing a null pointer")]
 pub(crate) struct BuiltinDerefNullptr {
-    #[label]
+    #[label("this code causes undefined behavior when executed")]
     pub label: Span,
 }
 
@@ -539,30 +641,32 @@ pub(crate) struct BuiltinDerefNullptr {
 
 #[derive(LintDiagnostic)]
 pub(crate) enum BuiltinSpecialModuleNameUsed {
-    #[diag(lint_builtin_special_module_name_used_lib)]
-    #[note]
-    #[help]
+    #[diag("found module declaration for lib.rs")]
+    #[note("lib.rs is the root of this crate's library target")]
+    #[help("to refer to it from other targets, use the library's name as the path")]
     Lib,
-    #[diag(lint_builtin_special_module_name_used_main)]
-    #[note]
+    #[diag("found module declaration for main.rs")]
+    #[note("a binary crate cannot be used as library")]
     Main,
 }
 
 // deref_into_dyn_supertrait.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_supertrait_as_deref_target)]
+#[diag("this `Deref` implementation is covered by an implicit supertrait coercion")]
 pub(crate) struct SupertraitAsDerefTarget<'a> {
     pub self_ty: Ty<'a>,
     pub supertrait_principal: PolyExistentialTraitRef<'a>,
     pub target_principal: PolyExistentialTraitRef<'a>,
-    #[label]
+    #[label(
+        "`{$self_ty}` implements `Deref` which conflicts with supertrait `{$supertrait_principal}`"
+    )]
     pub label: Span,
     #[subdiagnostic]
     pub label2: Option,
 }
 
 #[derive(Subdiagnostic)]
-#[label(lint_label2)]
+#[label("target type is a supertrait of `{$self_ty}`")]
 pub(crate) struct SupertraitAsDerefTargetLabel {
     #[primary_span]
     pub label: Span,
@@ -570,32 +674,38 @@ pub(crate) struct SupertraitAsDerefTargetLabel {
 
 // enum_intrinsics_non_enums.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_enum_intrinsics_mem_discriminant)]
+#[diag("the return value of `mem::discriminant` is unspecified when called with a non-enum type")]
 pub(crate) struct EnumIntrinsicsMemDiscriminate<'a> {
     pub ty_param: Ty<'a>,
-    #[note]
+    #[note(
+        "the argument to `discriminant` should be a reference to an enum, but it was passed a reference to a `{$ty_param}`, which is not an enum"
+    )]
     pub note: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_enum_intrinsics_mem_variant)]
-#[note]
+#[diag("the return value of `mem::variant_count` is unspecified when called with a non-enum type")]
+#[note(
+    "the type parameter of `variant_count` should be an enum, but it was instantiated with the type `{$ty_param}`, which is not an enum"
+)]
 pub(crate) struct EnumIntrinsicsMemVariant<'a> {
     pub ty_param: Ty<'a>,
 }
 
 // expect.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_expectation)]
+#[diag("this lint expectation is unfulfilled")]
 pub(crate) struct Expectation {
     #[subdiagnostic]
     pub rationale: Option,
-    #[note]
+    #[note(
+        "the `unfulfilled_lint_expectations` lint can't be expected and will always produce this message"
+    )]
     pub note: bool,
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_rationale)]
+#[note("{$rationale}")]
 pub(crate) struct ExpectationNote {
     pub rationale: Symbol,
 }
@@ -603,42 +713,58 @@ pub(crate) struct ExpectationNote {
 // ptr_nulls.rs
 #[derive(LintDiagnostic)]
 pub(crate) enum UselessPtrNullChecksDiag<'a> {
-    #[diag(lint_useless_ptr_null_checks_fn_ptr)]
-    #[help]
+    #[diag(
+        "function pointers are not nullable, so checking them for null will always return false"
+    )]
+    #[help(
+        "wrap the function pointer inside an `Option` and use `Option::is_none` to check for null pointer value"
+    )]
     FnPtr {
         orig_ty: Ty<'a>,
-        #[label]
+        #[label("expression has type `{$orig_ty}`")]
         label: Span,
     },
-    #[diag(lint_useless_ptr_null_checks_ref)]
+    #[diag("references are not nullable, so checking them for null will always return false")]
     Ref {
         orig_ty: Ty<'a>,
-        #[label]
+        #[label("expression has type `{$orig_ty}`")]
         label: Span,
     },
-    #[diag(lint_useless_ptr_null_checks_fn_ret)]
+    #[diag(
+        "returned pointer of `{$fn_name}` call is never null, so checking it for null will always return false"
+    )]
     FnRet { fn_name: Ident },
 }
 
 #[derive(LintDiagnostic)]
 pub(crate) enum InvalidNullArgumentsDiag {
-    #[diag(lint_invalid_null_arguments)]
-    #[help(lint_doc)]
+    #[diag(
+        "calling this function with a null pointer is undefined behavior, even if the result of the function is unused"
+    )]
+    #[help(
+        "for more information, visit  and "
+    )]
     NullPtrInline {
-        #[label(lint_origin)]
+        #[label("null pointer originates from here")]
         null_span: Span,
     },
-    #[diag(lint_invalid_null_arguments)]
-    #[help(lint_doc)]
+    #[diag(
+        "calling this function with a null pointer is undefined behavior, even if the result of the function is unused"
+    )]
+    #[help(
+        "for more information, visit  and "
+    )]
     NullPtrThroughBinding {
-        #[note(lint_origin)]
+        #[note("null pointer originates from here")]
         null_span: Span,
     },
 }
 
 // for_loops_over_fallibles.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_for_loops_over_fallibles)]
+#[diag(
+    "for loop over {$article} `{$ref_prefix}{$ty}`. This is more readably written as an `if let` statement"
+)]
 pub(crate) struct ForLoopsOverFalliblesDiag<'a> {
     pub article: &'static str,
     pub ref_prefix: &'static str,
@@ -653,13 +779,20 @@ pub(crate) struct ForLoopsOverFalliblesDiag<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum ForLoopsOverFalliblesLoopSub<'a> {
-    #[suggestion(lint_remove_next, code = ".by_ref()", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "to iterate over `{$recv_snip}` remove the call to `next`",
+        code = ".by_ref()",
+        applicability = "maybe-incorrect"
+    )]
     RemoveNext {
         #[primary_span]
         suggestion: Span,
         recv_snip: String,
     },
-    #[multipart_suggestion(lint_use_while_let, applicability = "maybe-incorrect")]
+    #[multipart_suggestion(
+        "to check pattern in a loop use `while let`",
+        applicability = "maybe-incorrect"
+    )]
     UseWhileLet {
         #[suggestion_part(code = "while let {var}(")]
         start_span: Span,
@@ -670,14 +803,21 @@ pub(crate) enum ForLoopsOverFalliblesLoopSub<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(lint_use_question_mark, code = "?", applicability = "maybe-incorrect")]
+#[suggestion(
+    "consider unwrapping the `Result` with `?` to iterate over its contents",
+    code = "?",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct ForLoopsOverFalliblesQuestionMark {
     #[primary_span]
     pub suggestion: Span,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "consider using `if let` to clear intent",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct ForLoopsOverFalliblesSuggestion<'a> {
     pub var: &'a str,
     #[suggestion_part(code = "if let {var}(")]
@@ -688,10 +828,10 @@ pub(crate) struct ForLoopsOverFalliblesSuggestion<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum UseLetUnderscoreIgnoreSuggestion {
-    #[note(lint_use_let_underscore_ignore_suggestion)]
+    #[note("use `let _ = ...` to ignore the expression or result")]
     Note,
     #[multipart_suggestion(
-        lint_use_let_underscore_ignore_suggestion,
+        "use `let _ = ...` to ignore the expression or result",
         style = "verbose",
         applicability = "maybe-incorrect"
     )]
@@ -705,57 +845,62 @@ pub(crate) enum UseLetUnderscoreIgnoreSuggestion {
 
 // drop_forget_useless.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_dropping_references)]
+#[diag("calls to `std::mem::drop` with a reference instead of an owned value does nothing")]
 pub(crate) struct DropRefDiag<'a> {
     pub arg_ty: Ty<'a>,
-    #[label]
+    #[label("argument has type `{$arg_ty}`")]
     pub label: Span,
     #[subdiagnostic]
     pub sugg: UseLetUnderscoreIgnoreSuggestion,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_dropping_copy_types)]
+#[diag("calls to `std::mem::drop` with a value that implements `Copy` does nothing")]
 pub(crate) struct DropCopyDiag<'a> {
     pub arg_ty: Ty<'a>,
-    #[label]
+    #[label("argument has type `{$arg_ty}`")]
     pub label: Span,
     #[subdiagnostic]
     pub sugg: UseLetUnderscoreIgnoreSuggestion,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_forgetting_references)]
+#[diag("calls to `std::mem::forget` with a reference instead of an owned value does nothing")]
 pub(crate) struct ForgetRefDiag<'a> {
     pub arg_ty: Ty<'a>,
-    #[label]
+    #[label("argument has type `{$arg_ty}`")]
     pub label: Span,
     #[subdiagnostic]
     pub sugg: UseLetUnderscoreIgnoreSuggestion,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_forgetting_copy_types)]
+#[diag("calls to `std::mem::forget` with a value that implements `Copy` does nothing")]
 pub(crate) struct ForgetCopyDiag<'a> {
     pub arg_ty: Ty<'a>,
-    #[label]
+    #[label("argument has type `{$arg_ty}`")]
     pub label: Span,
     #[subdiagnostic]
     pub sugg: UseLetUnderscoreIgnoreSuggestion,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_undropped_manually_drops)]
+#[diag(
+    "calls to `std::mem::drop` with `std::mem::ManuallyDrop` instead of the inner value does nothing"
+)]
 pub(crate) struct UndroppedManuallyDropsDiag<'a> {
     pub arg_ty: Ty<'a>,
-    #[label]
+    #[label("argument has type `{$arg_ty}`")]
     pub label: Span,
     #[subdiagnostic]
     pub suggestion: UndroppedManuallyDropsSuggestion,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(
+    "use `std::mem::ManuallyDrop::into_inner` to get the inner value",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct UndroppedManuallyDropsSuggestion {
     #[suggestion_part(code = "std::mem::ManuallyDrop::into_inner(")]
     pub start_span: Span,
@@ -766,33 +911,35 @@ pub(crate) struct UndroppedManuallyDropsSuggestion {
 // invalid_from_utf8.rs
 #[derive(LintDiagnostic)]
 pub(crate) enum InvalidFromUtf8Diag {
-    #[diag(lint_invalid_from_utf8_unchecked)]
+    #[diag("calls to `{$method}` with an invalid literal are undefined behavior")]
     Unchecked {
         method: String,
         valid_up_to: usize,
-        #[label]
+        #[label("the literal was valid UTF-8 up to the {$valid_up_to} bytes")]
         label: Span,
     },
-    #[diag(lint_invalid_from_utf8_checked)]
+    #[diag("calls to `{$method}` with an invalid literal always return an error")]
     Checked {
         method: String,
         valid_up_to: usize,
-        #[label]
+        #[label("the literal was valid UTF-8 up to the {$valid_up_to} bytes")]
         label: Span,
     },
 }
 
 // interior_mutable_consts.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_const_item_interior_mutations)]
-#[note(lint_temporary)]
-#[note(lint_never_original)]
-#[help]
+#[diag("mutation of an interior mutable `const` item with call to `{$method_name}`")]
+#[note("each usage of a `const` item creates a new temporary")]
+#[note("only the temporaries and never the original `const {$const_name}` will be modified")]
+#[help(
+    "for more details on interior mutability see "
+)]
 pub(crate) struct ConstItemInteriorMutationsDiag<'tcx> {
     pub method_name: Ident,
     pub const_name: Ident,
     pub const_ty: Ty<'tcx>,
-    #[label]
+    #[label("`{$const_name}` is a interior mutable `const` item of type `{$const_ty}`")]
     pub receiver_span: Span,
     #[subdiagnostic]
     pub sugg_static: Option,
@@ -801,7 +948,7 @@ pub(crate) struct ConstItemInteriorMutationsDiag<'tcx> {
 #[derive(Subdiagnostic)]
 pub(crate) enum ConstItemInteriorMutationsSuggestionStatic {
     #[suggestion(
-        lint_suggestion_static,
+        "for a shared instance of `{$const_name}`, consider making it a `static` item instead",
         code = "{before}static ",
         style = "verbose",
         applicability = "maybe-incorrect"
@@ -811,35 +958,47 @@ pub(crate) enum ConstItemInteriorMutationsSuggestionStatic {
         const_: Span,
         before: &'static str,
     },
-    #[help(lint_suggestion_static)]
+    #[help("for a shared instance of `{$const_name}`, consider making it a `static` item instead")]
     Spanless,
 }
 
 // reference_casting.rs
 #[derive(LintDiagnostic)]
 pub(crate) enum InvalidReferenceCastingDiag<'tcx> {
-    #[diag(lint_invalid_reference_casting_borrow_as_mut)]
-    #[note(lint_invalid_reference_casting_note_book)]
+    #[diag(
+        "casting `&T` to `&mut T` is undefined behavior, even if the reference is unused, consider instead using an `UnsafeCell`"
+    )]
+    #[note(
+        "for more information, visit "
+    )]
     BorrowAsMut {
-        #[label]
+        #[label("casting happened here")]
         orig_cast: Option,
-        #[note(lint_invalid_reference_casting_note_ty_has_interior_mutability)]
+        #[note(
+            "even for types with interior mutability, the only legal way to obtain a mutable pointer from a shared reference is through `UnsafeCell::get`"
+        )]
         ty_has_interior_mutability: bool,
     },
-    #[diag(lint_invalid_reference_casting_assign_to_ref)]
-    #[note(lint_invalid_reference_casting_note_book)]
+    #[diag("assigning to `&T` is undefined behavior, consider using an `UnsafeCell`")]
+    #[note(
+        "for more information, visit "
+    )]
     AssignToRef {
-        #[label]
+        #[label("casting happened here")]
         orig_cast: Option,
-        #[note(lint_invalid_reference_casting_note_ty_has_interior_mutability)]
+        #[note(
+            "even for types with interior mutability, the only legal way to obtain a mutable pointer from a shared reference is through `UnsafeCell::get`"
+        )]
         ty_has_interior_mutability: bool,
     },
-    #[diag(lint_invalid_reference_casting_bigger_layout)]
-    #[note(lint_layout)]
+    #[diag(
+        "casting references to a bigger memory layout than the backing allocation is undefined behavior, even if the reference is unused"
+    )]
+    #[note("casting from `{$from_ty}` ({$from_size} bytes) to `{$to_ty}` ({$to_size} bytes)")]
     BiggerLayout {
-        #[label]
+        #[label("casting happened here")]
         orig_cast: Option,
-        #[label(lint_alloc)]
+        #[label("backing allocation comes from here")]
         alloc: Span,
         from_ty: Ty<'tcx>,
         from_size: u64,
@@ -850,108 +1009,143 @@ pub(crate) enum InvalidReferenceCastingDiag<'tcx> {
 
 // map_unit_fn.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_map_unit_fn)]
-#[note]
+#[diag("`Iterator::map` call that discard the iterator's values")]
+#[note(
+    "`Iterator::map`, like many of the methods on `Iterator`, gets executed lazily, meaning that its effects won't be visible until it is iterated"
+)]
 pub(crate) struct MappingToUnit {
-    #[label(lint_function_label)]
+    #[label("this function returns `()`, which is likely not what you wanted")]
     pub function_label: Span,
-    #[label(lint_argument_label)]
+    #[label("called `Iterator::map` with callable that returns `()`")]
     pub argument_label: Span,
-    #[label(lint_map_label)]
+    #[label(
+        "after this call to map, the resulting iterator is `impl Iterator`, which means the only information carried by the iterator is the number of items"
+    )]
     pub map_label: Span,
-    #[suggestion(style = "verbose", code = "for_each", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "you might have meant to use `Iterator::for_each`",
+        style = "verbose",
+        code = "for_each",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Span,
 }
 
 // internal.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_default_hash_types)]
-#[note]
+#[diag("prefer `{$preferred}` over `{$used}`, it has better performance")]
+#[note("a `use rustc_data_structures::fx::{$preferred}` may be necessary")]
 pub(crate) struct DefaultHashTypesDiag<'a> {
     pub preferred: &'a str,
     pub used: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_query_instability)]
-#[note]
+#[diag("using `{$query}` can result in unstable query results")]
+#[note(
+    "if you believe this case to be fine, allow this lint and add a comment explaining your rationale"
+)]
 pub(crate) struct QueryInstability {
     pub query: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_query_untracked)]
-#[note]
+#[diag("`{$method}` accesses information that is not tracked by the query system")]
+#[note(
+    "if you believe this case to be fine, allow this lint and add a comment explaining your rationale"
+)]
 pub(crate) struct QueryUntracked {
     pub method: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_span_use_eq_ctxt)]
+#[diag("use `.eq_ctxt()` instead of `.ctxt() == .ctxt()`")]
 pub(crate) struct SpanUseEqCtxtDiag;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_symbol_intern_string_literal)]
-#[help]
+#[diag("using `Symbol::intern` on a string literal")]
+#[help("consider adding the symbol to `compiler/rustc_span/src/symbol.rs`")]
 pub(crate) struct SymbolInternStringLiteralDiag;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_tykind_kind)]
+#[diag("usage of `ty::TyKind::`")]
 pub(crate) struct TykindKind {
-    #[suggestion(code = "ty", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "try using `ty::` directly",
+        code = "ty",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_tykind)]
-#[help]
+#[diag("usage of `ty::TyKind`")]
+#[help("try using `Ty` instead")]
 pub(crate) struct TykindDiag;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_ty_qualified)]
+#[diag("usage of qualified `ty::{$ty}`")]
 pub(crate) struct TyQualified {
     pub ty: String,
-    #[suggestion(code = "{ty}", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "try importing it and using it unqualified",
+        code = "{ty}",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_type_ir_inherent_usage)]
-#[note]
+#[diag("do not use `rustc_type_ir::inherent` unless you're inside of the trait solver")]
+#[note(
+    "the method or struct you're looking for is likely defined somewhere else downstream in the compiler"
+)]
 pub(crate) struct TypeIrInherentUsage;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_type_ir_trait_usage)]
-#[note]
+#[diag(
+    "do not use `rustc_type_ir::Interner` or `rustc_type_ir::InferCtxtLike` unless you're inside of the trait solver"
+)]
+#[note(
+    "the method or struct you're looking for is likely defined somewhere else downstream in the compiler"
+)]
 pub(crate) struct TypeIrTraitUsage;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_type_ir_direct_use)]
-#[note]
+#[diag("do not use `rustc_type_ir` unless you are implementing type system internals")]
+#[note("use `rustc_middle::ty` instead")]
 pub(crate) struct TypeIrDirectUse;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_non_glob_import_type_ir_inherent)]
+#[diag("non-glob import of `rustc_type_ir::inherent`")]
 pub(crate) struct NonGlobImportTypeIrInherent {
-    #[suggestion(code = "{snippet}", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "try using a glob import instead",
+        code = "{snippet}",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Option,
     pub snippet: &'static str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_lintpass_by_hand)]
-#[help]
+#[diag("implementing `LintPass` by hand")]
+#[help("try using `declare_lint_pass!` or `impl_lint_pass!` instead")]
 pub(crate) struct LintPassByHand;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_bad_opt_access)]
+#[diag("{$msg}")]
 pub(crate) struct BadOptAccessDiag<'a> {
     pub msg: &'a str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_implicit_sysroot_crate_import)]
-#[help]
+#[diag(
+    "dangerous use of `extern crate {$name}` which is not guaranteed to exist exactly once in the sysroot"
+)]
+#[help(
+    "try using a cargo dependency or using a re-export of the dependency provided by a rustc_* crate"
+)]
 pub(crate) struct ImplicitSysrootCrateImportDiag<'a> {
     pub name: &'a str,
 }
@@ -959,14 +1153,14 @@ pub(crate) struct ImplicitSysrootCrateImportDiag<'a> {
 // let_underscore.rs
 #[derive(LintDiagnostic)]
 pub(crate) enum NonBindingLet {
-    #[diag(lint_non_binding_let_on_sync_lock)]
+    #[diag("non-binding let on a synchronization lock")]
     SyncLock {
-        #[label]
+        #[label("this lock is not assigned to a binding and is immediately dropped")]
         pat: Span,
         #[subdiagnostic]
         sub: NonBindingLetSub,
     },
-    #[diag(lint_non_binding_let_on_drop_type)]
+    #[diag("non-binding let on a type that has a destructor")]
     DropType {
         #[subdiagnostic]
         sub: NonBindingLetSub,
@@ -987,16 +1181,23 @@ impl Subdiagnostic for NonBindingLetSub {
             let prefix = if self.is_assign_desugar { "let " } else { "" };
             diag.span_suggestion_verbose(
                 self.suggestion,
-                fluent::lint_non_binding_let_suggestion,
+                inline_fluent!(
+                    "consider binding to an unused variable to avoid immediately dropping the value"
+                ),
                 format!("{prefix}_unused"),
                 Applicability::MachineApplicable,
             );
         } else {
-            diag.span_help(self.suggestion, fluent::lint_non_binding_let_suggestion);
+            diag.span_help(
+                self.suggestion,
+                inline_fluent!(
+                    "consider binding to an unused variable to avoid immediately dropping the value"
+                ),
+            );
         }
         if let Some(drop_fn_start_end) = self.drop_fn_start_end {
             diag.multipart_suggestion(
-                fluent::lint_non_binding_let_multi_suggestion,
+                inline_fluent!("consider immediately dropping the value"),
                 vec![
                     (drop_fn_start_end.0, "drop(".to_string()),
                     (drop_fn_start_end.1, ")".to_string()),
@@ -1004,16 +1205,18 @@ impl Subdiagnostic for NonBindingLetSub {
                 Applicability::MachineApplicable,
             );
         } else {
-            diag.help(fluent::lint_non_binding_let_multi_drop_fn);
+            diag.help(inline_fluent!(
+                "consider immediately dropping the value using `drop(..)` after the `let` statement"
+            ));
         }
     }
 }
 
 // levels.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_overruled_attribute)]
+#[diag("{$lint_level}({$lint_source}) incompatible with previous forbid")]
 pub(crate) struct OverruledAttributeLint<'a> {
-    #[label]
+    #[label("overruled by previous forbid")]
     pub overruled: Span,
     pub lint_level: &'a str,
     pub lint_source: Symbol,
@@ -1022,17 +1225,17 @@ pub(crate) struct OverruledAttributeLint<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_deprecated_lint_name)]
+#[diag("lint name `{$name}` is deprecated and may not have an effect in the future")]
 pub(crate) struct DeprecatedLintName<'a> {
     pub name: String,
-    #[suggestion(code = "{replace}", applicability = "machine-applicable")]
+    #[suggestion("change it to", code = "{replace}", applicability = "machine-applicable")]
     pub suggestion: Span,
     pub replace: &'a str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_deprecated_lint_name)]
-#[help]
+#[diag("lint name `{$name}` is deprecated and may not have an effect in the future")]
+#[help("change it to {$replace}")]
 pub(crate) struct DeprecatedLintNameFromCommandLine<'a> {
     pub name: String,
     pub replace: &'a str,
@@ -1041,7 +1244,7 @@ pub(crate) struct DeprecatedLintNameFromCommandLine<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_renamed_lint)]
+#[diag("lint `{$name}` has been renamed to `{$replace}`")]
 pub(crate) struct RenamedLint<'a> {
     pub name: &'a str,
     pub replace: &'a str,
@@ -1051,18 +1254,18 @@ pub(crate) struct RenamedLint<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum RenamedLintSuggestion<'a> {
-    #[suggestion(lint_suggestion, code = "{replace}", applicability = "machine-applicable")]
+    #[suggestion("use the new name", code = "{replace}", applicability = "machine-applicable")]
     WithSpan {
         #[primary_span]
         suggestion: Span,
         replace: &'a str,
     },
-    #[help(lint_help)]
+    #[help("use the new name `{$replace}`")]
     WithoutSpan { replace: &'a str },
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_renamed_lint)]
+#[diag("lint `{$name}` has been renamed to `{$replace}`")]
 pub(crate) struct RenamedLintFromCommandLine<'a> {
     pub name: &'a str,
     pub replace: &'a str,
@@ -1073,14 +1276,14 @@ pub(crate) struct RenamedLintFromCommandLine<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_removed_lint)]
+#[diag("lint `{$name}` has been removed: {$reason}")]
 pub(crate) struct RemovedLint<'a> {
     pub name: &'a str,
     pub reason: &'a str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_removed_lint)]
+#[diag("lint `{$name}` has been removed: {$reason}")]
 pub(crate) struct RemovedLintFromCommandLine<'a> {
     pub name: &'a str,
     pub reason: &'a str,
@@ -1089,7 +1292,7 @@ pub(crate) struct RemovedLintFromCommandLine<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unknown_lint)]
+#[diag("unknown lint: `{$name}`")]
 pub(crate) struct UnknownLint {
     pub name: String,
     #[subdiagnostic]
@@ -1098,19 +1301,31 @@ pub(crate) struct UnknownLint {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum UnknownLintSuggestion {
-    #[suggestion(lint_suggestion, code = "{replace}", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "{$from_rustc ->
+            [true] a lint with a similar name exists in `rustc` lints
+            *[false] did you mean
+        }",
+        code = "{replace}",
+        applicability = "maybe-incorrect"
+    )]
     WithSpan {
         #[primary_span]
         suggestion: Span,
         replace: Symbol,
         from_rustc: bool,
     },
-    #[help(lint_help)]
+    #[help(
+        "{$from_rustc ->
+            [true] a lint with a similar name exists in `rustc` lints: `{$replace}`
+            *[false] did you mean: `{$replace}`
+        }"
+    )]
     WithoutSpan { replace: Symbol, from_rustc: bool },
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unknown_lint, code = E0602)]
+#[diag("unknown lint: `{$name}`", code = E0602)]
 pub(crate) struct UnknownLintFromCommandLine<'a> {
     pub name: String,
     #[subdiagnostic]
@@ -1120,7 +1335,7 @@ pub(crate) struct UnknownLintFromCommandLine<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_ignored_unless_crate_specified)]
+#[diag("{$level}({$name}) is ignored unless specified at crate level")]
 pub(crate) struct IgnoredUnlessCrateSpecified<'a> {
     pub level: &'a str,
     pub name: Symbol,
@@ -1128,53 +1343,78 @@ pub(crate) struct IgnoredUnlessCrateSpecified<'a> {
 
 // dangling.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_dangling_pointers_from_temporaries)]
-#[help(lint_help_bind)]
-#[note(lint_note_safe)]
-#[note(lint_note_return)]
-#[note(lint_note_more_info)]
+#[diag("this creates a dangling pointer because temporary `{$ty}` is dropped at end of statement")]
+#[help("bind the `{$ty}` to a variable such that it outlives the pointer returned by `{$callee}`")]
+#[note("a dangling pointer is safe, but dereferencing one is undefined behavior")]
+#[note("returning a pointer to a local variable will always result in a dangling pointer")]
+#[note("for more information, see ")]
 // FIXME: put #[primary_span] on `ptr_span` once it does not cause conflicts
 pub(crate) struct DanglingPointersFromTemporaries<'tcx> {
     pub callee: Ident,
     pub ty: Ty<'tcx>,
-    #[label(lint_label_ptr)]
+    #[label("pointer created here")]
     pub ptr_span: Span,
-    #[label(lint_label_temporary)]
+    #[label("this `{$ty}` is dropped at end of statement")]
     pub temporary_span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_dangling_pointers_from_locals)]
-#[note(lint_note_safe)]
-#[note(lint_note_more_info)]
+#[diag("{$fn_kind} returns a dangling pointer to dropped local variable `{$local_var_name}`")]
+#[note("a dangling pointer is safe, but dereferencing one is undefined behavior")]
+#[note("for more information, see ")]
 pub(crate) struct DanglingPointersFromLocals<'tcx> {
     pub ret_ty: Ty<'tcx>,
-    #[label(lint_ret_ty)]
+    #[label("return type is `{$ret_ty}`")]
     pub ret_ty_span: Span,
     pub fn_kind: &'static str,
-    #[label(lint_local_var)]
+    #[label("local variable `{$local_var_name}` is dropped at the end of the {$fn_kind}")]
     pub local_var: Span,
     pub local_var_name: Ident,
     pub local_var_ty: Ty<'tcx>,
-    #[label(lint_created_at)]
+    #[label("dangling pointer created here")]
     pub created_at: Option,
 }
 
 // multiple_supertrait_upcastable.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_multiple_supertrait_upcastable)]
+#[diag("`{$ident}` is dyn-compatible and has multiple supertraits")]
 pub(crate) struct MultipleSupertraitUpcastable {
     pub ident: Ident,
 }
 
 // non_ascii_idents.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_identifier_non_ascii_char)]
+#[diag("identifier contains non-ASCII characters")]
 pub(crate) struct IdentifierNonAsciiChar;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_identifier_uncommon_codepoints)]
-#[note]
+#[diag(
+    "identifier contains {$codepoints_len ->
+        [one] { $identifier_type ->
+            [Exclusion] a character from an archaic script
+            [Technical] a character that is for non-linguistic, specialized usage
+            [Limited_Use] a character from a script in limited use
+            [Not_NFKC] a non normalized (NFKC) character
+            *[other] an uncommon character
+        }
+        *[other] { $identifier_type ->
+            [Exclusion] {$codepoints_len} characters from archaic scripts
+            [Technical] {$codepoints_len} characters that are for non-linguistic, specialized usage
+            [Limited_Use] {$codepoints_len} characters from scripts in limited use
+            [Not_NFKC] {$codepoints_len} non normalized (NFKC) characters
+            *[other] uncommon characters
+        }
+    }: {$codepoints}"
+)]
+#[note(
+    r#"{$codepoints_len ->
+        [one] this character is
+        *[other] these characters are
+    } included in the{$identifier_type ->
+        [Restricted] {""}
+        *[other] {" "}{$identifier_type}
+    } Unicode general security profile"#
+)]
 pub(crate) struct IdentifierUncommonCodepoints {
     pub codepoints: Vec,
     pub codepoints_len: usize,
@@ -1182,20 +1422,22 @@ pub(crate) struct IdentifierUncommonCodepoints {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_confusable_identifier_pair)]
+#[diag("found both `{$existing_sym}` and `{$sym}` as identifiers, which look alike")]
 pub(crate) struct ConfusableIdentifierPair {
     pub existing_sym: Symbol,
     pub sym: Symbol,
-    #[label(lint_other_use)]
+    #[label("other identifier used here")]
     pub label: Span,
-    #[label(lint_current_use)]
+    #[label("this identifier can be confused with `{$existing_sym}`")]
     pub main_label: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_mixed_script_confusables)]
-#[note(lint_includes_note)]
-#[note]
+#[diag(
+    "the usage of Script Group `{$set}` in this crate consists solely of mixed script confusables"
+)]
+#[note("the usage includes {$includes}")]
+#[note("please recheck to make sure their usages are indeed what you want")]
 pub(crate) struct MixedScriptConfusables {
     pub set: String,
     pub includes: String,
@@ -1210,19 +1452,34 @@ pub(crate) struct NonFmtPanicUnused {
 // Used because of two suggestions based on one Option
 impl<'a> LintDiagnostic<'a, ()> for NonFmtPanicUnused {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_non_fmt_panic_unused);
+        diag.primary_message(inline_fluent!(
+            "panic message contains {$count ->
+                [one] an unused
+                *[other] unused
+            } formatting {$count ->
+                [one] placeholder
+                *[other] placeholders
+            }"
+        ));
         diag.arg("count", self.count);
-        diag.note(fluent::lint_note);
+        diag.note(inline_fluent!("this message is not used as a format string when given without arguments, but will be in Rust 2021"));
         if let Some(span) = self.suggestion {
             diag.span_suggestion(
                 span.shrink_to_hi(),
-                fluent::lint_add_args_suggestion,
+                inline_fluent!(
+                    "add the missing {$count ->
+                        [one] argument
+                        *[other] arguments
+                    }"
+                ),
                 ", ...",
                 Applicability::HasPlaceholders,
             );
             diag.span_suggestion(
                 span.shrink_to_lo(),
-                fluent::lint_add_fmt_suggestion,
+                inline_fluent!(
+                    r#"or add a "{"{"}{"}"}" format string to use the message literally"#
+                ),
                 "\"{}\", ",
                 Applicability::MachineApplicable,
             );
@@ -1231,17 +1488,26 @@ impl<'a> LintDiagnostic<'a, ()> for NonFmtPanicUnused {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_non_fmt_panic_braces)]
-#[note]
+#[diag(
+    "panic message contains {$count ->
+        [one] a brace
+        *[other] braces
+    }"
+)]
+#[note("this message is not used as a format string, but will be in Rust 2021")]
 pub(crate) struct NonFmtPanicBraces {
     pub count: usize,
-    #[suggestion(code = "\"{{}}\", ", applicability = "machine-applicable")]
+    #[suggestion(
+        "add a \"{\"{\"}{\"}\"}\" format string to use the message literally",
+        code = "\"{{}}\", ",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Option,
 }
 
 // nonstandard_style.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_non_camel_case_type)]
+#[diag("{$sort} `{$name}` should have an upper camel case name")]
 pub(crate) struct NonCamelCaseType<'a> {
     pub sort: &'a str,
     pub name: &'a str,
@@ -1251,12 +1517,16 @@ pub(crate) struct NonCamelCaseType<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum NonCamelCaseTypeSub {
-    #[label(lint_label)]
+    #[label("should have an UpperCamelCase name")]
     Label {
         #[primary_span]
         span: Span,
     },
-    #[suggestion(lint_suggestion, code = "{replace}", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "convert the identifier to upper camel case",
+        code = "{replace}",
+        applicability = "maybe-incorrect"
+    )]
     Suggestion {
         #[primary_span]
         span: Span,
@@ -1265,7 +1535,7 @@ pub(crate) enum NonCamelCaseTypeSub {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_non_snake_case)]
+#[diag("{$sort} `{$name}` should have a snake case name")]
 pub(crate) struct NonSnakeCaseDiag<'a> {
     pub sort: &'a str,
     pub name: &'a str,
@@ -1286,15 +1556,15 @@ impl Subdiagnostic for NonSnakeCaseDiagSub {
     fn add_to_diag(self, diag: &mut Diag<'_, G>) {
         match self {
             NonSnakeCaseDiagSub::Label { span } => {
-                diag.span_label(span, fluent::lint_label);
+                diag.span_label(span, inline_fluent!("should have a snake_case name"));
             }
             NonSnakeCaseDiagSub::Help => {
-                diag.help(fluent::lint_help);
+                diag.help(inline_fluent!("convert the identifier to snake case: `{$sc}`"));
             }
             NonSnakeCaseDiagSub::ConvertSuggestion { span, suggestion } => {
                 diag.span_suggestion(
                     span,
-                    fluent::lint_convert_suggestion,
+                    inline_fluent!("convert the identifier to snake case"),
                     suggestion,
                     Applicability::MaybeIncorrect,
                 );
@@ -1302,16 +1572,18 @@ impl Subdiagnostic for NonSnakeCaseDiagSub {
             NonSnakeCaseDiagSub::RenameOrConvertSuggestion { span, suggestion } => {
                 diag.span_suggestion(
                     span,
-                    fluent::lint_rename_or_convert_suggestion,
+                    inline_fluent!(
+                        "rename the identifier or convert it to a snake case raw identifier"
+                    ),
                     suggestion,
                     Applicability::MaybeIncorrect,
                 );
             }
             NonSnakeCaseDiagSub::SuggestionAndNote { span } => {
-                diag.note(fluent::lint_cannot_convert_note);
+                diag.note(inline_fluent!("`{$sc}` cannot be used as a raw identifier"));
                 diag.span_suggestion(
                     span,
-                    fluent::lint_rename_suggestion,
+                    inline_fluent!("rename the identifier"),
                     "",
                     Applicability::MaybeIncorrect,
                 );
@@ -1321,7 +1593,7 @@ impl Subdiagnostic for NonSnakeCaseDiagSub {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_non_upper_case_global)]
+#[diag("{$sort} `{$name}` should have an upper case name")]
 pub(crate) struct NonUpperCaseGlobal<'a> {
     pub sort: &'a str,
     pub name: &'a str,
@@ -1333,12 +1605,12 @@ pub(crate) struct NonUpperCaseGlobal<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum NonUpperCaseGlobalSub {
-    #[label(lint_label)]
+    #[label("should have an UPPER_CASE name")]
     Label {
         #[primary_span]
         span: Span,
     },
-    #[suggestion(lint_suggestion, code = "{replace}")]
+    #[suggestion("convert the identifier to upper case", code = "{replace}")]
     Suggestion {
         #[primary_span]
         span: Span,
@@ -1350,7 +1622,7 @@ pub(crate) enum NonUpperCaseGlobalSub {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    lint_suggestion,
+    "convert the identifier to upper case",
     code = "{replace}",
     applicability = "machine-applicable",
     style = "tool-only"
@@ -1363,16 +1635,18 @@ pub(crate) struct NonUpperCaseGlobalSubTool {
 
 // noop_method_call.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_noop_method_call)]
-#[note]
+#[diag("call to `.{$method}()` on a reference in this situation does nothing")]
+#[note(
+    "the type `{$orig_ty}` does not implement `{$trait_}`, so calling `{$method}` on `&{$orig_ty}` copies the reference, which does not do anything and can be removed"
+)]
 pub(crate) struct NoopMethodCallDiag<'a> {
     pub method: Ident,
     pub orig_ty: Ty<'a>,
     pub trait_: Symbol,
-    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[suggestion("remove this redundant call", code = "", applicability = "machine-applicable")]
     pub label: Span,
     #[suggestion(
-        lint_derive_suggestion,
+        "if you meant to clone `{$orig_ty}`, implement `Clone` for it",
         code = "#[derive(Clone)]\n",
         applicability = "maybe-incorrect"
     )]
@@ -1380,13 +1654,17 @@ pub(crate) struct NoopMethodCallDiag<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_suspicious_double_ref_deref)]
+#[diag(
+    "using `.deref()` on a double reference, which returns `{$ty}` instead of dereferencing the inner type"
+)]
 pub(crate) struct SuspiciousDoubleRefDerefDiag<'a> {
     pub ty: Ty<'a>,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_suspicious_double_ref_clone)]
+#[diag(
+    "using `.clone()` on a double reference, which returns `{$ty}` instead of cloning the inner type"
+)]
 pub(crate) struct SuspiciousDoubleRefCloneDiag<'a> {
     pub ty: Ty<'a>,
 }
@@ -1423,7 +1701,7 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 doctest,
                 macro_to_change,
             } => {
-                diag.primary_message(fluent::lint_non_local_definitions_impl);
+                diag.primary_message(inline_fluent!("non-local `impl` definition, `impl` blocks should be written at the same level as their item"));
                 diag.arg("depth", depth);
                 diag.arg("body_kind_descr", body_kind_descr);
                 diag.arg("body_name", body_name);
@@ -1431,24 +1709,24 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 if let Some((macro_to_change, macro_kind)) = macro_to_change {
                     diag.arg("macro_to_change", macro_to_change);
                     diag.arg("macro_kind", macro_kind);
-                    diag.note(fluent::lint_macro_to_change);
+                    diag.note(inline_fluent!("the {$macro_kind} `{$macro_to_change}` defines the non-local `impl`, and may need to be changed"));
                 }
                 if let Some(cargo_update) = cargo_update {
                     diag.subdiagnostic(cargo_update);
                 }
 
-                diag.note(fluent::lint_non_local);
+                diag.note(inline_fluent!("an `impl` is never scoped, even when it is nested inside an item, as it may impact type checking outside of that item, which can be the case if neither the trait or the self type are at the same nesting level as the `impl`"));
 
                 if doctest {
-                    diag.help(fluent::lint_doctest);
+                    diag.help(inline_fluent!("make this doc-test a standalone test with its own `fn main() {\"{\"} ... {\"}\"}`"));
                 }
 
                 if let Some(const_anon) = const_anon {
-                    diag.note(fluent::lint_exception);
+                    diag.note(inline_fluent!("items in an anonymous const item (`const _: () = {\"{\"} ... {\"}\"}`) are treated as in the same scope as the anonymous const's declaration for the purpose of this lint"));
                     if let Some(const_anon) = const_anon {
                         diag.span_suggestion(
                             const_anon,
-                            fluent::lint_const_anon,
+                            inline_fluent!("use a const-anon item to suppress this lint"),
                             "_",
                             Applicability::MachineApplicable,
                         );
@@ -1462,18 +1740,21 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 doctest,
                 cargo_update,
             } => {
-                diag.primary_message(fluent::lint_non_local_definitions_macro_rules);
+                diag.primary_message(inline_fluent!("non-local `macro_rules!` definition, `#[macro_export]` macro should be written at top level module"));
                 diag.arg("depth", depth);
                 diag.arg("body_kind_descr", body_kind_descr);
                 diag.arg("body_name", body_name);
 
                 if doctest {
-                    diag.help(fluent::lint_help_doctest);
+                    diag.help(inline_fluent!(r#"remove the `#[macro_export]` or make this doc-test a standalone test with its own `fn main() {"{"} ... {"}"}`"#));
                 } else {
-                    diag.help(fluent::lint_help);
+                    diag.help(inline_fluent!("remove the `#[macro_export]` or move this `macro_rules!` outside the of the current {$body_kind_descr} {$depth ->
+                        [one] `{$body_name}`
+                       *[other] `{$body_name}` and up {$depth} bodies
+                    }"));
                 }
 
-                diag.note(fluent::lint_non_local);
+                diag.note(inline_fluent!("a `macro_rules!` definition is non-local if it is nested inside an item and has a `#[macro_export]` attribute"));
 
                 if let Some(cargo_update) = cargo_update {
                     diag.subdiagnostic(cargo_update);
@@ -1484,7 +1765,9 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_non_local_definitions_cargo_update)]
+#[note(
+    "the {$macro_kind} `{$macro_name}` may come from an old version of the `{$crate_name}` crate, try updating your dependency with `cargo update -p {$crate_name}`"
+)]
 pub(crate) struct NonLocalDefinitionsCargoUpdateNote {
     pub macro_kind: &'static str,
     pub macro_name: Symbol,
@@ -1493,8 +1776,8 @@ pub(crate) struct NonLocalDefinitionsCargoUpdateNote {
 
 // precedence.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_ambiguous_negative_literals)]
-#[note(lint_example)]
+#[diag("`-` has lower precedence than method calls, which might be unexpected")]
+#[note("e.g. `-4.abs()` equals `-4`; while `(-4).abs()` equals `4`")]
 pub(crate) struct AmbiguousNegativeLiteralsDiag {
     #[subdiagnostic]
     pub negative_literal: AmbiguousNegativeLiteralsNegativeLiteralSuggestion,
@@ -1503,7 +1786,10 @@ pub(crate) struct AmbiguousNegativeLiteralsDiag {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_negative_literal, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "add parentheses around the `-` and the literal to call the method on a negative literal",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct AmbiguousNegativeLiteralsNegativeLiteralSuggestion {
     #[suggestion_part(code = "(")]
     pub start_span: Span,
@@ -1512,7 +1798,10 @@ pub(crate) struct AmbiguousNegativeLiteralsNegativeLiteralSuggestion {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_current_behavior, applicability = "maybe-incorrect")]
+#[multipart_suggestion(
+    "add parentheses around the literal and the method call to keep the current behavior",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct AmbiguousNegativeLiteralsCurrentBehaviorSuggestion {
     #[suggestion_part(code = "(")]
     pub start_span: Span,
@@ -1522,16 +1811,21 @@ pub(crate) struct AmbiguousNegativeLiteralsCurrentBehaviorSuggestion {
 
 // pass_by_value.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_pass_by_value)]
+#[diag("passing `{$ty}` by reference")]
 pub(crate) struct PassByValueDiag {
     pub ty: String,
-    #[suggestion(code = "{ty}", applicability = "maybe-incorrect")]
+    #[suggestion("try passing by value", code = "{ty}", applicability = "maybe-incorrect")]
     pub suggestion: Span,
 }
 
 // redundant_semicolon.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_redundant_semicolons)]
+#[diag(
+    "unnecessary trailing {$multiple ->
+        [true] semicolons
+        *[false] semicolon
+    }"
+)]
 pub(crate) struct RedundantSemicolonsDiag {
     pub multiple: bool,
     #[subdiagnostic]
@@ -1539,7 +1833,14 @@ pub(crate) struct RedundantSemicolonsDiag {
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(lint_redundant_semicolons_suggestion, code = "", applicability = "maybe-incorrect")]
+#[suggestion(
+    "remove {$multiple_semicolons ->
+        [true] these semicolons
+        *[false] this semicolon
+    }",
+    code = "",
+    applicability = "maybe-incorrect"
+)]
 pub(crate) struct RedundantSemicolonsSuggestion {
     pub multiple_semicolons: bool,
     #[primary_span]
@@ -1556,7 +1857,7 @@ pub(crate) struct DropTraitConstraintsDiag<'a> {
 // Needed for def_path_str
 impl<'a> LintDiagnostic<'a, ()> for DropTraitConstraintsDiag<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_drop_trait_constraints);
+        diag.primary_message(inline_fluent!("bounds on `{$predicate}` are most likely incorrect, consider instead using `{$needs_drop}` to detect whether a type can be trivially dropped"));
         diag.arg("predicate", self.predicate);
         diag.arg("needs_drop", self.tcx.def_path_str(self.def_id));
     }
@@ -1570,19 +1871,27 @@ pub(crate) struct DropGlue<'a> {
 // Needed for def_path_str
 impl<'a> LintDiagnostic<'a, ()> for DropGlue<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_drop_glue);
+        diag.primary_message(inline_fluent!("types that do not implement `Drop` can still have drop glue, consider instead using `{$needs_drop}` to detect whether a type is trivially dropped"));
         diag.arg("needs_drop", self.tcx.def_path_str(self.def_id));
     }
 }
 
 // transmute.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_int_to_ptr_transmutes)]
-#[note]
-#[note(lint_note_exposed_provenance)]
-#[help(lint_suggestion_without_provenance_mut)]
-#[help(lint_help_transmute)]
-#[help(lint_help_exposed_provenance)]
+#[diag("transmuting an integer to a pointer creates a pointer without provenance")]
+#[note("this is dangerous because dereferencing the resulting pointer is undefined behavior")]
+#[note(
+    "exposed provenance semantics can be used to create a pointer based on some previously exposed provenance"
+)]
+#[help(
+    "if you truly mean to create a pointer without provenance, use `std::ptr::without_provenance_mut`"
+)]
+#[help(
+    "for more information about transmute, see "
+)]
+#[help(
+    "for more information about exposed provenance, see "
+)]
 pub(crate) struct IntegerToPtrTransmutes<'tcx> {
     #[subdiagnostic]
     pub suggestion: Option>,
@@ -1591,7 +1900,7 @@ pub(crate) struct IntegerToPtrTransmutes<'tcx> {
 #[derive(Subdiagnostic)]
 pub(crate) enum IntegerToPtrTransmutesSuggestion<'tcx> {
     #[multipart_suggestion(
-        lint_suggestion_with_exposed_provenance,
+        "use `std::ptr::with_exposed_provenance{$suffix}` instead to use a previously exposed provenance",
         applicability = "machine-applicable",
         style = "verbose"
     )]
@@ -1602,7 +1911,7 @@ pub(crate) enum IntegerToPtrTransmutesSuggestion<'tcx> {
         start_call: Span,
     },
     #[multipart_suggestion(
-        lint_suggestion_with_exposed_provenance,
+        "use `std::ptr::with_exposed_provenance{$suffix}` instead to use a previously exposed provenance",
         applicability = "machine-applicable",
         style = "verbose"
     )]
@@ -1619,7 +1928,7 @@ pub(crate) enum IntegerToPtrTransmutesSuggestion<'tcx> {
 
 // types.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_range_endpoint_out_of_range)]
+#[diag("range endpoint is out of range for `{$ty}`")]
 pub(crate) struct RangeEndpointOutOfRange<'a> {
     pub ty: &'a str,
     #[subdiagnostic]
@@ -1629,7 +1938,7 @@ pub(crate) struct RangeEndpointOutOfRange<'a> {
 #[derive(Subdiagnostic)]
 pub(crate) enum UseInclusiveRange<'a> {
     #[suggestion(
-        lint_range_use_inclusive_range,
+        "use an inclusive range instead",
         code = "{start}..={literal}{suffix}",
         applicability = "machine-applicable"
     )]
@@ -1640,7 +1949,7 @@ pub(crate) enum UseInclusiveRange<'a> {
         literal: u128,
         suffix: &'a str,
     },
-    #[multipart_suggestion(lint_range_use_inclusive_range, applicability = "machine-applicable")]
+    #[multipart_suggestion("use an inclusive range instead", applicability = "machine-applicable")]
     WithParen {
         #[suggestion_part(code = "=")]
         eq_sugg: Span,
@@ -1652,7 +1961,7 @@ pub(crate) enum UseInclusiveRange<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_overflowing_bin_hex)]
+#[diag("literal out of range for `{$ty}`")]
 pub(crate) struct OverflowingBinHex<'a> {
     pub ty: &'a str,
     pub lit: String,
@@ -1675,11 +1984,13 @@ impl Subdiagnostic for OverflowingBinHexSign {
     fn add_to_diag(self, diag: &mut Diag<'_, G>) {
         match self {
             OverflowingBinHexSign::Positive => {
-                diag.note(fluent::lint_positive_note);
+                diag.note(inline_fluent!("the literal `{$lit}` (decimal `{$dec}`) does not fit into the type `{$ty}` and will become `{$actually}{$ty}`"));
             }
             OverflowingBinHexSign::Negative => {
-                diag.note(fluent::lint_negative_note);
-                diag.note(fluent::lint_negative_becomes_note);
+                diag.note(inline_fluent!(
+                    "the literal `{$lit}` (decimal `{$dec}`) does not fit into the type `{$ty}`"
+                ));
+                diag.note(inline_fluent!("and the value `-{$lit}` will become `{$actually}{$ty}`"));
             }
         }
     }
@@ -1688,7 +1999,7 @@ impl Subdiagnostic for OverflowingBinHexSign {
 #[derive(Subdiagnostic)]
 pub(crate) enum OverflowingBinHexSub<'a> {
     #[suggestion(
-        lint_suggestion,
+        "consider using the type `{$suggestion_ty}` instead",
         code = "{sans_suffix}{suggestion_ty}",
         applicability = "machine-applicable"
     )]
@@ -1698,13 +2009,13 @@ pub(crate) enum OverflowingBinHexSub<'a> {
         suggestion_ty: &'a str,
         sans_suffix: &'a str,
     },
-    #[help(lint_help)]
+    #[help("consider using the type `{$suggestion_ty}` instead")]
     Help { suggestion_ty: &'a str },
 }
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    lint_sign_bit_suggestion,
+    "to use as a negative number (decimal `{$negative_val}`), consider using the type `{$uint_ty}` for the literal and cast it to `{$int_ty}`",
     code = "{lit_no_suffix}{uint_ty} as {int_ty}",
     applicability = "maybe-incorrect"
 )]
@@ -1718,8 +2029,8 @@ pub(crate) struct OverflowingBinHexSignBitSub<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_overflowing_int)]
-#[note]
+#[diag("literal out of range for `{$ty}`")]
+#[note("the literal `{$lit}` does not fit into the type `{$ty}` whose range is `{$min}..={$max}`")]
 pub(crate) struct OverflowingInt<'a> {
     pub ty: &'a str,
     pub lit: String,
@@ -1730,22 +2041,26 @@ pub(crate) struct OverflowingInt<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[help(lint_help)]
+#[help("consider using the type `{$suggestion_ty}` instead")]
 pub(crate) struct OverflowingIntHelp<'a> {
     pub suggestion_ty: &'a str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_only_cast_u8_to_char)]
+#[diag("only `u8` can be cast into `char`")]
 pub(crate) struct OnlyCastu8ToChar {
-    #[suggestion(code = "'\\u{{{literal:X}}}'", applicability = "machine-applicable")]
+    #[suggestion(
+        "use a `char` literal instead",
+        code = "'\\u{{{literal:X}}}'",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
     pub literal: u128,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_overflowing_uint)]
-#[note]
+#[diag("literal out of range for `{$ty}`")]
+#[note("the literal `{$lit}` does not fit into the type `{$ty}` whose range is `{$min}..={$max}`")]
 pub(crate) struct OverflowingUInt<'a> {
     pub ty: &'a str,
     pub lit: String,
@@ -1754,50 +2069,54 @@ pub(crate) struct OverflowingUInt<'a> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_overflowing_literal)]
-#[note]
+#[diag("literal out of range for `{$ty}`")]
+#[note(
+    "the literal `{$lit}` does not fit into the type `{$ty}` and will be converted to `{$ty}::INFINITY`"
+)]
 pub(crate) struct OverflowingLiteral<'a> {
     pub ty: &'a str,
     pub lit: String,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_surrogate_char_cast)]
-#[note]
+#[diag("surrogate values are not valid for `char`")]
+#[note("`0xD800..=0xDFFF` are reserved for Unicode surrogates and are not valid `char` values")]
 pub(crate) struct SurrogateCharCast {
     pub literal: u128,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_too_large_char_cast)]
-#[note]
+#[diag("value exceeds maximum `char` value")]
+#[note("maximum valid `char` value is `0x10FFFF`")]
 pub(crate) struct TooLargeCharCast {
     pub literal: u128,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_uses_power_alignment)]
+#[diag(
+    "repr(C) does not follow the power alignment rule. This may affect platform C ABI compatibility for this type"
+)]
 pub(crate) struct UsesPowerAlignment;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_comparisons)]
+#[diag("comparison is useless due to type limits")]
 pub(crate) struct UnusedComparisons;
 
 #[derive(LintDiagnostic)]
 pub(crate) enum InvalidNanComparisons {
-    #[diag(lint_invalid_nan_comparisons_eq_ne)]
+    #[diag("incorrect NaN comparison, NaN cannot be directly compared to itself")]
     EqNe {
         #[subdiagnostic]
         suggestion: InvalidNanComparisonsSuggestion,
     },
-    #[diag(lint_invalid_nan_comparisons_lt_le_gt_ge)]
+    #[diag("incorrect NaN comparison, NaN is not orderable")]
     LtLeGtGe,
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidNanComparisonsSuggestion {
     #[multipart_suggestion(
-        lint_suggestion,
+        "use `f32::is_nan()` or `f64::is_nan()` instead",
         style = "verbose",
         applicability = "machine-applicable"
     )]
@@ -1809,35 +2128,41 @@ pub(crate) enum InvalidNanComparisonsSuggestion {
         #[suggestion_part(code = "")]
         nan_plus_binop: Span,
     },
-    #[help(lint_suggestion)]
+    #[help("use `f32::is_nan()` or `f64::is_nan()` instead")]
     Spanless,
 }
 
 #[derive(LintDiagnostic)]
 pub(crate) enum AmbiguousWidePointerComparisons<'a> {
-    #[diag(lint_ambiguous_wide_pointer_comparisons)]
+    #[diag(
+        "ambiguous wide pointer comparison, the comparison includes metadata which may not be expected"
+    )]
     SpanfulEq {
         #[subdiagnostic]
         addr_suggestion: AmbiguousWidePointerComparisonsAddrSuggestion<'a>,
         #[subdiagnostic]
         addr_metadata_suggestion: Option>,
     },
-    #[diag(lint_ambiguous_wide_pointer_comparisons)]
+    #[diag(
+        "ambiguous wide pointer comparison, the comparison includes metadata which may not be expected"
+    )]
     SpanfulCmp {
         #[subdiagnostic]
         cast_suggestion: AmbiguousWidePointerComparisonsCastSuggestion<'a>,
         #[subdiagnostic]
         expect_suggestion: AmbiguousWidePointerComparisonsExpectSuggestion<'a>,
     },
-    #[diag(lint_ambiguous_wide_pointer_comparisons)]
-    #[help(lint_addr_metadata_suggestion)]
-    #[help(lint_addr_suggestion)]
+    #[diag(
+        "ambiguous wide pointer comparison, the comparison includes metadata which may not be expected"
+    )]
+    #[help("use explicit `std::ptr::eq` method to compare metadata and addresses")]
+    #[help("use `std::ptr::addr_eq` or untyped pointers to only compare their addresses")]
     Spanless,
 }
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    lint_addr_metadata_suggestion,
+    "use explicit `std::ptr::eq` method to compare metadata and addresses",
     style = "verbose",
     // FIXME(#53934): make machine-applicable again
     applicability = "maybe-incorrect"
@@ -1858,7 +2183,7 @@ pub(crate) struct AmbiguousWidePointerComparisonsAddrMetadataSuggestion<'a> {
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    lint_addr_suggestion,
+    "use `std::ptr::addr_eq` or untyped pointers to only compare their addresses",
     style = "verbose",
     // FIXME(#53934): make machine-applicable again
     applicability = "maybe-incorrect"
@@ -1879,7 +2204,7 @@ pub(crate) struct AmbiguousWidePointerComparisonsAddrSuggestion<'a> {
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    lint_cast_suggestion,
+    "use untyped pointers to only compare their addresses",
     style = "verbose",
     // FIXME(#53934): make machine-applicable again
     applicability = "maybe-incorrect"
@@ -1903,7 +2228,7 @@ pub(crate) struct AmbiguousWidePointerComparisonsCastSuggestion<'a> {
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    lint_expect_suggestion,
+    "or expect the lint to compare the pointers metadata and addresses",
     style = "verbose",
     // FIXME(#53934): make machine-applicable again
     applicability = "maybe-incorrect"
@@ -1922,25 +2247,37 @@ pub(crate) struct AmbiguousWidePointerComparisonsExpectSuggestion<'a> {
 
 #[derive(LintDiagnostic)]
 pub(crate) enum UnpredictableFunctionPointerComparisons<'a, 'tcx> {
-    #[diag(lint_unpredictable_fn_pointer_comparisons)]
-    #[note(lint_note_duplicated_fn)]
-    #[note(lint_note_deduplicated_fn)]
-    #[note(lint_note_visit_fn_addr_eq)]
+    #[diag(
+        "function pointer comparisons do not produce meaningful results since their addresses are not guaranteed to be unique"
+    )]
+    #[note("the address of the same function can vary between different codegen units")]
+    #[note(
+        "furthermore, different functions could have the same address after being merged together"
+    )]
+    #[note(
+        "for more information visit "
+    )]
     Suggestion {
         #[subdiagnostic]
         sugg: UnpredictableFunctionPointerComparisonsSuggestion<'a, 'tcx>,
     },
-    #[diag(lint_unpredictable_fn_pointer_comparisons)]
-    #[note(lint_note_duplicated_fn)]
-    #[note(lint_note_deduplicated_fn)]
-    #[note(lint_note_visit_fn_addr_eq)]
+    #[diag(
+        "function pointer comparisons do not produce meaningful results since their addresses are not guaranteed to be unique"
+    )]
+    #[note("the address of the same function can vary between different codegen units")]
+    #[note(
+        "furthermore, different functions could have the same address after being merged together"
+    )]
+    #[note(
+        "for more information visit "
+    )]
     Warn,
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum UnpredictableFunctionPointerComparisonsSuggestion<'a, 'tcx> {
     #[multipart_suggestion(
-        lint_fn_addr_eq_suggestion,
+        "refactor your code, or use `std::ptr::fn_addr_eq` to suppress the lint",
         style = "verbose",
         applicability = "maybe-incorrect"
     )]
@@ -1956,7 +2293,7 @@ pub(crate) enum UnpredictableFunctionPointerComparisonsSuggestion<'a, 'tcx> {
         right: Span,
     },
     #[multipart_suggestion(
-        lint_fn_addr_eq_suggestion,
+        "refactor your code, or use `std::ptr::fn_addr_eq` to suppress the lint",
         style = "verbose",
         applicability = "maybe-incorrect"
     )]
@@ -1986,69 +2323,73 @@ pub(crate) struct ImproperCTypes<'a> {
 // Used because of the complexity of Option, DiagMessage, and Option
 impl<'a> LintDiagnostic<'a, ()> for ImproperCTypes<'_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_improper_ctypes);
+        diag.primary_message(inline_fluent!(
+            "`extern` {$desc} uses type `{$ty}`, which is not FFI-safe"
+        ));
         diag.arg("ty", self.ty);
         diag.arg("desc", self.desc);
-        diag.span_label(self.label, fluent::lint_label);
+        diag.span_label(self.label, inline_fluent!("not FFI-safe"));
         if let Some(help) = self.help {
             diag.help(help);
         }
         diag.note(self.note);
         if let Some(note) = self.span_note {
-            diag.span_note(note, fluent::lint_note);
+            diag.span_note(note, inline_fluent!("the type is defined here"));
         }
     }
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_improper_gpu_kernel_arg)]
-#[help]
+#[diag("passing type `{$ty}` to a function with \"gpu-kernel\" ABI may have unexpected behavior")]
+#[help("use primitive types and raw pointers to get reliable behavior")]
 pub(crate) struct ImproperGpuKernelArg<'a> {
     pub ty: Ty<'a>,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_missing_gpu_kernel_export_name)]
-#[help]
-#[note]
+#[diag("function with the \"gpu-kernel\" ABI has a mangled name")]
+#[help("use `unsafe(no_mangle)` or `unsafe(export_name = \"\")`")]
+#[note("mangled names make it hard to find the kernel, this is usually not intended")]
 pub(crate) struct MissingGpuKernelExportName;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_variant_size_differences)]
+#[diag("enum variant is more than three times larger ({$largest} bytes) than the next largest")]
 pub(crate) struct VariantSizeDifferencesDiag {
     pub largest: u64,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_atomic_ordering_load)]
-#[help]
+#[diag("atomic loads cannot have `Release` or `AcqRel` ordering")]
+#[help("consider using ordering modes `Acquire`, `SeqCst` or `Relaxed`")]
 pub(crate) struct AtomicOrderingLoad;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_atomic_ordering_store)]
-#[help]
+#[diag("atomic stores cannot have `Acquire` or `AcqRel` ordering")]
+#[help("consider using ordering modes `Release`, `SeqCst` or `Relaxed`")]
 pub(crate) struct AtomicOrderingStore;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_atomic_ordering_fence)]
-#[help]
+#[diag("memory fences cannot have `Relaxed` ordering")]
+#[help("consider using ordering modes `Acquire`, `Release`, `AcqRel` or `SeqCst`")]
 pub(crate) struct AtomicOrderingFence;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_atomic_ordering_invalid)]
-#[help]
+#[diag(
+    "`{$method}`'s failure ordering may not be `Release` or `AcqRel`, since a failed `{$method}` does not result in a write"
+)]
+#[help("consider using `Acquire` or `Relaxed` failure ordering instead")]
 pub(crate) struct InvalidAtomicOrderingDiag {
     pub method: Symbol,
-    #[label]
+    #[label("invalid failure ordering")]
     pub fail_order_arg_span: Span,
 }
 
 // unused.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_op)]
+#[diag("unused {$op} that must be used")]
 pub(crate) struct UnusedOp<'a> {
     pub op: &'a str,
-    #[label]
+    #[label("the {$op} produces a value")]
     pub label: Span,
     #[subdiagnostic]
     pub suggestion: UnusedOpSuggestion,
@@ -2057,7 +2398,7 @@ pub(crate) struct UnusedOp<'a> {
 #[derive(Subdiagnostic)]
 pub(crate) enum UnusedOpSuggestion {
     #[suggestion(
-        lint_suggestion,
+        "use `let _ = ...` to ignore the resulting value",
         style = "verbose",
         code = "let _ = ",
         applicability = "maybe-incorrect"
@@ -2066,7 +2407,11 @@ pub(crate) enum UnusedOpSuggestion {
         #[primary_span]
         span: Span,
     },
-    #[multipart_suggestion(lint_suggestion, style = "verbose", applicability = "maybe-incorrect")]
+    #[multipart_suggestion(
+        "use `let _ = ...` to ignore the resulting value",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
     BlockTailExpr {
         #[suggestion_part(code = "let _ = ")]
         before_span: Span,
@@ -2076,7 +2421,7 @@ pub(crate) enum UnusedOpSuggestion {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_result)]
+#[diag("unused result of type `{$ty}`")]
 pub(crate) struct UnusedResult<'a> {
     pub ty: Ty<'a>,
 }
@@ -2084,8 +2429,13 @@ pub(crate) struct UnusedResult<'a> {
 // FIXME(davidtwco): this isn't properly translatable because of the
 // pre/post strings
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_closure)]
-#[note]
+#[diag(
+    "unused {$pre}{$count ->
+        [one] closure
+        *[other] closures
+    }{$post} that must be used"
+)]
+#[note("closures are lazy and do nothing unless called")]
 pub(crate) struct UnusedClosure<'a> {
     pub count: usize,
     pub pre: &'a str,
@@ -2095,8 +2445,13 @@ pub(crate) struct UnusedClosure<'a> {
 // FIXME(davidtwco): this isn't properly translatable because of the
 // pre/post strings
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_coroutine)]
-#[note]
+#[diag(
+    "unused {$pre}{$count ->
+        [one] coroutine
+        *[other] coroutine
+    }{$post} that must be used"
+)]
+#[note("coroutines are lazy and do nothing unless resumed")]
 pub(crate) struct UnusedCoroutine<'a> {
     pub count: usize,
     pub pre: &'a str,
@@ -2115,10 +2470,9 @@ pub(crate) struct UnusedDef<'a, 'b> {
 }
 
 #[derive(Subdiagnostic)]
-
 pub(crate) enum UnusedDefSuggestion {
     #[suggestion(
-        lint_suggestion,
+        "use `let _ = ...` to ignore the resulting value",
         style = "verbose",
         code = "let _ = ",
         applicability = "maybe-incorrect"
@@ -2127,7 +2481,11 @@ pub(crate) enum UnusedDefSuggestion {
         #[primary_span]
         span: Span,
     },
-    #[multipart_suggestion(lint_suggestion, style = "verbose", applicability = "maybe-incorrect")]
+    #[multipart_suggestion(
+        "use `let _ = ...` to ignore the resulting value",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
     BlockTailExpr {
         #[suggestion_part(code = "let _ = ")]
         before_span: Span,
@@ -2139,7 +2497,7 @@ pub(crate) enum UnusedDefSuggestion {
 // Needed because of def_path_str
 impl<'a> LintDiagnostic<'a, ()> for UnusedDef<'_, '_> {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_unused_def);
+        diag.primary_message(inline_fluent!("unused {$pre}`{$def}`{$post} that must be used"));
         diag.arg("pre", self.pre);
         diag.arg("post", self.post);
         diag.arg("def", self.cx.tcx.def_path_str(self.def_id));
@@ -2154,7 +2512,7 @@ impl<'a> LintDiagnostic<'a, ()> for UnusedDef<'_, '_> {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_path_statement_drop)]
+#[diag("path statement drops value")]
 pub(crate) struct PathStatementDrop {
     #[subdiagnostic]
     pub sub: PathStatementDropSub,
@@ -2162,13 +2520,17 @@ pub(crate) struct PathStatementDrop {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum PathStatementDropSub {
-    #[suggestion(lint_suggestion, code = "drop({snippet});", applicability = "machine-applicable")]
+    #[suggestion(
+        "use `drop` to clarify the intent",
+        code = "drop({snippet});",
+        applicability = "machine-applicable"
+    )]
     Suggestion {
         #[primary_span]
         span: Span,
         snippet: String,
     },
-    #[help(lint_help)]
+    #[help("use `drop` to clarify the intent")]
     Help {
         #[primary_span]
         span: Span,
@@ -2176,11 +2538,11 @@ pub(crate) enum PathStatementDropSub {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_path_statement_no_effect)]
+#[diag("path statement with no effect")]
 pub(crate) struct PathStatementNoEffect;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_delim)]
+#[diag("unnecessary {$delim} around {$item}")]
 pub(crate) struct UnusedDelim<'a> {
     pub delim: &'static str,
     pub item: &'a str,
@@ -2189,7 +2551,7 @@ pub(crate) struct UnusedDelim<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("remove these {$delim}", applicability = "machine-applicable")]
 pub(crate) struct UnusedDelimSuggestion {
     #[suggestion_part(code = "{start_replace}")]
     pub start_span: Span,
@@ -2200,17 +2562,17 @@ pub(crate) struct UnusedDelimSuggestion {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_import_braces)]
+#[diag("braces around {$node} is unnecessary")]
 pub(crate) struct UnusedImportBracesDiag {
     pub node: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_allocation)]
+#[diag("unnecessary allocation, use `&` instead")]
 pub(crate) struct UnusedAllocationDiag;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_allocation_mut)]
+#[diag("unnecessary allocation, use `&mut` instead")]
 pub(crate) struct UnusedAllocationMutDiag;
 
 pub(crate) struct AsyncFnInTraitDiag {
@@ -2219,59 +2581,67 @@ pub(crate) struct AsyncFnInTraitDiag {
 
 impl<'a> LintDiagnostic<'a, ()> for AsyncFnInTraitDiag {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, ()>) {
-        diag.primary_message(fluent::lint_async_fn_in_trait);
-        diag.note(fluent::lint_note);
+        diag.primary_message(inline_fluent!("use of `async fn` in public traits is discouraged as auto trait bounds cannot be specified"));
+        diag.note(inline_fluent!("you can suppress this lint if you plan to use the trait only in your own code, or do not care about auto traits like `Send` on the `Future`"));
         if let Some(sugg) = self.sugg {
-            diag.multipart_suggestion(fluent::lint_suggestion, sugg, Applicability::MaybeIncorrect);
+            diag.multipart_suggestion(inline_fluent!("you can alternatively desugar to a normal `fn` that returns `impl Future` and add any desired bounds such as `Send`, but these cannot be relaxed without a breaking API change"), sugg, Applicability::MaybeIncorrect);
         }
     }
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unit_bindings)]
+#[diag("binding has unit type `()`")]
 pub(crate) struct UnitBindingsDiag {
-    #[label]
+    #[label("this pattern is inferred to be the unit type `()`")]
     pub label: Span,
 }
 
 #[derive(LintDiagnostic)]
 pub(crate) enum InvalidAsmLabel {
-    #[diag(lint_invalid_asm_label_named)]
-    #[help]
-    #[note]
+    #[diag("avoid using named labels in inline assembly")]
+    #[help("only local labels of the form `:` should be used in inline asm")]
+    #[note(
+        "see the asm section of Rust By Example  for more information"
+    )]
     Named {
-        #[note(lint_invalid_asm_label_no_span)]
+        #[note("the label may be declared in the expansion of a macro")]
         missing_precise_span: bool,
     },
-    #[diag(lint_invalid_asm_label_format_arg)]
-    #[help]
-    #[note(lint_note1)]
-    #[note(lint_note2)]
+    #[diag("avoid using named labels in inline assembly")]
+    #[help("only local labels of the form `:` should be used in inline asm")]
+    #[note("format arguments may expand to a non-numeric value")]
+    #[note(
+        "see the asm section of Rust By Example  for more information"
+    )]
     FormatArg {
-        #[note(lint_invalid_asm_label_no_span)]
+        #[note("the label may be declared in the expansion of a macro")]
         missing_precise_span: bool,
     },
-    #[diag(lint_invalid_asm_label_binary)]
-    #[help]
-    #[note(lint_note1)]
-    #[note(lint_note2)]
+    #[diag("avoid using labels containing only the digits `0` and `1` in inline assembly")]
+    #[help("start numbering with `2` instead")]
+    #[note("an LLVM bug makes these labels ambiguous with a binary literal number on x86")]
+    #[note("see  for more information")]
     Binary {
-        #[note(lint_invalid_asm_label_no_span)]
+        #[note("the label may be declared in the expansion of a macro")]
         missing_precise_span: bool,
         // hack to get a label on the whole span, must match the emitted span
-        #[label]
+        #[label("use a different label that doesn't start with `0` or `1`")]
         span: Span,
     },
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum UnexpectedCfgCargoHelp {
-    #[help(lint_unexpected_cfg_add_cargo_feature)]
-    #[help(lint_unexpected_cfg_add_cargo_toml_lint_cfg)]
+    #[help("consider using a Cargo feature instead")]
+    #[help(
+        "or consider adding in `Cargo.toml` the `check-cfg` lint config for the lint:{$cargo_toml_lint_cfg}"
+    )]
     LintCfg { cargo_toml_lint_cfg: String },
-    #[help(lint_unexpected_cfg_add_cargo_feature)]
-    #[help(lint_unexpected_cfg_add_cargo_toml_lint_cfg)]
-    #[help(lint_unexpected_cfg_add_build_rs_println)]
+    #[help("consider using a Cargo feature instead")]
+    #[help(
+        "or consider adding in `Cargo.toml` the `check-cfg` lint config for the lint:{$cargo_toml_lint_cfg}"
+    )]
+    #[help("or consider adding `{$build_rs_println}` to the top of the `build.rs`")]
     LintCfgAndBuildRs { cargo_toml_lint_cfg: String, build_rs_println: String },
 }
 
@@ -2297,7 +2667,7 @@ impl UnexpectedCfgCargoHelp {
 }
 
 #[derive(Subdiagnostic)]
-#[help(lint_unexpected_cfg_add_cmdline_arg)]
+#[help("to expect this configuration use `{$cmdline_arg}`")]
 pub(crate) struct UnexpectedCfgRustcHelp {
     pub cmdline_arg: String,
 }
@@ -2309,17 +2679,23 @@ impl UnexpectedCfgRustcHelp {
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_unexpected_cfg_from_external_macro_origin)]
-#[help(lint_unexpected_cfg_from_external_macro_refer)]
+#[note(
+    "using a cfg inside a {$macro_kind} will use the cfgs from the destination crate and not the ones from the defining crate"
+)]
+#[help("try referring to `{$macro_name}` crate for guidance on how handle this unexpected cfg")]
 pub(crate) struct UnexpectedCfgRustcMacroHelp {
     pub macro_kind: &'static str,
     pub macro_name: Symbol,
 }
 
 #[derive(Subdiagnostic)]
-#[note(lint_unexpected_cfg_from_external_macro_origin)]
-#[help(lint_unexpected_cfg_from_external_macro_refer)]
-#[help(lint_unexpected_cfg_cargo_update)]
+#[note(
+    "using a cfg inside a {$macro_kind} will use the cfgs from the destination crate and not the ones from the defining crate"
+)]
+#[help("try referring to `{$macro_name}` crate for guidance on how handle this unexpected cfg")]
+#[help(
+    "the {$macro_kind} `{$macro_name}` may come from an old version of the `{$crate_name}` crate, try updating your dependency with `cargo update -p {$crate_name}`"
+)]
 pub(crate) struct UnexpectedCfgCargoMacroHelp {
     pub macro_kind: &'static str,
     pub macro_name: Symbol,
@@ -2327,7 +2703,7 @@ pub(crate) struct UnexpectedCfgCargoMacroHelp {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unexpected_cfg_name)]
+#[diag("unexpected `cfg` condition name: `{$name}`")]
 pub(crate) struct UnexpectedCfgName {
     #[subdiagnostic]
     pub code_sugg: unexpected_cfg_name::CodeSuggestion,
@@ -2344,10 +2720,10 @@ pub(crate) mod unexpected_cfg_name {
 
     #[derive(Subdiagnostic)]
     pub(crate) enum CodeSuggestion {
-        #[help(lint_unexpected_cfg_define_features)]
+        #[help("consider defining some features in `Cargo.toml`")]
         DefineFeatures,
         #[multipart_suggestion(
-            lint_unexpected_cfg_name_version_syntax,
+            "there is a similar config predicate: `version(\"..\")`",
             applicability = "machine-applicable"
         )]
         VersionSyntax {
@@ -2357,7 +2733,7 @@ pub(crate) mod unexpected_cfg_name {
             after_value: Span,
         },
         #[suggestion(
-            lint_unexpected_cfg_name_similar_name_value,
+            "there is a config with a similar name and value",
             applicability = "maybe-incorrect",
             code = "{code}"
         )]
@@ -2367,7 +2743,7 @@ pub(crate) mod unexpected_cfg_name {
             code: String,
         },
         #[suggestion(
-            lint_unexpected_cfg_name_similar_name_no_value,
+            "there is a config with a similar name and no value",
             applicability = "maybe-incorrect",
             code = "{code}"
         )]
@@ -2377,7 +2753,7 @@ pub(crate) mod unexpected_cfg_name {
             code: String,
         },
         #[suggestion(
-            lint_unexpected_cfg_name_similar_name_different_values,
+            "there is a config with a similar name and different values",
             applicability = "maybe-incorrect",
             code = "{code}"
         )]
@@ -2389,7 +2765,7 @@ pub(crate) mod unexpected_cfg_name {
             expected: Option,
         },
         #[suggestion(
-            lint_unexpected_cfg_name_similar_name,
+            "there is a config with a similar name",
             applicability = "maybe-incorrect",
             code = "{code}"
         )]
@@ -2407,7 +2783,7 @@ pub(crate) mod unexpected_cfg_name {
             expected_names: Option,
         },
         #[suggestion(
-            lint_unexpected_cfg_boolean,
+            "you may have meant to use `{$literal}` (notice the capitalization). Doing so makes this predicate evaluate to `{$literal}` unconditionally",
             applicability = "machine-applicable",
             style = "verbose",
             code = "{literal}"
@@ -2420,7 +2796,7 @@ pub(crate) mod unexpected_cfg_name {
     }
 
     #[derive(Subdiagnostic)]
-    #[help(lint_unexpected_cfg_name_expected_values)]
+    #[help("expected values for `{$best_match}` are: {$possibilities}")]
     pub(crate) struct ExpectedValues {
         pub best_match: Symbol,
         pub possibilities: DiagSymbolList,
@@ -2428,7 +2804,7 @@ pub(crate) mod unexpected_cfg_name {
 
     #[derive(Subdiagnostic)]
     #[suggestion(
-        lint_unexpected_cfg_name_with_similar_value,
+        "found config with similar value",
         applicability = "maybe-incorrect",
         code = "{code}"
     )]
@@ -2439,7 +2815,12 @@ pub(crate) mod unexpected_cfg_name {
     }
 
     #[derive(Subdiagnostic)]
-    #[help_once(lint_unexpected_cfg_name_expected_names)]
+    #[help_once(
+        "expected names are: {$possibilities}{$and_more ->
+            [0] {\"\"}
+            *[other] {\" \"}and {$and_more} more
+        }"
+    )]
     pub(crate) struct ExpectedNames {
         pub possibilities: DiagSymbolList,
         pub and_more: usize,
@@ -2447,14 +2828,18 @@ pub(crate) mod unexpected_cfg_name {
 
     #[derive(Subdiagnostic)]
     pub(crate) enum InvocationHelp {
-        #[note(lint_unexpected_cfg_doc_cargo)]
+        #[note(
+            "see  for more information about checking conditional configuration"
+        )]
         Cargo {
             #[subdiagnostic]
             macro_help: Option,
             #[subdiagnostic]
             help: Option,
         },
-        #[note(lint_unexpected_cfg_doc_rustc)]
+        #[note(
+            "see  for more information about checking conditional configuration"
+        )]
         Rustc {
             #[subdiagnostic]
             macro_help: Option,
@@ -2465,7 +2850,12 @@ pub(crate) mod unexpected_cfg_name {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unexpected_cfg_value)]
+#[diag(
+    "unexpected `cfg` condition value: {$has_value ->
+        [true] `{$value}`
+        *[false] (none)
+    }"
+)]
 pub(crate) struct UnexpectedCfgValue {
     #[subdiagnostic]
     pub code_sugg: unexpected_cfg_value::CodeSuggestion,
@@ -2489,14 +2879,14 @@ pub(crate) mod unexpected_cfg_value {
             #[subdiagnostic]
             suggestion: Option,
         },
-        #[note(lint_unexpected_cfg_value_no_expected_value)]
+        #[note("no expected value for `{$name}`")]
         RemoveValue {
             #[subdiagnostic]
             suggestion: Option,
 
             name: Symbol,
         },
-        #[note(lint_unexpected_cfg_value_no_expected_values)]
+        #[note("no expected values for `{$name}`")]
         RemoveCondition {
             #[subdiagnostic]
             suggestion: RemoveConditionSuggestion,
@@ -2508,7 +2898,7 @@ pub(crate) mod unexpected_cfg_value {
     #[derive(Subdiagnostic)]
     pub(crate) enum ChangeValueSuggestion {
         #[suggestion(
-            lint_unexpected_cfg_value_similar_name,
+            "there is a expected value with a similar name",
             code = r#""{best_match}""#,
             applicability = "maybe-incorrect"
         )]
@@ -2518,7 +2908,7 @@ pub(crate) mod unexpected_cfg_value {
             best_match: Symbol,
         },
         #[suggestion(
-            lint_unexpected_cfg_value_specify_value,
+            "specify a config value",
             code = r#" = "{first_possibility}""#,
             applicability = "maybe-incorrect"
         )]
@@ -2530,29 +2920,29 @@ pub(crate) mod unexpected_cfg_value {
     }
 
     #[derive(Subdiagnostic)]
-    #[suggestion(
-        lint_unexpected_cfg_value_remove_value,
-        code = "",
-        applicability = "maybe-incorrect"
-    )]
+    #[suggestion("remove the value", code = "", applicability = "maybe-incorrect")]
     pub(crate) struct RemoveValueSuggestion {
         #[primary_span]
         pub span: Span,
     }
 
     #[derive(Subdiagnostic)]
-    #[suggestion(
-        lint_unexpected_cfg_value_remove_condition,
-        code = "",
-        applicability = "maybe-incorrect"
-    )]
+    #[suggestion("remove the condition", code = "", applicability = "maybe-incorrect")]
     pub(crate) struct RemoveConditionSuggestion {
         #[primary_span]
         pub span: Span,
     }
 
     #[derive(Subdiagnostic)]
-    #[note(lint_unexpected_cfg_value_expected_values)]
+    #[note(
+        "expected values for `{$name}` are: {$have_none_possibility ->
+            [true] {\"(none), \"}
+            *[false] {\"\"}
+        }{$possibilities}{$and_more ->
+            [0] {\"\"}
+            *[other] {\" \"}and {$and_more} more
+        }"
+    )]
     pub(crate) struct ExpectedValues {
         pub name: Symbol,
         pub have_none_possibility: bool,
@@ -2562,14 +2952,18 @@ pub(crate) mod unexpected_cfg_value {
 
     #[derive(Subdiagnostic)]
     pub(crate) enum InvocationHelp {
-        #[note(lint_unexpected_cfg_doc_cargo)]
+        #[note(
+            "see  for more information about checking conditional configuration"
+        )]
         Cargo {
             #[subdiagnostic]
             help: Option,
             #[subdiagnostic]
             macro_help: Option,
         },
-        #[note(lint_unexpected_cfg_doc_rustc)]
+        #[note(
+            "see  for more information about checking conditional configuration"
+        )]
         Rustc {
             #[subdiagnostic]
             help: Option,
@@ -2580,19 +2974,19 @@ pub(crate) mod unexpected_cfg_value {
 
     #[derive(Subdiagnostic)]
     pub(crate) enum CargoHelp {
-        #[help(lint_unexpected_cfg_value_add_feature)]
+        #[help("consider adding `{$value}` as a feature in `Cargo.toml`")]
         AddFeature {
             value: Symbol,
         },
-        #[help(lint_unexpected_cfg_define_features)]
+        #[help("consider defining some features in `Cargo.toml`")]
         DefineFeatures,
         Other(#[subdiagnostic] super::UnexpectedCfgCargoHelp),
     }
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_crate_dependency)]
-#[help]
+#[diag("extern crate `{$extern_crate}` is unused in crate `{$local_crate}`")]
+#[help("remove the dependency or add `use {$extern_crate} as _;` to the crate root")]
 pub(crate) struct UnusedCrateDependency {
     pub extern_crate: Symbol,
     pub local_crate: Symbol,
@@ -2600,20 +2994,32 @@ pub(crate) struct UnusedCrateDependency {
 
 // FIXME(jdonszelmann): duplicated in rustc_attr_parsing, should be moved there completely.
 #[derive(LintDiagnostic)]
-#[diag(lint_ill_formed_attribute_input)]
+#[diag(
+    "{$num_suggestions ->
+        [1] attribute must be of the form {$suggestions}
+        *[other] valid forms for the attribute are {$suggestions}
+    }"
+)]
 pub(crate) struct IllFormedAttributeInput {
     pub num_suggestions: usize,
     pub suggestions: DiagArgValue,
-    #[note]
+    #[note("for more information, visit <{$docs}>")]
     pub has_docs: bool,
     pub docs: &'static str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unicode_text_flow)]
-#[note]
+#[diag("unicode codepoint changing visible direction of text present in comment")]
+#[note(
+    "these kind of unicode codepoints change the way text flows on applications that support them, but can cause confusion because they change the order of characters on the screen"
+)]
 pub(crate) struct UnicodeTextFlow {
-    #[label]
+    #[label(
+        "{$num_codepoints ->
+            [1] this comment contains an invisible unicode text flow control codepoint
+            *[other] this comment contains invisible unicode text flow control codepoints
+        }"
+    )]
     pub comment_span: Span,
     #[subdiagnostic]
     pub characters: Vec,
@@ -2624,7 +3030,7 @@ pub(crate) struct UnicodeTextFlow {
 }
 
 #[derive(Subdiagnostic)]
-#[label(lint_label_comment_char)]
+#[label("{$c_debug}")]
 pub(crate) struct UnicodeCharNoteSub {
     #[primary_span]
     pub span: Span,
@@ -2632,21 +3038,27 @@ pub(crate) struct UnicodeCharNoteSub {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "machine-applicable", style = "hidden")]
+#[multipart_suggestion(
+    "if their presence wasn't intentional, you can remove them",
+    applicability = "machine-applicable",
+    style = "hidden"
+)]
 pub(crate) struct UnicodeTextFlowSuggestion {
     #[suggestion_part(code = "")]
     pub spans: Vec,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_abs_path_with_module)]
+#[diag(
+    "absolute paths must start with `self`, `super`, `crate`, or an external crate name in the 2018 edition"
+)]
 pub(crate) struct AbsPathWithModule {
     #[subdiagnostic]
     pub sugg: AbsPathWithModuleSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(lint_suggestion, code = "{replacement}")]
+#[suggestion("use `crate`", code = "{replacement}")]
 pub(crate) struct AbsPathWithModuleSugg {
     #[primary_span]
     pub span: Span,
@@ -2656,18 +3068,23 @@ pub(crate) struct AbsPathWithModuleSugg {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_hidden_lifetime_parameters)]
+#[diag("hidden lifetime parameters in types are deprecated")]
 pub(crate) struct ElidedLifetimesInPaths {
     #[subdiagnostic]
     pub subdiag: ElidedLifetimeInPathSubdiag,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_imports)]
+#[diag(
+    "{$num_snippets ->
+        [one] unused import: {$span_snippets}
+        *[other] unused imports: {$span_snippets}
+    }"
+)]
 pub(crate) struct UnusedImports {
     #[subdiagnostic]
     pub sugg: UnusedImportsSugg,
-    #[help]
+    #[help("if this is a test module, consider adding a `#[cfg(test)]` to the containing module")]
     pub test_module_span: Option,
 
     pub span_snippets: DiagArgValue,
@@ -2677,7 +3094,7 @@ pub(crate) struct UnusedImports {
 #[derive(Subdiagnostic)]
 pub(crate) enum UnusedImportsSugg {
     #[suggestion(
-        lint_suggestion_remove_whole_use,
+        "remove the whole `use` item",
         applicability = "machine-applicable",
         code = "",
         style = "tool-only"
@@ -2687,7 +3104,10 @@ pub(crate) enum UnusedImportsSugg {
         span: Span,
     },
     #[multipart_suggestion(
-        lint_suggestion_remove_imports,
+        "{$num_to_remove ->
+            [one] remove the unused import
+            *[other] remove the unused imports
+        }",
         applicability = "machine-applicable",
         style = "tool-only"
     )]
@@ -2699,7 +3119,7 @@ pub(crate) enum UnusedImportsSugg {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_redundant_import)]
+#[diag("the item `{$ident}` is imported redundantly")]
 pub(crate) struct RedundantImport {
     #[subdiagnostic]
     pub subs: Vec,
@@ -2709,24 +3129,24 @@ pub(crate) struct RedundantImport {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum RedundantImportSub {
-    #[label(lint_label_imported_here)]
+    #[label("the item `{$ident}` is already imported here")]
     ImportedHere(#[primary_span] Span),
-    #[label(lint_label_defined_here)]
+    #[label("the item `{$ident}` is already defined here")]
     DefinedHere(#[primary_span] Span),
-    #[label(lint_label_imported_prelude)]
+    #[label("the item `{$ident}` is already imported by the extern prelude")]
     ImportedPrelude(#[primary_span] Span),
-    #[label(lint_label_defined_prelude)]
+    #[label("the item `{$ident}` is already defined by the extern prelude")]
     DefinedPrelude(#[primary_span] Span),
 }
 
 #[derive(LintDiagnostic)]
 pub(crate) enum PatternsInFnsWithoutBody {
-    #[diag(lint_pattern_in_foreign)]
+    #[diag("patterns aren't allowed in foreign function declarations")]
     Foreign {
         #[subdiagnostic]
         sub: PatternsInFnsWithoutBodySub,
     },
-    #[diag(lint_pattern_in_bodiless)]
+    #[diag("patterns aren't allowed in functions without bodies")]
     Bodiless {
         #[subdiagnostic]
         sub: PatternsInFnsWithoutBodySub,
@@ -2734,7 +3154,11 @@ pub(crate) enum PatternsInFnsWithoutBody {
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(lint_remove_mut_from_pattern, code = "{ident}", applicability = "machine-applicable")]
+#[suggestion(
+    "remove `mut` from the parameter",
+    code = "{ident}",
+    applicability = "machine-applicable"
+)]
 pub(crate) struct PatternsInFnsWithoutBodySub {
     #[primary_span]
     pub span: Span,
@@ -2743,34 +3167,44 @@ pub(crate) struct PatternsInFnsWithoutBodySub {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_reserved_prefix)]
+#[diag("prefix `{$prefix}` is unknown")]
 pub(crate) struct ReservedPrefix {
-    #[label]
+    #[label("unknown prefix")]
     pub label: Span,
-    #[suggestion(code = " ", applicability = "machine-applicable")]
+    #[suggestion(
+        "insert whitespace here to avoid this being parsed as a prefix in Rust 2021",
+        code = " ",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
 
     pub prefix: String,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_raw_prefix)]
+#[diag("prefix `'r` is reserved")]
 pub(crate) struct RawPrefix {
-    #[label]
+    #[label("reserved prefix")]
     pub label: Span,
-    #[suggestion(code = " ", applicability = "machine-applicable")]
+    #[suggestion(
+        "insert whitespace here to avoid this being parsed as a prefix in Rust 2021",
+        code = " ",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_break_with_label_and_loop)]
+#[diag(
+    "this labeled break expression is easy to confuse with an unlabeled break with a labeled value expression"
+)]
 pub(crate) struct BreakWithLabelAndLoop {
     #[subdiagnostic]
     pub sub: BreakWithLabelAndLoopSub,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("wrap this expression in parentheses", applicability = "machine-applicable")]
 pub(crate) struct BreakWithLabelAndLoopSub {
     #[suggestion_part(code = "(")]
     pub left: Span,
@@ -2779,8 +3213,8 @@ pub(crate) struct BreakWithLabelAndLoopSub {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_deprecated_where_clause_location)]
-#[note]
+#[diag("where clause not allowed here")]
+#[note("see issue #89122  for more information")]
 pub(crate) struct DeprecatedWhereClauseLocation {
     #[subdiagnostic]
     pub suggestion: DeprecatedWhereClauseLocationSugg,
@@ -2788,7 +3222,10 @@ pub(crate) struct DeprecatedWhereClauseLocation {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum DeprecatedWhereClauseLocationSugg {
-    #[multipart_suggestion(lint_suggestion_move_to_end, applicability = "machine-applicable")]
+    #[multipart_suggestion(
+        "move it to the end of the type declaration",
+        applicability = "machine-applicable"
+    )]
     MoveToEnd {
         #[suggestion_part(code = "")]
         left: Span,
@@ -2797,7 +3234,7 @@ pub(crate) enum DeprecatedWhereClauseLocationSugg {
 
         sugg: String,
     },
-    #[suggestion(lint_suggestion_remove_where, code = "", applicability = "machine-applicable")]
+    #[suggestion("remove this `where`", code = "", applicability = "machine-applicable")]
     RemoveWhere {
         #[primary_span]
         span: Span,
@@ -2805,11 +3242,11 @@ pub(crate) enum DeprecatedWhereClauseLocationSugg {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_single_use_lifetime)]
+#[diag("lifetime parameter `{$ident}` only used once")]
 pub(crate) struct SingleUseLifetime {
-    #[label(lint_label_param)]
+    #[label("this lifetime...")]
     pub param_span: Span,
-    #[label(lint_label_use)]
+    #[label("...is used only here")]
     pub use_span: Span,
     #[subdiagnostic]
     pub suggestion: Option,
@@ -2818,7 +3255,7 @@ pub(crate) struct SingleUseLifetime {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(lint_suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion("elide the single-use lifetime", applicability = "machine-applicable")]
 pub(crate) struct SingleUseLifetimeSugg {
     #[suggestion_part(code = "")]
     pub deletion_span: Option,
@@ -2829,22 +3266,27 @@ pub(crate) struct SingleUseLifetimeSugg {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_lifetime)]
+#[diag("lifetime parameter `{$ident}` never used")]
 pub(crate) struct UnusedLifetime {
-    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[suggestion("elide the unused lifetime", code = "", applicability = "machine-applicable")]
     pub deletion_span: Option,
 
     pub ident: Ident,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_named_argument_used_positionally)]
+#[diag("named argument `{$named_arg_name}` is not used by name")]
 pub(crate) struct NamedArgumentUsedPositionally {
-    #[label(lint_label_named_arg)]
+    #[label("this named argument is referred to by position in formatting string")]
     pub named_arg_sp: Span,
-    #[label(lint_label_position_arg)]
+    #[label("this formatting argument uses named argument `{$named_arg_name}` by position")]
     pub position_label_sp: Option,
-    #[suggestion(style = "verbose", code = "{name}", applicability = "maybe-incorrect")]
+    #[suggestion(
+        "use the named argument by name to avoid ambiguity",
+        style = "verbose",
+        code = "{name}",
+        applicability = "maybe-incorrect"
+    )]
     pub suggestion: Option,
 
     pub name: String,
@@ -2852,11 +3294,11 @@ pub(crate) struct NamedArgumentUsedPositionally {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_ambiguous_glob_reexport)]
+#[diag("ambiguous glob re-exports")]
 pub(crate) struct AmbiguousGlobReexports {
-    #[label(lint_label_first_reexport)]
+    #[label("the name `{$name}` in the {$namespace} namespace is first re-exported here")]
     pub first_reexport: Span,
-    #[label(lint_label_duplicate_reexport)]
+    #[label("but the name `{$name}` in the {$namespace} namespace is also re-exported here")]
     pub duplicate_reexport: Span,
 
     pub name: String,
@@ -2864,11 +3306,13 @@ pub(crate) struct AmbiguousGlobReexports {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_hidden_glob_reexport)]
+#[diag("private item shadows public glob re-export")]
 pub(crate) struct HiddenGlobReexports {
-    #[note(lint_note_glob_reexport)]
+    #[note(
+        "the name `{$name}` in the {$namespace} namespace is supposed to be publicly re-exported here"
+    )]
     pub glob_reexport: Span,
-    #[note(lint_note_private_item)]
+    #[note("but the private item here shadows it")]
     pub private_item: Span,
 
     pub name: String,
@@ -2876,47 +3320,70 @@ pub(crate) struct HiddenGlobReexports {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unnecessary_qualification)]
+#[diag("unnecessary qualification")]
 pub(crate) struct UnusedQualifications {
-    #[suggestion(style = "verbose", code = "", applicability = "machine-applicable")]
+    #[suggestion(
+        "remove the unnecessary path segments",
+        style = "verbose",
+        code = "",
+        applicability = "machine-applicable"
+    )]
     pub removal_span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_associated_const_elided_lifetime)]
+#[diag(
+    "{$elided ->
+        [true] `&` without an explicit lifetime name cannot be used here
+        *[false] `'_` cannot be used here
+    }"
+)]
 pub(crate) struct AssociatedConstElidedLifetime {
-    #[suggestion(style = "verbose", code = "{code}", applicability = "machine-applicable")]
+    #[suggestion(
+        "use the `'static` lifetime",
+        style = "verbose",
+        code = "{code}",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
 
     pub code: &'static str,
     pub elided: bool,
-    #[note]
+    #[note("cannot automatically infer `'static` because of other lifetimes in scope")]
     pub lifetimes_in_scope: MultiSpan,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_static_mut_refs_lint)]
+#[diag("creating a {$shared_label}reference to mutable static")]
 pub(crate) struct RefOfMutStatic<'a> {
-    #[label]
+    #[label("{$shared_label}reference to mutable static")]
     pub span: Span,
     #[subdiagnostic]
     pub sugg: Option,
     pub shared_label: &'a str,
-    #[note(lint_shared_note)]
+    #[note(
+        "shared references to mutable statics are dangerous; it's undefined behavior if the static is mutated or if a mutable reference is created for it while the shared reference lives"
+    )]
     pub shared_note: bool,
-    #[note(lint_mut_note)]
+    #[note(
+        "mutable references to mutable statics are dangerous; it's undefined behavior if any other pointer to the static is used or if any other reference is created for the static while the mutable reference lives"
+    )]
     pub mut_note: bool,
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum MutRefSugg {
-    #[multipart_suggestion(lint_suggestion, style = "verbose", applicability = "maybe-incorrect")]
+    #[multipart_suggestion(
+        "use `&raw const` instead to create a raw pointer",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
     Shared {
         #[suggestion_part(code = "&raw const ")]
         span: Span,
     },
     #[multipart_suggestion(
-        lint_suggestion_mut,
+        "use `&raw mut` instead to create a raw pointer",
         style = "verbose",
         applicability = "maybe-incorrect"
     )]
@@ -2927,25 +3394,33 @@ pub(crate) enum MutRefSugg {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unqualified_local_imports)]
+#[diag("`use` of a local item without leading `self::`, `super::`, or `crate::`")]
 pub(crate) struct UnqualifiedLocalImportsDiag {}
 
 #[derive(LintDiagnostic)]
-#[diag(lint_reserved_string)]
+#[diag("will be parsed as a guarded string in Rust 2024")]
 pub(crate) struct ReservedString {
-    #[suggestion(code = " ", applicability = "machine-applicable")]
+    #[suggestion(
+        "insert whitespace here to avoid this being parsed as a guarded string in Rust 2024",
+        code = " ",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_reserved_multihash)]
+#[diag("reserved token in Rust 2024")]
 pub(crate) struct ReservedMultihash {
-    #[suggestion(code = " ", applicability = "machine-applicable")]
+    #[suggestion(
+        "insert whitespace here to avoid this being parsed as a forbidden token in Rust 2024",
+        code = " ",
+        applicability = "machine-applicable"
+    )]
     pub suggestion: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_function_casts_as_integer)]
+#[diag("direct cast of function item into an integer")]
 pub(crate) struct FunctionCastsAsIntegerDiag<'tcx> {
     #[subdiagnostic]
     pub(crate) sugg: FunctionCastsAsIntegerSugg<'tcx>,
@@ -2953,7 +3428,7 @@ pub(crate) struct FunctionCastsAsIntegerDiag<'tcx> {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    lint_cast_as_fn,
+    "first cast to a pointer `as *const ()`",
     code = " as *const ()",
     applicability = "machine-applicable",
     style = "verbose"
@@ -2981,44 +3456,46 @@ impl<'a, G: EmissionGuarantee> LintDiagnostic<'a, G> for MismatchedLifetimeSynta
             }
 
             LifetimeSyntaxCategories { hidden: _, elided: _, named: 0 } => {
-                fluent::lint_mismatched_lifetime_syntaxes_hiding_while_elided
+                inline_fluent!("hiding a lifetime that's elided elsewhere is confusing")
             }
 
             LifetimeSyntaxCategories { hidden: _, elided: 0, named: _ } => {
-                fluent::lint_mismatched_lifetime_syntaxes_hiding_while_named
+                inline_fluent!("hiding a lifetime that's named elsewhere is confusing")
             }
 
             LifetimeSyntaxCategories { hidden: 0, elided: _, named: _ } => {
-                fluent::lint_mismatched_lifetime_syntaxes_eliding_while_named
+                inline_fluent!("eliding a lifetime that's named elsewhere is confusing")
             }
 
             LifetimeSyntaxCategories { hidden: _, elided: _, named: _ } => {
-                fluent::lint_mismatched_lifetime_syntaxes_hiding_and_eliding_while_named
+                inline_fluent!("hiding or eliding a lifetime that's named elsewhere is confusing")
             }
         };
         diag.primary_message(message);
 
         for s in self.inputs.hidden {
-            diag.span_label(s, fluent::lint_mismatched_lifetime_syntaxes_input_hidden);
+            diag.span_label(s, inline_fluent!("the lifetime is hidden here"));
         }
         for s in self.inputs.elided {
-            diag.span_label(s, fluent::lint_mismatched_lifetime_syntaxes_input_elided);
+            diag.span_label(s, inline_fluent!("the lifetime is elided here"));
         }
         for s in self.inputs.named {
-            diag.span_label(s, fluent::lint_mismatched_lifetime_syntaxes_input_named);
+            diag.span_label(s, inline_fluent!("the lifetime is named here"));
         }
 
         for s in self.outputs.hidden {
-            diag.span_label(s, fluent::lint_mismatched_lifetime_syntaxes_output_hidden);
+            diag.span_label(s, inline_fluent!("the same lifetime is hidden here"));
         }
         for s in self.outputs.elided {
-            diag.span_label(s, fluent::lint_mismatched_lifetime_syntaxes_output_elided);
+            diag.span_label(s, inline_fluent!("the same lifetime is elided here"));
         }
         for s in self.outputs.named {
-            diag.span_label(s, fluent::lint_mismatched_lifetime_syntaxes_output_named);
+            diag.span_label(s, inline_fluent!("the same lifetime is named here"));
         }
 
-        diag.help(fluent::lint_mismatched_lifetime_syntaxes_help);
+        diag.help(inline_fluent!(
+            "the same lifetime is referred to in inconsistent ways, making the signature confusing"
+        ));
 
         let mut suggestions = self.suggestions.into_iter();
         if let Some(s) = suggestions.next() {
@@ -3092,7 +3569,7 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
             Implicit { suggestions, optional_alternative } => {
                 let suggestions = suggestions.into_iter().map(|s| (s, String::new())).collect();
                 diag.multipart_suggestion_with_style(
-                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_implicit,
+                    inline_fluent!("remove the lifetime name from references"),
                     suggestions,
                     applicability(optional_alternative),
                     style(optional_alternative),
@@ -3105,9 +3582,11 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
                 optional_alternative,
             } => {
                 let message = if implicit_suggestions.is_empty() {
-                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_mixed_only_paths
+                    inline_fluent!("use `'_` for type paths")
                 } else {
-                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_mixed
+                    inline_fluent!(
+                        "remove the lifetime name from references and use `'_` for type paths"
+                    )
                 };
 
                 let implicit_suggestions =
@@ -3126,9 +3605,8 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
 
             Explicit { lifetime_name, suggestions, optional_alternative } => {
                 diag.arg("lifetime_name", lifetime_name);
-                let msg = diag.eagerly_translate(
-                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_explicit,
-                );
+                let msg =
+                    diag.eagerly_translate(inline_fluent!("consistently use `{$lifetime_name}`"));
                 diag.remove_arg("lifetime_name");
                 diag.multipart_suggestion_with_style(
                     msg,
@@ -3142,78 +3620,107 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_empty_attribute)]
-#[note]
+#[diag("unused attribute")]
+#[note(
+    "{$valid_without_list ->
+        [true] using `{$attr_path}` with an empty list is equivalent to not using a list at all
+        *[other] using `{$attr_path}` with an empty list has no effect
+    }"
+)]
 pub(crate) struct EmptyAttributeList {
-    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[suggestion(
+        "{$valid_without_list ->
+            [true] remove these parentheses
+            *[other] remove this attribute
+        }",
+        code = "",
+        applicability = "machine-applicable"
+    )]
     pub attr_span: Span,
     pub attr_path: String,
     pub valid_without_list: bool,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_invalid_target)]
-#[warning]
-#[help]
+#[diag("`#[{$name}]` attribute cannot be used on {$target}")]
+#[warning(
+    "this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!"
+)]
+#[help("`#[{$name}]` can {$only}be applied to {$applied}")]
 pub(crate) struct InvalidTargetLint {
     pub name: String,
     pub target: &'static str,
     pub applied: DiagArgValue,
     pub only: &'static str,
-    #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")]
+    #[suggestion(
+        "remove the attribute",
+        code = "",
+        applicability = "machine-applicable",
+        style = "tool-only"
+    )]
     pub attr_span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_invalid_style)]
+#[diag(
+    "{$is_used_as_inner ->
+        [false] crate-level attribute should be an inner attribute: add an exclamation mark: `#![{$name}]`
+        *[other] the `#![{$name}]` attribute can only be used at the crate root
+    }"
+)]
 pub(crate) struct InvalidAttrStyle {
     pub name: String,
     pub is_used_as_inner: bool,
-    #[note]
+    #[note("this attribute does not have an `!`, which means it is applied to this {$target}")]
     pub target_span: Option,
     pub target: &'static str,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_duplicate)]
+#[diag("unused attribute")]
 pub(crate) struct UnusedDuplicate {
-    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[suggestion("remove this attribute", code = "", applicability = "machine-applicable")]
     pub this: Span,
-    #[note]
+    #[note("attribute also specified here")]
     pub other: Span,
-    #[warning]
+    #[warning(
+        "this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!"
+    )]
     pub warning: bool,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_malformed_doc)]
-#[warning]
+#[diag("malformed `doc` attribute input")]
+#[warning(
+    "this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!"
+)]
 pub(crate) struct MalformedDoc;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_expected_no_args)]
-#[warning]
+#[diag("didn't expect any arguments here")]
+#[warning(
+    "this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!"
+)]
 pub(crate) struct ExpectedNoArgs;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_expected_name_value)]
-#[warning]
+#[diag("expected this to be of the form `... = \"...\"`")]
+#[warning(
+    "this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!"
+)]
 pub(crate) struct ExpectedNameValue;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unsafe_attr_outside_unsafe)]
+#[diag("unsafe attribute used without unsafe")]
 pub(crate) struct UnsafeAttrOutsideUnsafeLint {
-    #[label]
+    #[label("usage of unsafe attribute")]
     pub span: Span,
     #[subdiagnostic]
     pub suggestion: Option,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(
-    lint_unsafe_attr_outside_unsafe_suggestion,
-    applicability = "machine-applicable"
-)]
+#[multipart_suggestion("wrap the attribute in `unsafe(...)`", applicability = "machine-applicable")]
 pub(crate) struct UnsafeAttrOutsideUnsafeSuggestion {
     #[suggestion_part(code = "unsafe(")]
     pub left: Span,
@@ -3222,119 +3729,138 @@ pub(crate) struct UnsafeAttrOutsideUnsafeSuggestion {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unused_visibilities)]
-#[note]
+#[diag("visibility qualifiers have no effect on `const _` declarations")]
+#[note("`const _` does not declare a name, so there is nothing for the qualifier to apply to")]
 pub(crate) struct UnusedVisibility {
-    #[suggestion(style = "short", code = "", applicability = "machine-applicable")]
+    #[suggestion(
+        "remove the qualifier",
+        style = "short",
+        code = "",
+        applicability = "machine-applicable"
+    )]
     pub span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_alias_duplicated)]
+#[diag("doc alias is duplicated")]
 pub(crate) struct DocAliasDuplicated {
-    #[label]
+    #[label("first defined here")]
     pub first_defn: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_auto_cfg_expects_hide_or_show)]
+#[diag("only `hide` or `show` are allowed in `#[doc(auto_cfg(...))]`")]
 pub(crate) struct DocAutoCfgExpectsHideOrShow;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_auto_cfg_hide_show_unexpected_item)]
+#[diag("`#![doc(auto_cfg({$attr_name}(...)))]` only accepts identifiers or key/value items")]
 pub(crate) struct DocAutoCfgHideShowUnexpectedItem {
     pub attr_name: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_auto_cfg_hide_show_expects_list)]
+#[diag("`#![doc(auto_cfg({$attr_name}(...)))]` expects a list of items")]
 pub(crate) struct DocAutoCfgHideShowExpectsList {
     pub attr_name: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_invalid)]
+#[diag("invalid `doc` attribute")]
 pub(crate) struct DocInvalid;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_unknown_include)]
+#[diag("unknown `doc` attribute `include`")]
 pub(crate) struct DocUnknownInclude {
     pub inner: &'static str,
     pub value: Symbol,
-    #[suggestion(code = "#{inner}[doc = include_str!(\"{value}\")]")]
+    #[suggestion(
+        "use `doc = include_str!` instead",
+        code = "#{inner}[doc = include_str!(\"{value}\")]"
+    )]
     pub sugg: (Span, Applicability),
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_unknown_spotlight)]
-#[note]
-#[note(lint_no_op_note)]
+#[diag("unknown `doc` attribute `spotlight`")]
+#[note("`doc(spotlight)` was renamed to `doc(notable_trait)`")]
+#[note("`doc(spotlight)` is now a no-op")]
 pub(crate) struct DocUnknownSpotlight {
-    #[suggestion(style = "short", applicability = "machine-applicable", code = "notable_trait")]
+    #[suggestion(
+        "use `notable_trait` instead",
+        style = "short",
+        applicability = "machine-applicable",
+        code = "notable_trait"
+    )]
     pub sugg_span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_unknown_passes)]
-#[note]
-#[note(lint_no_op_note)]
+#[diag("unknown `doc` attribute `{$name}`")]
+#[note(
+    "`doc` attribute `{$name}` no longer functions; see issue #44136 "
+)]
+#[note("`doc({$name})` is now a no-op")]
 pub(crate) struct DocUnknownPasses {
     pub name: Symbol,
-    #[label]
+    #[label("no longer functions")]
     pub note_span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_unknown_plugins)]
-#[note]
-#[note(lint_no_op_note)]
+#[diag("unknown `doc` attribute `plugins`")]
+#[note(
+    "`doc` attribute `plugins` no longer functions; see issue #44136  and CVE-2018-1000622 "
+)]
+#[note("`doc(plugins)` is now a no-op")]
 pub(crate) struct DocUnknownPlugins {
-    #[label]
+    #[label("no longer functions")]
     pub label_span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_unknown_any)]
+#[diag("unknown `doc` attribute `{$name}`")]
 pub(crate) struct DocUnknownAny {
     pub name: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_auto_cfg_wrong_literal)]
+#[diag("expected boolean for `#[doc(auto_cfg = ...)]`")]
 pub(crate) struct DocAutoCfgWrongLiteral;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_test_takes_list)]
+#[diag("`#[doc(test(...)]` takes a list of attributes")]
 pub(crate) struct DocTestTakesList;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_test_unknown)]
+#[diag("unknown `doc(test)` attribute `{$name}`")]
 pub(crate) struct DocTestUnknown {
     pub name: Symbol,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_doc_test_literal)]
+#[diag("`#![doc(test(...)]` does not take a literal")]
 pub(crate) struct DocTestLiteral;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_attr_crate_level)]
-#[note]
+#[diag("this attribute can only be applied at the crate level")]
+#[note(
+    "read  for more information"
+)]
 pub(crate) struct AttrCrateLevelOnly;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_incorrect_do_not_recommend_args)]
+#[diag("`#[diagnostic::do_not_recommend]` does not expect any arguments")]
 pub(crate) struct DoNotRecommendDoesNotExpectArgs;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_invalid_crate_type_value)]
+#[diag("invalid `crate_type` value")]
 pub(crate) struct UnknownCrateTypes {
     #[subdiagnostic]
     pub sugg: Option,
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(lint_suggestion, code = r#""{snippet}""#, applicability = "maybe-incorrect")]
+#[suggestion("did you mean", code = r#""{snippet}""#, applicability = "maybe-incorrect")]
 pub(crate) struct UnknownCrateTypesSuggestion {
     #[primary_span]
     pub span: Span,
@@ -3342,18 +3868,18 @@ pub(crate) struct UnknownCrateTypesSuggestion {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unreachable_cfg_select_predicate)]
+#[diag("unreachable configuration predicate")]
 pub(crate) struct UnreachableCfgSelectPredicate {
-    #[label]
+    #[label("this configuration predicate is never reached")]
     pub span: Span,
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_unreachable_cfg_select_predicate_wildcard)]
+#[diag("unreachable configuration predicate")]
 pub(crate) struct UnreachableCfgSelectPredicateWildcard {
-    #[label(lint_label2)]
+    #[label("this configuration predicate is never reached")]
     pub span: Span,
 
-    #[label]
+    #[label("always matches")]
     pub wildcard_span: Span,
 }
diff --git a/compiler/rustc_lint/src/non_fmt_panic.rs b/compiler/rustc_lint/src/non_fmt_panic.rs
index a07c3b372d34..daf62277693a 100644
--- a/compiler/rustc_lint/src/non_fmt_panic.rs
+++ b/compiler/rustc_lint/src/non_fmt_panic.rs
@@ -1,5 +1,5 @@
 use rustc_ast as ast;
-use rustc_errors::Applicability;
+use rustc_errors::{Applicability, inline_fluent};
 use rustc_hir::{self as hir, LangItem};
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::{bug, ty};
@@ -10,7 +10,7 @@ use rustc_span::{InnerSpan, Span, Symbol, hygiene, sym};
 use rustc_trait_selection::infer::InferCtxtExt;
 
 use crate::lints::{NonFmtPanicBraces, NonFmtPanicUnused};
-use crate::{LateContext, LateLintPass, LintContext, fluent_generated as fluent};
+use crate::{LateContext, LateLintPass, LintContext};
 
 declare_lint! {
     /// The `non_fmt_panics` lint detects `panic!(..)` invocations where the first
@@ -121,20 +121,20 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
     }
 
     cx.span_lint(NON_FMT_PANICS, arg_span, |lint| {
-        lint.primary_message(fluent::lint_non_fmt_panic);
+        lint.primary_message(inline_fluent!("panic message is not a string literal"));
         lint.arg("name", symbol);
-        lint.note(fluent::lint_note);
-        lint.note(fluent::lint_more_info_note);
+        lint.note(inline_fluent!("this usage of `{$name}!()` is deprecated; it will be a hard error in Rust 2021"));
+        lint.note(inline_fluent!("for more information, see "));
         if !is_arg_inside_call(arg_span, span) {
             // No clue where this argument is coming from.
             return;
         }
         if arg_macro.is_some_and(|id| cx.tcx.is_diagnostic_item(sym::format_macro, id)) {
             // A case of `panic!(format!(..))`.
-            lint.note(fluent::lint_supports_fmt_note);
+            lint.note(inline_fluent!("the `{$name}!()` macro supports formatting, so there's no need for the `format!()` macro here"));
             if let Some((open, close, _)) = find_delimiters(cx, arg_span) {
                 lint.multipart_suggestion(
-                    fluent::lint_supports_fmt_suggestion,
+                    inline_fluent!("remove the `format!(..)` macro call"),
                     vec![
                         (arg_span.until(open.shrink_to_hi()), "".into()),
                         (close.until(arg_span.shrink_to_hi()), "".into()),
@@ -178,7 +178,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
             if suggest_display {
                 lint.span_suggestion_verbose(
                     arg_span.shrink_to_lo(),
-                    fluent::lint_display_suggestion,
+                    inline_fluent!(r#"add a "{"{"}{"}"}" format string to `Display` the message"#),
                     "\"{}\", ",
                     fmt_applicability,
                 );
@@ -186,7 +186,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
                 lint.arg("ty", ty);
                 lint.span_suggestion_verbose(
                     arg_span.shrink_to_lo(),
-                    fluent::lint_debug_suggestion,
+                    inline_fluent!(r#"add a "{"{"}:?{"}"}" format string to use the `Debug` implementation of `{$ty}`"#),
                     "\"{:?}\", ",
                     fmt_applicability,
                 );
@@ -196,7 +196,10 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
                 if let Some((open, close, del)) = find_delimiters(cx, span) {
                     lint.arg("already_suggested", suggest_display || suggest_debug);
                     lint.multipart_suggestion(
-                        fluent::lint_panic_suggestion,
+                        inline_fluent!("{$already_suggested ->
+                            [true] or use
+                            *[false] use
+                        } std::panic::panic_any instead"),
                         if del == '(' {
                             vec![(span.until(open), "std::panic::panic_any".into())]
                         } else {
diff --git a/compiler/rustc_lint/src/non_local_def.rs b/compiler/rustc_lint/src/non_local_def.rs
index bcf4d9ec3b29..fc7e5a821b02 100644
--- a/compiler/rustc_lint/src/non_local_def.rs
+++ b/compiler/rustc_lint/src/non_local_def.rs
@@ -1,4 +1,4 @@
-use rustc_errors::MultiSpan;
+use rustc_errors::{MultiSpan, inline_fluent};
 use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit::{self, Visitor, VisitorExt};
@@ -9,7 +9,7 @@ use rustc_span::def_id::{DefId, LOCAL_CRATE};
 use rustc_span::{ExpnKind, Span, kw};
 
 use crate::lints::{NonLocalDefinitionsCargoUpdateNote, NonLocalDefinitionsDiag};
-use crate::{LateContext, LateLintPass, LintContext, fluent_generated as fluent};
+use crate::{LateContext, LateLintPass, LintContext};
 
 declare_lint! {
     /// The `non_local_definitions` lint checks for `impl` blocks and `#[macro_export]`
@@ -210,7 +210,12 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
                 if !doctest {
                     ms.push_span_label(
                         cx.tcx.def_span(parent),
-                        fluent::lint_non_local_definitions_impl_move_help,
+                        inline_fluent!(
+                            "move the `impl` block outside of this {$body_kind_descr} {$depth ->
+                                [one] `{$body_name}`
+                                *[other] `{$body_name}` and up {$depth} bodies
+                            }"
+                        ),
                     );
                 }
 
diff --git a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
index 3f2ca92a021a..b5b57eaa3389 100644
--- a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
+++ b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
@@ -202,11 +202,11 @@ impl<'tcx> LateLintPass<'tcx> for OpaqueHiddenInferredBound {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_opaque_hidden_inferred_bound)]
+#[diag("opaque type `{$ty}` does not satisfy its associated type bounds")]
 struct OpaqueHiddenInferredBoundLint<'tcx> {
     ty: Ty<'tcx>,
     proj_ty: Ty<'tcx>,
-    #[label(lint_specifically)]
+    #[label("this associated type bound is unsatisfied for `{$proj_ty}`")]
     assoc_pred_span: Span,
     #[subdiagnostic]
     add_bound: Option>,
@@ -214,7 +214,7 @@ struct OpaqueHiddenInferredBoundLint<'tcx> {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    lint_opaque_hidden_inferred_bound_sugg,
+    "add this bound",
     style = "verbose",
     applicability = "machine-applicable",
     code = " + {trait_ref}"
diff --git a/compiler/rustc_lint/src/transmute.rs b/compiler/rustc_lint/src/transmute.rs
index 6bc4617eb2dc..e4716c869c5f 100644
--- a/compiler/rustc_lint/src/transmute.rs
+++ b/compiler/rustc_lint/src/transmute.rs
@@ -369,8 +369,10 @@ fn check_unnecessary_transmute<'tcx>(
 }
 
 #[derive(LintDiagnostic)]
-#[diag(lint_undefined_transmute)]
-#[note]
-#[note(lint_note2)]
-#[help]
+#[diag("pointers cannot be transmuted to integers during const eval")]
+#[note("at compile-time, pointers do not have an integer value")]
+#[note(
+    "avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior"
+)]
+#[help("for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html")]
 pub(crate) struct UndefinedTransmuteLint;
diff --git a/compiler/rustc_lint/src/types/improper_ctypes.rs b/compiler/rustc_lint/src/types/improper_ctypes.rs
index 38094c67c34a..fb9b55efa220 100644
--- a/compiler/rustc_lint/src/types/improper_ctypes.rs
+++ b/compiler/rustc_lint/src/types/improper_ctypes.rs
@@ -4,7 +4,7 @@ use std::ops::ControlFlow;
 use bitflags::bitflags;
 use rustc_abi::VariantIdx;
 use rustc_data_structures::fx::FxHashSet;
-use rustc_errors::DiagMessage;
+use rustc_errors::{DiagMessage, inline_fluent};
 use rustc_hir::def::CtorKind;
 use rustc_hir::intravisit::VisitorExt;
 use rustc_hir::{self as hir, AmbigArg};
@@ -21,7 +21,7 @@ use tracing::debug;
 
 use super::repr_nullable_ptr;
 use crate::lints::{ImproperCTypes, UsesPowerAlignment};
-use crate::{LateContext, LateLintPass, LintContext, fluent_generated as fluent};
+use crate::{LateContext, LateLintPass, LintContext};
 
 declare_lint! {
     /// The `improper_ctypes` lint detects incorrect use of types in foreign
@@ -158,12 +158,12 @@ pub(crate) fn check_non_exhaustive_variant(
         // with an enum like `#[repr(u8)] enum Enum { A(DataA), B(DataB), }`
         // but exempt enums with unit ctors like C's (e.g. from rust-bindgen)
         if variant_has_complex_ctor(variant) {
-            return ControlFlow::Break(fluent::lint_improper_ctypes_non_exhaustive);
+            return ControlFlow::Break(inline_fluent!("this enum is non-exhaustive"));
         }
     }
 
     if variant.field_list_has_applicable_non_exhaustive() {
-        return ControlFlow::Break(fluent::lint_improper_ctypes_non_exhaustive_variant);
+        return ControlFlow::Break(inline_fluent!("this enum has non-exhaustive variants"));
     }
 
     ControlFlow::Continue(())
@@ -424,7 +424,11 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         if all_phantom {
             FfiPhantom(ty)
         } else if transparent_with_all_zst_fields {
-            FfiUnsafe { ty, reason: fluent::lint_improper_ctypes_struct_zst, help: None }
+            FfiUnsafe {
+                ty,
+                reason: inline_fluent!("this struct contains only zero-sized fields"),
+                help: None,
+            }
         } else {
             FfiSafe
         }
@@ -460,7 +464,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                     } else {
                         return FfiUnsafe {
                             ty,
-                            reason: fluent::lint_improper_ctypes_box,
+                            reason: inline_fluent!("box cannot be represented as a single pointer"),
                             help: None,
                         };
                     }
@@ -476,8 +480,12 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                         {
                             return FfiUnsafe {
                                 ty,
-                                reason: fluent::lint_improper_ctypes_cstr_reason,
-                                help: Some(fluent::lint_improper_ctypes_cstr_help),
+                                reason: inline_fluent!(
+                                    "`CStr`/`CString` do not have a guaranteed layout"
+                                ),
+                                help: Some(inline_fluent!(
+                                    "consider passing a `*const std::ffi::c_char` instead, and use `CStr::as_ptr()`"
+                                )),
                             };
                         }
 
@@ -485,14 +493,18 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                             return FfiUnsafe {
                                 ty,
                                 reason: if def.is_struct() {
-                                    fluent::lint_improper_ctypes_struct_layout_reason
+                                    inline_fluent!("this struct has unspecified layout")
                                 } else {
-                                    fluent::lint_improper_ctypes_union_layout_reason
+                                    inline_fluent!("this union has unspecified layout")
                                 },
                                 help: if def.is_struct() {
-                                    Some(fluent::lint_improper_ctypes_struct_layout_help)
+                                    Some(inline_fluent!(
+                                        "consider adding a `#[repr(C)]` or `#[repr(transparent)]` attribute to this struct"
+                                    ))
                                 } else {
-                                    Some(fluent::lint_improper_ctypes_union_layout_help)
+                                    Some(inline_fluent!(
+                                        "consider adding a `#[repr(C)]` or `#[repr(transparent)]` attribute to this union"
+                                    ))
                                 },
                             };
                         }
@@ -501,9 +513,9 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                             return FfiUnsafe {
                                 ty,
                                 reason: if def.is_struct() {
-                                    fluent::lint_improper_ctypes_struct_non_exhaustive
+                                    inline_fluent!("this struct is non-exhaustive")
                                 } else {
-                                    fluent::lint_improper_ctypes_union_non_exhaustive
+                                    inline_fluent!("this union is non-exhaustive")
                                 },
                                 help: None,
                             };
@@ -513,14 +525,14 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                             return FfiUnsafe {
                                 ty,
                                 reason: if def.is_struct() {
-                                    fluent::lint_improper_ctypes_struct_fieldless_reason
+                                    inline_fluent!("this struct has no fields")
                                 } else {
-                                    fluent::lint_improper_ctypes_union_fieldless_reason
+                                    inline_fluent!("this union has no fields")
                                 },
                                 help: if def.is_struct() {
-                                    Some(fluent::lint_improper_ctypes_struct_fieldless_help)
+                                    Some(inline_fluent!("consider adding a member to this struct"))
                                 } else {
-                                    Some(fluent::lint_improper_ctypes_union_fieldless_help)
+                                    Some(inline_fluent!("consider adding a member to this union"))
                                 },
                             };
                         }
@@ -545,8 +557,10 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
 
                             return FfiUnsafe {
                                 ty,
-                                reason: fluent::lint_improper_ctypes_enum_repr_reason,
-                                help: Some(fluent::lint_improper_ctypes_enum_repr_help),
+                                reason: inline_fluent!("enum has no representation hint"),
+                                help: Some(inline_fluent!(
+                                    "consider adding a `#[repr(C)]`, `#[repr(transparent)]`, or integer `#[repr(...)]` attribute to this enum"
+                                )),
                             };
                         }
 
@@ -572,8 +586,8 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
 
             ty::Char => FfiUnsafe {
                 ty,
-                reason: fluent::lint_improper_ctypes_char_reason,
-                help: Some(fluent::lint_improper_ctypes_char_help),
+                reason: inline_fluent!("the `char` type has no C equivalent"),
+                help: Some(inline_fluent!("consider using `u32` or `libc::wchar_t` instead")),
             },
 
             // It's just extra invariants on the type that you need to uphold,
@@ -585,24 +599,26 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
 
             ty::Slice(_) => FfiUnsafe {
                 ty,
-                reason: fluent::lint_improper_ctypes_slice_reason,
-                help: Some(fluent::lint_improper_ctypes_slice_help),
+                reason: inline_fluent!("slices have no C equivalent"),
+                help: Some(inline_fluent!("consider using a raw pointer instead")),
             },
 
-            ty::Dynamic(..) => {
-                FfiUnsafe { ty, reason: fluent::lint_improper_ctypes_dyn, help: None }
-            }
+            ty::Dynamic(..) => FfiUnsafe {
+                ty,
+                reason: inline_fluent!("trait objects have no C equivalent"),
+                help: None,
+            },
 
             ty::Str => FfiUnsafe {
                 ty,
-                reason: fluent::lint_improper_ctypes_str_reason,
-                help: Some(fluent::lint_improper_ctypes_str_help),
+                reason: inline_fluent!("string slices have no C equivalent"),
+                help: Some(inline_fluent!("consider using `*const u8` and a length instead")),
             },
 
             ty::Tuple(..) => FfiUnsafe {
                 ty,
-                reason: fluent::lint_improper_ctypes_tuple_reason,
-                help: Some(fluent::lint_improper_ctypes_tuple_help),
+                reason: inline_fluent!("tuples have unspecified layout"),
+                help: Some(inline_fluent!("consider using a struct instead")),
             },
 
             ty::RawPtr(ty, _) | ty::Ref(_, ty, _)
@@ -632,8 +648,12 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                 if sig.abi().is_rustic_abi() {
                     return FfiUnsafe {
                         ty,
-                        reason: fluent::lint_improper_ctypes_fnptr_reason,
-                        help: Some(fluent::lint_improper_ctypes_fnptr_help),
+                        reason: inline_fluent!(
+                            "this function pointer has Rust-specific calling convention"
+                        ),
+                        help: Some(inline_fluent!(
+                            "consider using an `extern fn(...) -> ...` function pointer instead"
+                        )),
                     };
                 }
 
@@ -657,9 +677,11 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
 
             // While opaque types are checked for earlier, if a projection in a struct field
             // normalizes to an opaque type, then it will reach this branch.
-            ty::Alias(ty::Opaque, ..) => {
-                FfiUnsafe { ty, reason: fluent::lint_improper_ctypes_opaque, help: None }
-            }
+            ty::Alias(ty::Opaque, ..) => FfiUnsafe {
+                ty,
+                reason: inline_fluent!("opaque types have no C equivalent"),
+                help: None,
+            },
 
             // `extern "C" fn` functions can have type parameters, which may or may not be FFI-safe,
             //  so they are currently ignored for the purposes of this lint.
@@ -669,9 +691,13 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                 FfiSafe
             }
 
-            ty::UnsafeBinder(_) => {
-                FfiUnsafe { ty, reason: fluent::lint_improper_ctypes_unsafe_binder, help: None }
-            }
+            ty::UnsafeBinder(_) => FfiUnsafe {
+                ty,
+                reason: inline_fluent!(
+                    "unsafe binders are incompatible with foreign function interfaces"
+                ),
+                help: None,
+            },
 
             ty::Param(..)
             | ty::Alias(ty::Projection | ty::Inherent | ty::Free, ..)
@@ -715,7 +741,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         {
             Some(FfiResult::FfiUnsafe {
                 ty,
-                reason: fluent::lint_improper_ctypes_opaque,
+                reason: inline_fluent!("opaque types have no C equivalent"),
                 help: None,
             })
         } else {
@@ -728,8 +754,8 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         if let ty::Array(..) = ty.kind() {
             Some(FfiResult::FfiUnsafe {
                 ty,
-                reason: fluent::lint_improper_ctypes_array_reason,
-                help: Some(fluent::lint_improper_ctypes_array_help),
+                reason: inline_fluent!("passing raw arrays by value is not FFI-safe"),
+                help: Some(inline_fluent!("consider passing a pointer to the array")),
             })
         } else {
             None
@@ -908,7 +934,7 @@ impl<'tcx> ImproperCTypesLint {
                     cx,
                     ty,
                     sp,
-                    fluent::lint_improper_ctypes_only_phantomdata,
+                    inline_fluent!("composed only of `PhantomData`"),
                     None,
                     fn_mode,
                 );

From f35d734d3fc0105c4f130be961c86deac299dd6c Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 19:32:52 +0100
Subject: [PATCH 1423/1843] Remove `rustc_fluent_macro`

---
 Cargo.lock                                    |  13 -
 compiler/rustc_fluent_macro/Cargo.toml        |  18 -
 compiler/rustc_fluent_macro/src/fluent.rs     | 315 ------------------
 compiler/rustc_fluent_macro/src/lib.rs        |  67 ----
 .../builder/cli_paths/snapshots/x_bench.snap  |   1 -
 .../cli_paths/snapshots/x_build_compiler.snap |   1 -
 .../builder/cli_paths/snapshots/x_check.snap  |   1 -
 .../cli_paths/snapshots/x_check_compiler.snap |   1 -
 ...eck_compiletest_include_default_paths.snap |   1 -
 .../builder/cli_paths/snapshots/x_clippy.snap |   1 -
 .../builder/cli_paths/snapshots/x_fix.snap    |   1 -
 .../builder/cli_paths/snapshots/x_test.snap   |   1 -
 .../snapshots/x_test_skip_coverage.snap       |   1 -
 .../snapshots/x_test_skip_tests.snap          |   2 -
 .../snapshots/x_test_skip_tests_etc.snap      |   2 -
 src/bootstrap/src/core/builder/tests.rs       |  10 +-
 src/bootstrap/src/utils/proc_macro_deps.rs    |   3 -
 17 files changed, 5 insertions(+), 434 deletions(-)
 delete mode 100644 compiler/rustc_fluent_macro/Cargo.toml
 delete mode 100644 compiler/rustc_fluent_macro/src/fluent.rs
 delete mode 100644 compiler/rustc_fluent_macro/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 03cf3bcba894..545c776b4805 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3888,19 +3888,6 @@ dependencies = [
  "serde_json",
 ]
 
-[[package]]
-name = "rustc_fluent_macro"
-version = "0.0.0"
-dependencies = [
- "annotate-snippets 0.11.5",
- "fluent-bundle",
- "fluent-syntax",
- "proc-macro2",
- "quote",
- "syn 2.0.110",
- "unic-langid",
-]
-
 [[package]]
 name = "rustc_fs_util"
 version = "0.0.0"
diff --git a/compiler/rustc_fluent_macro/Cargo.toml b/compiler/rustc_fluent_macro/Cargo.toml
deleted file mode 100644
index d7ef4280aef0..000000000000
--- a/compiler/rustc_fluent_macro/Cargo.toml
+++ /dev/null
@@ -1,18 +0,0 @@
-[package]
-name = "rustc_fluent_macro"
-version = "0.0.0"
-edition = "2024"
-
-[lib]
-proc-macro = true
-
-[dependencies]
-# tidy-alphabetical-start
-annotate-snippets = "0.11"
-fluent-bundle = "0.16"
-fluent-syntax = "0.12"
-proc-macro2 = "1"
-quote = "1"
-syn = { version = "2", features = ["full"] }
-unic-langid = { version = "0.9.0", features = ["macros"] }
-# tidy-alphabetical-end
diff --git a/compiler/rustc_fluent_macro/src/fluent.rs b/compiler/rustc_fluent_macro/src/fluent.rs
deleted file mode 100644
index 41357bb1ac76..000000000000
--- a/compiler/rustc_fluent_macro/src/fluent.rs
+++ /dev/null
@@ -1,315 +0,0 @@
-use std::collections::{HashMap, HashSet};
-use std::fs::read_to_string;
-use std::path::{Path, PathBuf};
-
-use annotate_snippets::{Renderer, Snippet};
-use fluent_bundle::{FluentBundle, FluentError, FluentResource};
-use fluent_syntax::ast::{
-    Attribute, Entry, Expression, Identifier, InlineExpression, Message, Pattern, PatternElement,
-};
-use fluent_syntax::parser::ParserError;
-use proc_macro::tracked::path;
-use proc_macro::{Diagnostic, Level, Span};
-use proc_macro2::TokenStream;
-use quote::quote;
-use syn::{Ident, LitStr, parse_macro_input};
-use unic_langid::langid;
-
-/// Helper function for returning an absolute path for macro-invocation relative file paths.
-///
-/// If the input is already absolute, then the input is returned. If the input is not absolute,
-/// then it is appended to the directory containing the source file with this macro invocation.
-fn invocation_relative_path_to_absolute(span: Span, path: &str) -> PathBuf {
-    let path = Path::new(path);
-    if path.is_absolute() {
-        path.to_path_buf()
-    } else {
-        // `/a/b/c/foo/bar.rs` contains the current macro invocation
-        let mut source_file_path = span.local_file().unwrap();
-        // `/a/b/c/foo/`
-        source_file_path.pop();
-        // `/a/b/c/foo/../locales/en-US/example.ftl`
-        source_file_path.push(path);
-        source_file_path
-    }
-}
-
-/// Final tokens.
-fn finish(body: TokenStream, resource: TokenStream) -> proc_macro::TokenStream {
-    quote! {
-        /// Raw content of Fluent resource for this crate, generated by `fluent_messages` macro,
-        /// imported by `rustc_driver` to include all crates' resources in one bundle.
-        pub static DEFAULT_LOCALE_RESOURCE: &'static str = #resource;
-
-        #[allow(non_upper_case_globals)]
-        #[doc(hidden)]
-        /// Auto-generated constants for type-checked references to Fluent messages.
-        pub(crate) mod fluent_generated {
-            #body
-
-            /// Constants expected to exist by the diagnostic derive macros to use as default Fluent
-            /// identifiers for different subdiagnostic kinds.
-            pub mod _subdiag {
-                /// Default for `#[help]`
-                pub const help: rustc_errors::SubdiagMessage =
-                    rustc_errors::SubdiagMessage::FluentAttr(std::borrow::Cow::Borrowed("help"));
-                /// Default for `#[note]`
-                pub const note: rustc_errors::SubdiagMessage =
-                    rustc_errors::SubdiagMessage::FluentAttr(std::borrow::Cow::Borrowed("note"));
-                /// Default for `#[warn]`
-                pub const warn: rustc_errors::SubdiagMessage =
-                    rustc_errors::SubdiagMessage::FluentAttr(std::borrow::Cow::Borrowed("warn"));
-                /// Default for `#[label]`
-                pub const label: rustc_errors::SubdiagMessage =
-                    rustc_errors::SubdiagMessage::FluentAttr(std::borrow::Cow::Borrowed("label"));
-                /// Default for `#[suggestion]`
-                pub const suggestion: rustc_errors::SubdiagMessage =
-                    rustc_errors::SubdiagMessage::FluentAttr(std::borrow::Cow::Borrowed("suggestion"));
-            }
-        }
-    }
-    .into()
-}
-
-/// Tokens to be returned when the macro cannot proceed.
-fn failed(crate_name: &Ident) -> proc_macro::TokenStream {
-    finish(quote! { pub mod #crate_name {} }, quote! { "" })
-}
-
-/// See [rustc_fluent_macro::fluent_messages].
-pub(crate) fn fluent_messages(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
-    let crate_name = std::env::var("CARGO_CRATE_NAME")
-        // If `CARGO_CRATE_NAME` is missing, then we're probably running in a test, so use
-        // `no_crate`.
-        .unwrap_or_else(|_| "no_crate".to_string())
-        .replace("rustc_", "");
-
-    // Cannot iterate over individual messages in a bundle, so do that using the
-    // `FluentResource` instead. Construct a bundle anyway to find out if there are conflicting
-    // messages in the resources.
-    let mut bundle = FluentBundle::new(vec![langid!("en-US")]);
-
-    // Set of Fluent attribute names already output, to avoid duplicate type errors - any given
-    // constant created for a given attribute is the same.
-    let mut previous_attrs = HashSet::new();
-
-    let resource_str = parse_macro_input!(input as LitStr);
-    let resource_span = resource_str.span().unwrap();
-    let relative_ftl_path = resource_str.value();
-    let absolute_ftl_path = invocation_relative_path_to_absolute(resource_span, &relative_ftl_path);
-
-    let crate_name = Ident::new(&crate_name, resource_str.span());
-
-    path(absolute_ftl_path.to_str().unwrap());
-    let resource_contents = match read_to_string(absolute_ftl_path) {
-        Ok(resource_contents) => resource_contents,
-        Err(e) => {
-            Diagnostic::spanned(
-                resource_span,
-                Level::Error,
-                format!("could not open Fluent resource: {e}"),
-            )
-            .emit();
-            return failed(&crate_name);
-        }
-    };
-    let mut bad = false;
-    for esc in ["\\n", "\\\"", "\\'"] {
-        for _ in resource_contents.matches(esc) {
-            bad = true;
-            Diagnostic::spanned(resource_span, Level::Error, format!("invalid escape `{esc}` in Fluent resource"))
-                .note("Fluent does not interpret these escape sequences ()")
-                .emit();
-        }
-    }
-    if bad {
-        return failed(&crate_name);
-    }
-
-    let resource = match FluentResource::try_new(resource_contents) {
-        Ok(resource) => resource,
-        Err((this, errs)) => {
-            Diagnostic::spanned(resource_span, Level::Error, "could not parse Fluent resource")
-                .help("see additional errors emitted")
-                .emit();
-            for ParserError { pos, slice: _, kind } in errs {
-                let mut err = kind.to_string();
-                // Entirely unnecessary string modification so that the error message starts
-                // with a lowercase as rustc errors do.
-                err.replace_range(0..1, &err.chars().next().unwrap().to_lowercase().to_string());
-
-                let message = annotate_snippets::Level::Error.title(&err).snippet(
-                    Snippet::source(this.source())
-                        .origin(&relative_ftl_path)
-                        .fold(true)
-                        .annotation(annotate_snippets::Level::Error.span(pos.start..pos.end - 1)),
-                );
-                let renderer = Renderer::plain();
-                eprintln!("{}\n", renderer.render(message));
-            }
-
-            return failed(&crate_name);
-        }
-    };
-
-    let mut constants = TokenStream::new();
-    let mut previous_defns = HashMap::new();
-    let mut message_refs = Vec::new();
-    for entry in resource.entries() {
-        if let Entry::Message(msg) = entry {
-            let Message { id: Identifier { name }, attributes, value, .. } = msg;
-            let _ = previous_defns.entry(name.to_string()).or_insert(resource_span);
-            if name.contains('-') {
-                Diagnostic::spanned(
-                    resource_span,
-                    Level::Error,
-                    format!("name `{name}` contains a '-' character"),
-                )
-                .help("replace any '-'s with '_'s")
-                .emit();
-            }
-
-            if let Some(Pattern { elements }) = value {
-                for elt in elements {
-                    if let PatternElement::Placeable {
-                        expression:
-                            Expression::Inline(InlineExpression::MessageReference { id, .. }),
-                    } = elt
-                    {
-                        message_refs.push((id.name, *name));
-                    }
-                }
-            }
-
-            // `typeck_foo_bar` => `foo_bar` (in `typeck.ftl`)
-            // `const_eval_baz` => `baz` (in `const_eval.ftl`)
-            // `const-eval-hyphen-having` => `hyphen_having` (in `const_eval.ftl`)
-            // The last case we error about above, but we want to fall back gracefully
-            // so that only the error is being emitted and not also one about the macro
-            // failing.
-            let crate_prefix = format!("{crate_name}_");
-
-            let snake_name = name.replace('-', "_");
-            if !snake_name.starts_with(&crate_prefix) {
-                Diagnostic::spanned(
-                    resource_span,
-                    Level::Error,
-                    format!("name `{name}` does not start with the crate name"),
-                )
-                .help(format!(
-                    "prepend `{crate_prefix}` to the slug name: `{crate_prefix}{snake_name}`"
-                ))
-                .emit();
-            };
-            let snake_name = Ident::new(&snake_name, resource_str.span());
-
-            if !previous_attrs.insert(snake_name.clone()) {
-                continue;
-            }
-
-            let docstr =
-                format!("Constant referring to Fluent message `{name}` from `{crate_name}`");
-            constants.extend(quote! {
-                #[doc = #docstr]
-                pub const #snake_name: rustc_errors::DiagMessage =
-                    rustc_errors::DiagMessage::FluentIdentifier(
-                        std::borrow::Cow::Borrowed(#name),
-                        None
-                    );
-            });
-
-            for Attribute { id: Identifier { name: attr_name }, .. } in attributes {
-                let snake_name = Ident::new(
-                    &format!("{crate_prefix}{}", attr_name.replace('-', "_")),
-                    resource_str.span(),
-                );
-                if !previous_attrs.insert(snake_name.clone()) {
-                    continue;
-                }
-
-                if attr_name.contains('-') {
-                    Diagnostic::spanned(
-                        resource_span,
-                        Level::Error,
-                        format!("attribute `{attr_name}` contains a '-' character"),
-                    )
-                    .help("replace any '-'s with '_'s")
-                    .emit();
-                }
-
-                let msg = format!(
-                    "Constant referring to Fluent message `{name}.{attr_name}` from `{crate_name}`"
-                );
-                constants.extend(quote! {
-                    #[doc = #msg]
-                    pub const #snake_name: rustc_errors::SubdiagMessage =
-                        rustc_errors::SubdiagMessage::FluentAttr(std::borrow::Cow::Borrowed(#attr_name));
-                });
-            }
-
-            // Record variables referenced by these messages so we can produce
-            // tests in the derive diagnostics to validate them.
-            let ident = quote::format_ident!("{snake_name}_refs");
-            let vrefs = variable_references(msg);
-            constants.extend(quote! {
-                #[cfg(test)]
-                pub const #ident: &[&str] = &[#(#vrefs),*];
-            })
-        }
-    }
-
-    for (mref, name) in message_refs.into_iter() {
-        if !previous_defns.contains_key(mref) {
-            Diagnostic::spanned(
-                resource_span,
-                Level::Error,
-                format!("referenced message `{mref}` does not exist (in message `{name}`)"),
-            )
-            .help(format!("you may have meant to use a variable reference (`{{${mref}}}`)"))
-            .emit();
-        }
-    }
-
-    if let Err(errs) = bundle.add_resource(resource) {
-        for e in errs {
-            match e {
-                FluentError::Overriding { kind, id } => {
-                    Diagnostic::spanned(
-                        resource_span,
-                        Level::Error,
-                        format!("overrides existing {kind}: `{id}`"),
-                    )
-                    .emit();
-                }
-                FluentError::ResolverError(_) | FluentError::ParserError(_) => unreachable!(),
-            }
-        }
-    }
-
-    finish(constants, quote! { include_str!(#relative_ftl_path) })
-}
-
-fn variable_references<'a>(msg: &Message<&'a str>) -> Vec<&'a str> {
-    let mut refs = vec![];
-    if let Some(Pattern { elements }) = &msg.value {
-        for elt in elements {
-            if let PatternElement::Placeable {
-                expression: Expression::Inline(InlineExpression::VariableReference { id }),
-            } = elt
-            {
-                refs.push(id.name);
-            }
-        }
-    }
-    for attr in &msg.attributes {
-        for elt in &attr.value.elements {
-            if let PatternElement::Placeable {
-                expression: Expression::Inline(InlineExpression::VariableReference { id }),
-            } = elt
-            {
-                refs.push(id.name);
-            }
-        }
-    }
-    refs
-}
diff --git a/compiler/rustc_fluent_macro/src/lib.rs b/compiler/rustc_fluent_macro/src/lib.rs
deleted file mode 100644
index 9cd947c4c672..000000000000
--- a/compiler/rustc_fluent_macro/src/lib.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-// tidy-alphabetical-start
-#![allow(rustc::default_hash_types)]
-#![feature(proc_macro_diagnostic)]
-#![feature(proc_macro_tracked_path)]
-// tidy-alphabetical-end
-
-use proc_macro::TokenStream;
-
-mod fluent;
-
-/// Implements the `fluent_messages` macro, which performs compile-time validation of the
-/// compiler's Fluent resources (i.e. that the resources parse and don't multiply define the same
-/// messages) and generates constants that make using those messages in diagnostics more ergonomic.
-///
-/// For example, given the following invocation of the macro..
-///
-/// ```ignore (rust)
-/// fluent_messages! { "./typeck.ftl" }
-/// ```
-/// ..where `typeck.ftl` has the following contents..
-///
-/// ```fluent
-/// typeck_field_multiply_specified_in_initializer =
-///     field `{$ident}` specified more than once
-///     .label = used more than once
-///     .label_previous_use = first use of `{$ident}`
-/// ```
-/// ...then the macro parse the Fluent resource, emitting a diagnostic if it fails to do so, and
-/// will generate the following code:
-///
-/// ```ignore (rust)
-/// pub static DEFAULT_LOCALE_RESOURCE: &'static [&'static str] = include_str!("./typeck.ftl");
-///
-/// mod fluent_generated {
-///     mod typeck {
-///         pub const field_multiply_specified_in_initializer: DiagMessage =
-///             DiagMessage::fluent("typeck_field_multiply_specified_in_initializer");
-///         pub const field_multiply_specified_in_initializer_label_previous_use: DiagMessage =
-///             DiagMessage::fluent_attr(
-///                 "typeck_field_multiply_specified_in_initializer",
-///                 "previous_use_label"
-///             );
-///     }
-/// }
-/// ```
-/// When emitting a diagnostic, the generated constants can be used as follows:
-///
-/// ```ignore (rust)
-/// let mut err = sess.struct_span_err(
-///     span,
-///     fluent::typeck::field_multiply_specified_in_initializer
-/// );
-/// err.span_default_label(span);
-/// err.span_label(
-///     previous_use_span,
-///     fluent::typeck::field_multiply_specified_in_initializer_label_previous_use
-/// );
-/// err.emit();
-/// ```
-///
-/// Note: any crate using this macro must also have a dependency on
-/// `rustc_errors`, because the generated code refers to things from that
-/// crate.
-#[proc_macro]
-pub fn fluent_messages(input: TokenStream) -> TokenStream {
-    fluent::fluent_messages(input)
-}
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap
index 13a06e353d89..3adf952d66e0 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_bench.snap
@@ -44,7 +44,6 @@ expression: bench
     - Set({bench::compiler/rustc_errors})
     - Set({bench::compiler/rustc_expand})
     - Set({bench::compiler/rustc_feature})
-    - Set({bench::compiler/rustc_fluent_macro})
     - Set({bench::compiler/rustc_fs_util})
     - Set({bench::compiler/rustc_graphviz})
     - Set({bench::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap
index 721ecaf4c487..1d6e63696b06 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_build_compiler.snap
@@ -26,7 +26,6 @@ expression: build compiler
     - Set({build::compiler/rustc_errors})
     - Set({build::compiler/rustc_expand})
     - Set({build::compiler/rustc_feature})
-    - Set({build::compiler/rustc_fluent_macro})
     - Set({build::compiler/rustc_fs_util})
     - Set({build::compiler/rustc_graphviz})
     - Set({build::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap
index 4c3b6e11b0e4..6fc2e190290e 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check.snap
@@ -28,7 +28,6 @@ expression: check
     - Set({check::compiler/rustc_errors})
     - Set({check::compiler/rustc_expand})
     - Set({check::compiler/rustc_feature})
-    - Set({check::compiler/rustc_fluent_macro})
     - Set({check::compiler/rustc_fs_util})
     - Set({check::compiler/rustc_graphviz})
     - Set({check::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap
index 71d9c0fafb80..c0456f7f84d3 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiler.snap
@@ -28,7 +28,6 @@ expression: check compiler
     - Set({check::compiler/rustc_errors})
     - Set({check::compiler/rustc_expand})
     - Set({check::compiler/rustc_feature})
-    - Set({check::compiler/rustc_fluent_macro})
     - Set({check::compiler/rustc_fs_util})
     - Set({check::compiler/rustc_graphviz})
     - Set({check::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap
index 16348adc79df..10f36ffa6748 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_check_compiletest_include_default_paths.snap
@@ -28,7 +28,6 @@ expression: check compiletest --include-default-paths
     - Set({check::compiler/rustc_errors})
     - Set({check::compiler/rustc_expand})
     - Set({check::compiler/rustc_feature})
-    - Set({check::compiler/rustc_fluent_macro})
     - Set({check::compiler/rustc_fs_util})
     - Set({check::compiler/rustc_graphviz})
     - Set({check::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap
index fe8334641285..492a10d3862a 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_clippy.snap
@@ -43,7 +43,6 @@ expression: clippy
     - Set({clippy::compiler/rustc_errors})
     - Set({clippy::compiler/rustc_expand})
     - Set({clippy::compiler/rustc_feature})
-    - Set({clippy::compiler/rustc_fluent_macro})
     - Set({clippy::compiler/rustc_fs_util})
     - Set({clippy::compiler/rustc_graphviz})
     - Set({clippy::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap
index 8a11b4d89dde..41889cd12480 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_fix.snap
@@ -28,7 +28,6 @@ expression: fix
     - Set({fix::compiler/rustc_errors})
     - Set({fix::compiler/rustc_expand})
     - Set({fix::compiler/rustc_feature})
-    - Set({fix::compiler/rustc_fluent_macro})
     - Set({fix::compiler/rustc_fs_util})
     - Set({fix::compiler/rustc_graphviz})
     - Set({fix::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap
index 46e93aa667cc..51e2c270e3ba 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test.snap
@@ -94,7 +94,6 @@ expression: test
     - Set({test::compiler/rustc_errors})
     - Set({test::compiler/rustc_expand})
     - Set({test::compiler/rustc_feature})
-    - Set({test::compiler/rustc_fluent_macro})
     - Set({test::compiler/rustc_fs_util})
     - Set({test::compiler/rustc_graphviz})
     - Set({test::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap
index c27da5a2a9e2..bc828c162bb0 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_coverage.snap
@@ -93,7 +93,6 @@ expression: test --skip=coverage
     - Set({test::compiler/rustc_errors})
     - Set({test::compiler/rustc_expand})
     - Set({test::compiler/rustc_feature})
-    - Set({test::compiler/rustc_fluent_macro})
     - Set({test::compiler/rustc_fs_util})
     - Set({test::compiler/rustc_graphviz})
     - Set({test::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap
index d5d709a60b49..ceb910e4cb36 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests.snap
@@ -1,6 +1,5 @@
 ---
 source: src/bootstrap/src/core/builder/cli_paths/tests.rs
-assertion_line: 68
 expression: test --skip=tests
 ---
 [Test] test::Tidy
@@ -58,7 +57,6 @@ expression: test --skip=tests
     - Set({test::compiler/rustc_errors})
     - Set({test::compiler/rustc_expand})
     - Set({test::compiler/rustc_feature})
-    - Set({test::compiler/rustc_fluent_macro})
     - Set({test::compiler/rustc_fs_util})
     - Set({test::compiler/rustc_graphviz})
     - Set({test::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap
index 85ca8f6ef7a0..f0e8f1aee2c7 100644
--- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap
+++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_test_skip_tests_etc.snap
@@ -1,6 +1,5 @@
 ---
 source: src/bootstrap/src/core/builder/cli_paths/tests.rs
-assertion_line: 68
 expression: test --skip=tests --skip=coverage-map --skip=coverage-run --skip=library --skip=tidyselftest
 ---
 [Test] test::Tidy
@@ -38,7 +37,6 @@ expression: test --skip=tests --skip=coverage-map --skip=coverage-run --skip=lib
     - Set({test::compiler/rustc_errors})
     - Set({test::compiler/rustc_expand})
     - Set({test::compiler/rustc_feature})
-    - Set({test::compiler/rustc_fluent_macro})
     - Set({test::compiler/rustc_fs_util})
     - Set({test::compiler/rustc_graphviz})
     - Set({test::compiler/rustc_hashes})
diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs
index af26c2f87d02..61db494c8c18 100644
--- a/src/bootstrap/src/core/builder/tests.rs
+++ b/src/bootstrap/src/core/builder/tests.rs
@@ -1815,7 +1815,7 @@ mod snapshot {
         insta::assert_snapshot!(
             ctx.config("check")
                 .path("compiler")
-                .render_steps(), @"[check] rustc 0  -> rustc 1  (75 crates)");
+                .render_steps(), @"[check] rustc 0  -> rustc 1  (74 crates)");
     }
 
     #[test]
@@ -1841,7 +1841,7 @@ mod snapshot {
             ctx.config("check")
                 .path("compiler")
                 .stage(1)
-                .render_steps(), @"[check] rustc 0  -> rustc 1  (75 crates)");
+                .render_steps(), @"[check] rustc 0  -> rustc 1  (74 crates)");
     }
 
     #[test]
@@ -1855,7 +1855,7 @@ mod snapshot {
         [build] llvm 
         [build] rustc 0  -> rustc 1 
         [build] rustc 1  -> std 1 
-        [check] rustc 1  -> rustc 2  (75 crates)
+        [check] rustc 1  -> rustc 2  (74 crates)
         ");
     }
 
@@ -1871,7 +1871,7 @@ mod snapshot {
         [build] rustc 0  -> rustc 1 
         [build] rustc 1  -> std 1 
         [check] rustc 1  -> std 1 
-        [check] rustc 1  -> rustc 2  (75 crates)
+        [check] rustc 1  -> rustc 2  (74 crates)
         [check] rustc 1  -> rustc 2 
         [check] rustc 1  -> Rustdoc 2 
         [check] rustc 1  -> rustc_codegen_cranelift 2 
@@ -1967,7 +1967,7 @@ mod snapshot {
             ctx.config("check")
                 .paths(&["library", "compiler"])
                 .args(&args)
-                .render_steps(), @"[check] rustc 0  -> rustc 1  (75 crates)");
+                .render_steps(), @"[check] rustc 0  -> rustc 1  (74 crates)");
     }
 
     #[test]
diff --git a/src/bootstrap/src/utils/proc_macro_deps.rs b/src/bootstrap/src/utils/proc_macro_deps.rs
index f1bf6e399fb1..fee9b12aca06 100644
--- a/src/bootstrap/src/utils/proc_macro_deps.rs
+++ b/src/bootstrap/src/utils/proc_macro_deps.rs
@@ -3,8 +3,6 @@
 /// See 
 pub static CRATES: &[&str] = &[
     // tidy-alphabetical-start
-    "annotate-snippets",
-    "anstyle",
     "askama_derive",
     "askama_parser",
     "basic-toml",
@@ -59,7 +57,6 @@ pub static CRATES: &[&str] = &[
     "unic-langid-impl",
     "unic-langid-macros",
     "unicode-ident",
-    "unicode-width",
     "version_check",
     "wasm-bindgen-macro-support",
     "wasm-bindgen-shared",

From 38374315165c83e77008f4457b862fd7aeee7d6d Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 19:04:39 +0100
Subject: [PATCH 1424/1843] Remove `ui-fulldeps` tests for slugs

---
 .../fluent-messages/duplicate-a-b.ftl         |   1 -
 .../fluent-messages/duplicate-a.ftl           |   1 -
 .../ui-fulldeps/fluent-messages/duplicate.ftl |   3 -
 .../fluent-messages/invalid-escape.ftl        |   1 -
 .../fluent-messages/label-with-hyphens.ftl    |   2 -
 .../fluent-messages/many-lines.ftl            |  11 -
 .../fluent-messages/missing-crate-name.ftl    |   2 -
 .../fluent-messages/missing-message-ref.ftl   |   1 -
 .../fluent-messages/missing-message.ftl       |   1 -
 .../fluent-messages/slug-with-hyphens.ftl     |   1 -
 tests/ui-fulldeps/fluent-messages/test.rs     |  87 --
 tests/ui-fulldeps/fluent-messages/test.stderr | 122 ---
 tests/ui-fulldeps/fluent-messages/valid.ftl   |   1 -
 .../session-diagnostic/diagnostic-derive.rs   | 827 ------------------
 .../diagnostic-derive.stderr                  | 634 --------------
 .../session-diagnostic/enforce_slug_naming.rs |  24 -
 .../enforce_slug_naming.stderr                |  11 -
 .../session-diagnostic/example.ftl            |   7 -
 .../session-diagnostic/invalid-variable.rs    |  20 -
 .../subdiagnostic-derive-2.rs                 |  37 -
 .../subdiagnostic-derive-2.stderr             |   9 -
 .../subdiagnostic-derive.rs                   | 809 -----------------
 .../subdiagnostic-derive.stderr               | 549 ------------
 23 files changed, 3161 deletions(-)
 delete mode 100644 tests/ui-fulldeps/fluent-messages/duplicate-a-b.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/duplicate-a.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/duplicate.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/invalid-escape.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/label-with-hyphens.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/many-lines.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/missing-crate-name.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/missing-message-ref.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/missing-message.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/slug-with-hyphens.ftl
 delete mode 100644 tests/ui-fulldeps/fluent-messages/test.rs
 delete mode 100644 tests/ui-fulldeps/fluent-messages/test.stderr
 delete mode 100644 tests/ui-fulldeps/fluent-messages/valid.ftl
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/diagnostic-derive.rs
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.rs
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.stderr
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/example.ftl
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/invalid-variable.rs
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.rs
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.stderr
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.rs
 delete mode 100644 tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.stderr

diff --git a/tests/ui-fulldeps/fluent-messages/duplicate-a-b.ftl b/tests/ui-fulldeps/fluent-messages/duplicate-a-b.ftl
deleted file mode 100644
index 9407c5170450..000000000000
--- a/tests/ui-fulldeps/fluent-messages/duplicate-a-b.ftl
+++ /dev/null
@@ -1 +0,0 @@
-a_b_key = Value
diff --git a/tests/ui-fulldeps/fluent-messages/duplicate-a.ftl b/tests/ui-fulldeps/fluent-messages/duplicate-a.ftl
deleted file mode 100644
index 9407c5170450..000000000000
--- a/tests/ui-fulldeps/fluent-messages/duplicate-a.ftl
+++ /dev/null
@@ -1 +0,0 @@
-a_b_key = Value
diff --git a/tests/ui-fulldeps/fluent-messages/duplicate.ftl b/tests/ui-fulldeps/fluent-messages/duplicate.ftl
deleted file mode 100644
index 871550b231a3..000000000000
--- a/tests/ui-fulldeps/fluent-messages/duplicate.ftl
+++ /dev/null
@@ -1,3 +0,0 @@
-no_crate_a_b_key = Value
-
-no_crate_a_b_key = Another Value
diff --git a/tests/ui-fulldeps/fluent-messages/invalid-escape.ftl b/tests/ui-fulldeps/fluent-messages/invalid-escape.ftl
deleted file mode 100644
index e28852ea0050..000000000000
--- a/tests/ui-fulldeps/fluent-messages/invalid-escape.ftl
+++ /dev/null
@@ -1 +0,0 @@
-no_crate_bad_escape = don't use \n, \', or \"
diff --git a/tests/ui-fulldeps/fluent-messages/label-with-hyphens.ftl b/tests/ui-fulldeps/fluent-messages/label-with-hyphens.ftl
deleted file mode 100644
index 3088b1f8dc83..000000000000
--- a/tests/ui-fulldeps/fluent-messages/label-with-hyphens.ftl
+++ /dev/null
@@ -1,2 +0,0 @@
-no_crate_some_slug = hi
-    .label-has-hyphens = test
diff --git a/tests/ui-fulldeps/fluent-messages/many-lines.ftl b/tests/ui-fulldeps/fluent-messages/many-lines.ftl
deleted file mode 100644
index 43660ebeacdc..000000000000
--- a/tests/ui-fulldeps/fluent-messages/many-lines.ftl
+++ /dev/null
@@ -1,11 +0,0 @@
-no_crate_foo = foo
-
-# This file tests error reporting for
-# fluent files with many lines.
-# The error message should point to the correct line number
-# and include no more context than necessary.
-
-no_crate_bar =
-
-no_crate_baz =
-    baz
diff --git a/tests/ui-fulldeps/fluent-messages/missing-crate-name.ftl b/tests/ui-fulldeps/fluent-messages/missing-crate-name.ftl
deleted file mode 100644
index 0a64e3894bd5..000000000000
--- a/tests/ui-fulldeps/fluent-messages/missing-crate-name.ftl
+++ /dev/null
@@ -1,2 +0,0 @@
-with-hyphens = 1234
-no-crate_foo = abcd
diff --git a/tests/ui-fulldeps/fluent-messages/missing-message-ref.ftl b/tests/ui-fulldeps/fluent-messages/missing-message-ref.ftl
deleted file mode 100644
index 4c6514a97700..000000000000
--- a/tests/ui-fulldeps/fluent-messages/missing-message-ref.ftl
+++ /dev/null
@@ -1 +0,0 @@
-no_crate_missing_message_ref = {message}
diff --git a/tests/ui-fulldeps/fluent-messages/missing-message.ftl b/tests/ui-fulldeps/fluent-messages/missing-message.ftl
deleted file mode 100644
index 61f56fd4d57b..000000000000
--- a/tests/ui-fulldeps/fluent-messages/missing-message.ftl
+++ /dev/null
@@ -1 +0,0 @@
-no_crate_missing_message =
diff --git a/tests/ui-fulldeps/fluent-messages/slug-with-hyphens.ftl b/tests/ui-fulldeps/fluent-messages/slug-with-hyphens.ftl
deleted file mode 100644
index a64c85094f1d..000000000000
--- a/tests/ui-fulldeps/fluent-messages/slug-with-hyphens.ftl
+++ /dev/null
@@ -1 +0,0 @@
-no_crate_this-slug-has-hyphens = hi
diff --git a/tests/ui-fulldeps/fluent-messages/test.rs b/tests/ui-fulldeps/fluent-messages/test.rs
deleted file mode 100644
index c1f5fe730c76..000000000000
--- a/tests/ui-fulldeps/fluent-messages/test.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-//@ normalize-stderr: "could not open Fluent resource:.*" -> "could not open Fluent resource: os-specific message"
-
-#![feature(rustc_private)]
-#![crate_type = "lib"]
-extern crate rustc_errors;
-extern crate rustc_fluent_macro;
-
-/// Copy of the relevant `DiagMessage` variant constructed by `fluent_messages` as it
-/// expects `crate::DiagMessage` to exist.
-pub enum DiagMessage {
-    FluentIdentifier(std::borrow::Cow<'static, str>, Option>),
-}
-
-/// Copy of the relevant `SubdiagMessage` variant constructed by `fluent_messages` as it
-/// expects `crate::SubdiagMessage` to exist.
-pub enum SubdiagMessage {
-    FluentAttr(std::borrow::Cow<'static, str>),
-}
-
-mod missing_absolute {
-    rustc_fluent_macro::fluent_messages! { "/definitely_does_not_exist.ftl" }
-    //~^ ERROR could not open Fluent resource
-}
-
-mod missing_relative {
-    rustc_fluent_macro::fluent_messages! { "../definitely_does_not_exist.ftl" }
-    //~^ ERROR could not open Fluent resource
-}
-
-mod missing_message {
-    rustc_fluent_macro::fluent_messages! { "./missing-message.ftl" }
-    //~^ ERROR could not parse Fluent resource
-}
-
-mod duplicate {
-    rustc_fluent_macro::fluent_messages! { "./duplicate.ftl" }
-    //~^ ERROR overrides existing message: `no_crate_a_b_key`
-}
-
-mod slug_with_hyphens {
-    rustc_fluent_macro::fluent_messages! { "./slug-with-hyphens.ftl" }
-    //~^ ERROR name `no_crate_this-slug-has-hyphens` contains a '-' character
-}
-
-mod label_with_hyphens {
-    rustc_fluent_macro::fluent_messages! { "./label-with-hyphens.ftl" }
-    //~^ ERROR attribute `label-has-hyphens` contains a '-' character
-}
-
-mod valid {
-    rustc_fluent_macro::fluent_messages! { "./valid.ftl" }
-
-    mod test_generated {
-        use super::{fluent_generated::no_crate_key, DEFAULT_LOCALE_RESOURCE};
-    }
-}
-
-mod missing_crate_name {
-    rustc_fluent_macro::fluent_messages! { "./missing-crate-name.ftl" }
-    //~^ ERROR name `no-crate_foo` contains a '-' character
-    //~| ERROR name `with-hyphens` contains a '-' character
-    //~| ERROR name `with-hyphens` does not start with the crate name
-
-    mod test_generated {
-        use super::{
-            fluent_generated::{no_crate_foo, with_hyphens},
-            DEFAULT_LOCALE_RESOURCE,
-        };
-    }
-}
-
-mod missing_message_ref {
-    rustc_fluent_macro::fluent_messages! { "./missing-message-ref.ftl" }
-    //~^ ERROR referenced message `message` does not exist
-}
-
-mod bad_escape {
-    rustc_fluent_macro::fluent_messages! { "./invalid-escape.ftl" }
-    //~^ ERROR invalid escape `\n`
-    //~| ERROR invalid escape `\"`
-    //~| ERROR invalid escape `\'`
-}
-
-mod many_lines {
-    rustc_fluent_macro::fluent_messages! { "./many-lines.ftl" }
-    //~^ ERROR could not parse Fluent resource
-}
diff --git a/tests/ui-fulldeps/fluent-messages/test.stderr b/tests/ui-fulldeps/fluent-messages/test.stderr
deleted file mode 100644
index 0b3bb14ce513..000000000000
--- a/tests/ui-fulldeps/fluent-messages/test.stderr
+++ /dev/null
@@ -1,122 +0,0 @@
-error: could not open Fluent resource: os-specific message
-  --> $DIR/test.rs:21:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "/definitely_does_not_exist.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-error: could not open Fluent resource: os-specific message
-  --> $DIR/test.rs:26:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "../definitely_does_not_exist.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-error: could not parse Fluent resource
-  --> $DIR/test.rs:31:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./missing-message.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: see additional errors emitted
-
-error: expected a message field for "no_crate_missing_message"
- --> ./missing-message.ftl:1:1
-  |
-1 | no_crate_missing_message =
-  | ^^^^^^^^^^^^^^^^^^^^^^^^^^
-  |
-
-error: overrides existing message: `no_crate_a_b_key`
-  --> $DIR/test.rs:36:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./duplicate.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^
-
-error: name `no_crate_this-slug-has-hyphens` contains a '-' character
-  --> $DIR/test.rs:41:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./slug-with-hyphens.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: replace any '-'s with '_'s
-
-error: attribute `label-has-hyphens` contains a '-' character
-  --> $DIR/test.rs:46:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./label-with-hyphens.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: replace any '-'s with '_'s
-
-error: name `with-hyphens` contains a '-' character
-  --> $DIR/test.rs:59:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./missing-crate-name.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: replace any '-'s with '_'s
-
-error: name `with-hyphens` does not start with the crate name
-  --> $DIR/test.rs:59:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./missing-crate-name.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: prepend `no_crate_` to the slug name: `no_crate_with_hyphens`
-
-error: name `no-crate_foo` contains a '-' character
-  --> $DIR/test.rs:59:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./missing-crate-name.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: replace any '-'s with '_'s
-
-error: referenced message `message` does not exist (in message `no_crate_missing_message_ref`)
-  --> $DIR/test.rs:73:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./missing-message-ref.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: you may have meant to use a variable reference (`{$message}`)
-
-error: invalid escape `\n` in Fluent resource
-  --> $DIR/test.rs:78:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./invalid-escape.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: Fluent does not interpret these escape sequences ()
-
-error: invalid escape `\"` in Fluent resource
-  --> $DIR/test.rs:78:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./invalid-escape.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: Fluent does not interpret these escape sequences ()
-
-error: invalid escape `\'` in Fluent resource
-  --> $DIR/test.rs:78:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./invalid-escape.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: Fluent does not interpret these escape sequences ()
-
-error: could not parse Fluent resource
-  --> $DIR/test.rs:85:44
-   |
-LL |     rustc_fluent_macro::fluent_messages! { "./many-lines.ftl" }
-   |                                            ^^^^^^^^^^^^^^^^^^
-   |
-   = help: see additional errors emitted
-
-error: expected a message field for "no_crate_bar"
- --> ./many-lines.ftl:8:1
-  |
-8 | no_crate_bar =
-  | ^^^^^^^^^^^^^^
-  |
-
-error: aborting due to 14 previous errors
-
diff --git a/tests/ui-fulldeps/fluent-messages/valid.ftl b/tests/ui-fulldeps/fluent-messages/valid.ftl
deleted file mode 100644
index 598473adb680..000000000000
--- a/tests/ui-fulldeps/fluent-messages/valid.ftl
+++ /dev/null
@@ -1 +0,0 @@
-no_crate_key = Valid!
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.rs b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.rs
deleted file mode 100644
index 798eca68b1c9..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.rs
+++ /dev/null
@@ -1,827 +0,0 @@
-//@ check-fail
-// Tests error conditions for specifying diagnostics using #[derive(Diagnostic)]
-//@ normalize-stderr: "the following other types implement trait `IntoDiagArg`:(?:.*\n){0,9}\s+and \d+ others" -> "normalized in stderr"
-//@ normalize-stderr: "(COMPILER_DIR/.*\.rs):[0-9]+:[0-9]+" -> "$1:LL:CC"
-
-// The proc_macro2 crate handles spans differently when on beta/stable release rather than nightly,
-// changing the output of this test. Since Diagnostic is strictly internal to the compiler
-// the test is just ignored on stable and beta:
-//@ ignore-stage1
-//@ ignore-beta
-//@ ignore-stable
-
-#![feature(rustc_private)]
-#![crate_type = "lib"]
-
-extern crate rustc_span;
-use rustc_span::symbol::Ident;
-use rustc_span::Span;
-
-extern crate rustc_fluent_macro;
-extern crate rustc_macros;
-use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-
-extern crate rustc_middle;
-use rustc_middle::ty::Ty;
-
-extern crate rustc_errors;
-use rustc_errors::{Applicability, DiagMessage, ErrCode, MultiSpan, SubdiagMessage};
-
-extern crate rustc_session;
-
-extern crate core;
-
-rustc_fluent_macro::fluent_messages! { "./example.ftl" }
-
-// E0123 and E0456 are no longer used, so we define our own constants here just for this test.
-const E0123: ErrCode = ErrCode::from_u32(0123);
-const E0456: ErrCode = ErrCode::from_u32(0456);
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct Hello {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct HelloWarn {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-//~^ ERROR unsupported type attribute for diagnostic derive enum
-enum DiagnosticOnEnum {
-    Foo,
-    //~^ ERROR diagnostic slug not specified
-    Bar,
-    //~^ ERROR diagnostic slug not specified
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[diag = "E0123"]
-//~^ ERROR expected parentheses: #[diag(...)]
-struct WrongStructAttrStyle {}
-
-#[derive(Diagnostic)]
-#[nonsense(no_crate_example, code = E0123)]
-//~^ ERROR `#[nonsense(...)]` is not a valid attribute
-//~^^ ERROR diagnostic slug not specified
-//~^^^ ERROR cannot find attribute `nonsense` in this scope
-struct InvalidStructAttr {}
-
-#[derive(Diagnostic)]
-#[diag(code = E0123)]
-//~^ ERROR diagnostic slug not specified
-struct InvalidLitNestedAttr {}
-
-#[derive(Diagnostic)]
-#[diag(nonsense, code = E0123)]
-//~^ ERROR cannot find value `nonsense` in module `crate::fluent_generated`
-struct InvalidNestedStructAttr {}
-
-#[derive(Diagnostic)]
-#[diag(nonsense("foo"), code = E0123, slug = "foo")]
-//~^ ERROR derive(Diagnostic): diagnostic slug not specified
-struct InvalidNestedStructAttr1 {}
-
-#[derive(Diagnostic)]
-#[diag(nonsense = "...", code = E0123, slug = "foo")]
-//~^ ERROR diagnostic slug not specified
-struct InvalidNestedStructAttr2 {}
-
-#[derive(Diagnostic)]
-#[diag(nonsense = 4, code = E0123, slug = "foo")]
-//~^ ERROR diagnostic slug not specified
-struct InvalidNestedStructAttr3 {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123, slug = "foo")]
-//~^ ERROR unknown argument
-struct InvalidNestedStructAttr4 {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct WrongPlaceField {
-    #[suggestion = "bar"]
-    //~^ ERROR `#[suggestion = ...]` is not a valid attribute
-    sp: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[diag(no_crate_example, code = E0456)]
-//~^ ERROR specified multiple times
-struct DiagSpecifiedTwice {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123, code = E0456)]
-//~^ ERROR specified multiple times
-struct CodeSpecifiedTwice {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, no_crate::example, code = E0123)]
-//~^ ERROR diagnostic slug must be the first argument
-struct SlugSpecifiedTwice {}
-
-#[derive(Diagnostic)]
-struct KindNotProvided {} //~ ERROR diagnostic slug not specified
-
-#[derive(Diagnostic)]
-#[diag(code = E0123)]
-//~^ ERROR diagnostic slug not specified
-struct SlugNotProvided {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct CodeNotProvided {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct MessageWrongType {
-    #[primary_span]
-    //~^ ERROR `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-    foo: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct InvalidPathFieldAttr {
-    #[nonsense]
-    //~^ ERROR `#[nonsense]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `nonsense` in this scope
-    foo: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithField {
-    name: String,
-    #[label(no_crate_label)]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithMessageAppliedToField {
-    #[label(no_crate_label)]
-    //~^ ERROR the `#[label(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-    name: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithNonexistentField {
-    #[suggestion(no_crate_suggestion, code = "{name}")]
-    //~^ ERROR `name` doesn't refer to a field on this type
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-//~^ ERROR invalid format string: expected `}`
-#[diag(no_crate_example, code = E0123)]
-struct ErrorMissingClosingBrace {
-    #[suggestion(no_crate_suggestion, code = "{name")]
-    suggestion: (Span, Applicability),
-    name: String,
-    val: usize,
-}
-
-#[derive(Diagnostic)]
-//~^ ERROR invalid format string: unmatched `}`
-#[diag(no_crate_example, code = E0123)]
-struct ErrorMissingOpeningBrace {
-    #[suggestion(no_crate_suggestion, code = "name}")]
-    suggestion: (Span, Applicability),
-    name: String,
-    val: usize,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct LabelOnSpan {
-    #[label(no_crate_label)]
-    sp: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct LabelOnNonSpan {
-    #[label(no_crate_label)]
-    //~^ ERROR the `#[label(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-    id: u32,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct Suggest {
-    #[suggestion(no_crate_suggestion, code = "This is the suggested code")]
-    #[suggestion(no_crate_suggestion, code = "This is the suggested code", style = "normal")]
-    #[suggestion(no_crate_suggestion, code = "This is the suggested code", style = "short")]
-    #[suggestion(no_crate_suggestion, code = "This is the suggested code", style = "hidden")]
-    #[suggestion(no_crate_suggestion, code = "This is the suggested code", style = "verbose")]
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithoutCode {
-    #[suggestion(no_crate_suggestion)]
-    //~^ ERROR suggestion without `code = "..."`
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithBadKey {
-    #[suggestion(nonsense = "bar")]
-    //~^ ERROR invalid nested attribute
-    //~| ERROR suggestion without `code = "..."`
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithShorthandMsg {
-    #[suggestion(msg = "bar")]
-    //~^ ERROR invalid nested attribute
-    //~| ERROR suggestion without `code = "..."`
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithoutMsg {
-    #[suggestion(code = "bar")]
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithTypesSwapped {
-    #[suggestion(no_crate_suggestion, code = "This is suggested code")]
-    suggestion: (Applicability, Span),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithWrongTypeApplicabilityOnly {
-    #[suggestion(no_crate_suggestion, code = "This is suggested code")]
-    //~^ ERROR wrong field type for suggestion
-    suggestion: Applicability,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithSpanOnly {
-    #[suggestion(no_crate_suggestion, code = "This is suggested code")]
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithDuplicateSpanAndApplicability {
-    #[suggestion(no_crate_suggestion, code = "This is suggested code")]
-    suggestion: (Span, Span, Applicability),
-    //~^ ERROR specified multiple times
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct SuggestWithDuplicateApplicabilityAndSpan {
-    #[suggestion(no_crate_suggestion, code = "This is suggested code")]
-    suggestion: (Applicability, Applicability, Span),
-    //~^ ERROR specified multiple times
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct WrongKindOfAnnotation {
-    #[label = "bar"]
-    //~^ ERROR `#[label = ...]` is not a valid attribute
-    z: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct OptionsInErrors {
-    #[label(no_crate_label)]
-    label: Option,
-    #[suggestion(no_crate_suggestion, code = "...")]
-    opt_sugg: Option<(Span, Applicability)>,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct MoveOutOfBorrowError<'tcx> {
-    name: Ident,
-    ty: Ty<'tcx>,
-    #[primary_span]
-    #[label(no_crate_label)]
-    span: Span,
-    #[label(no_crate_label)]
-    other_span: Span,
-    #[suggestion(no_crate_suggestion, code = "{name}.clone()")]
-    opt_sugg: Option<(Span, Applicability)>,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithLifetime<'a> {
-    #[label(no_crate_label)]
-    span: Span,
-    name: &'a str,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithDefaultLabelAttr<'a> {
-    #[label]
-    span: Span,
-    name: &'a str,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ArgFieldWithoutSkip {
-    #[primary_span]
-    span: Span,
-    other: Hello,
-    //~^ ERROR the trait bound `Hello: IntoDiagArg` is not satisfied
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ArgFieldWithSkip {
-    #[primary_span]
-    span: Span,
-    // `Hello` does not implement `IntoDiagArg` so this would result in an error if
-    // not for `#[skip_arg]`.
-    #[skip_arg]
-    other: Hello,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithSpannedNote {
-    #[note]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithSpannedNoteCustom {
-    #[note(no_crate_note)]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[note]
-struct ErrorWithNote {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[note(no_crate_note)]
-struct ErrorWithNoteCustom {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithSpannedHelp {
-    #[help]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithSpannedHelpCustom {
-    #[help(no_crate_help)]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[help]
-struct ErrorWithHelp {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[help(no_crate_help)]
-struct ErrorWithHelpCustom {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[help]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithHelpWrongOrder {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[help(no_crate_help)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithHelpCustomWrongOrder {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[note]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithNoteWrongOrder {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[note(no_crate_note)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithNoteCustomWrongOrder {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ApplicabilityInBoth {
-    #[suggestion(no_crate_suggestion, code = "...", applicability = "maybe-incorrect")]
-    //~^ ERROR specified multiple times
-    suggestion: (Span, Applicability),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct InvalidApplicability {
-    #[suggestion(no_crate_suggestion, code = "...", applicability = "batman")]
-    //~^ ERROR invalid applicability
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ValidApplicability {
-    #[suggestion(no_crate_suggestion, code = "...", applicability = "maybe-incorrect")]
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct NoApplicability {
-    #[suggestion(no_crate_suggestion, code = "...")]
-    suggestion: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[note(no_crate_example)]
-struct Note;
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct Subdiagnostic {
-    #[subdiagnostic]
-    note: Note,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct VecField {
-    #[primary_span]
-    #[label]
-    spans: Vec,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct UnitField {
-    #[primary_span]
-    spans: Span,
-    #[help]
-    foo: (),
-    #[help(no_crate_help)]
-    bar: (),
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct OptUnitField {
-    #[primary_span]
-    spans: Span,
-    #[help]
-    foo: Option<()>,
-    #[help(no_crate_help)]
-    bar: Option<()>,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct BoolField {
-    #[primary_span]
-    spans: Span,
-    #[help]
-    foo: bool,
-    #[help(no_crate_help)]
-    //~^ ERROR the `#[help(...)]` attribute can only be applied to fields of type
-    // only allow plain 'bool' fields
-    bar: Option,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct LabelWithTrailingPath {
-    #[label(no_crate_label, foo)]
-    //~^ ERROR a diagnostic slug must be the first argument to the attribute
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct LabelWithTrailingNameValue {
-    #[label(no_crate_label, foo = "...")]
-    //~^ ERROR no nested attribute expected here
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct LabelWithTrailingList {
-    #[label(no_crate_label, foo("..."))]
-    //~^ ERROR no nested attribute expected here
-    span: Span,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(no_crate_example)]
-struct LintsGood {}
-
-#[derive(LintDiagnostic)]
-#[diag(no_crate_example)]
-struct PrimarySpanOnLint {
-    #[primary_span]
-    //~^ ERROR `#[primary_span]` is not a valid attribute
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct ErrorWithMultiSpan {
-    #[primary_span]
-    span: MultiSpan,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[warning]
-struct ErrorWithWarn {
-    val: String,
-}
-
-#[derive(Diagnostic)]
-#[error(no_crate_example, code = E0123)]
-//~^ ERROR `#[error(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
-//~| ERROR cannot find attribute `error` in this scope
-struct ErrorAttribute {}
-
-#[derive(Diagnostic)]
-#[warn_(no_crate_example, code = E0123)]
-//~^ ERROR `#[warn_(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
-//~| ERROR cannot find attribute `warn_` in this scope
-struct WarnAttribute {}
-
-#[derive(Diagnostic)]
-#[lint(no_crate_example, code = E0123)]
-//~^ ERROR `#[lint(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
-//~| ERROR cannot find attribute `lint` in this scope
-struct LintAttributeOnSessionDiag {}
-
-#[derive(LintDiagnostic)]
-#[lint(no_crate_example, code = E0123)]
-//~^ ERROR `#[lint(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
-//~| ERROR cannot find attribute `lint` in this scope
-struct LintAttributeOnLintDiag {}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct DuplicatedSuggestionCode {
-    #[suggestion(no_crate_suggestion, code = "...", code = ",,,")]
-    //~^ ERROR specified multiple times
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct InvalidTypeInSuggestionTuple {
-    #[suggestion(no_crate_suggestion, code = "...")]
-    suggestion: (Span, usize),
-    //~^ ERROR wrong types for suggestion
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct MissingApplicabilityInSuggestionTuple {
-    #[suggestion(no_crate_suggestion, code = "...")]
-    suggestion: (Span,),
-    //~^ ERROR wrong types for suggestion
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct MissingCodeInSuggestion {
-    #[suggestion(no_crate_suggestion)]
-    //~^ ERROR suggestion without `code = "..."`
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[multipart_suggestion(no_crate_suggestion)]
-//~^ ERROR `#[multipart_suggestion(...)]` is not a valid attribute
-//~| ERROR cannot find attribute `multipart_suggestion` in this scope
-#[multipart_suggestion()]
-//~^ ERROR cannot find attribute `multipart_suggestion` in this scope
-//~| ERROR `#[multipart_suggestion(...)]` is not a valid attribute
-struct MultipartSuggestion {
-    #[multipart_suggestion(no_crate_suggestion)]
-    //~^ ERROR `#[multipart_suggestion(...)]` is not a valid attribute
-    //~| ERROR cannot find attribute `multipart_suggestion` in this scope
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[suggestion(no_crate_suggestion, code = "...")]
-//~^ ERROR `#[suggestion(...)]` is not a valid attribute
-struct SuggestionOnStruct {
-    #[primary_span]
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-#[label]
-//~^ ERROR `#[label]` is not a valid attribute
-struct LabelOnStruct {
-    #[primary_span]
-    suggestion: Span,
-}
-
-#[derive(Diagnostic)]
-enum ExampleEnum {
-    #[diag(no_crate_example)]
-    Foo {
-        #[primary_span]
-        sp: Span,
-        #[note]
-        note_sp: Span,
-    },
-    #[diag(no_crate_example)]
-    Bar {
-        #[primary_span]
-        sp: Span,
-    },
-    #[diag(no_crate_example)]
-    Baz,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct RawIdentDiagnosticArg {
-    pub r#type: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticBad {
-    #[subdiagnostic(bad)]
-    //~^ ERROR `#[subdiagnostic(...)]` is not a valid attribute
-    note: Note,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticBadStr {
-    #[subdiagnostic = "bad"]
-    //~^ ERROR `#[subdiagnostic = ...]` is not a valid attribute
-    note: Note,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticBadTwice {
-    #[subdiagnostic(bad, bad)]
-    //~^ ERROR `#[subdiagnostic(...)]` is not a valid attribute
-    note: Note,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticBadLitStr {
-    #[subdiagnostic("bad")]
-    //~^ ERROR `#[subdiagnostic(...)]` is not a valid attribute
-    note: Note,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticEagerLint {
-    #[subdiagnostic(eager)]
-    //~^ ERROR `#[subdiagnostic(...)]` is not a valid attribute
-    note: Note,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticEagerFormerlyCorrect {
-    #[subdiagnostic(eager)]
-    //~^ ERROR `#[subdiagnostic(...)]` is not a valid attribute
-    note: Note,
-}
-
-// Check that formatting of `correct` in suggestion doesn't move the binding for that field, making
-// the `arg` call a compile error; and that isn't worked around by moving the `arg` call
-// after the `span_suggestion` call - which breaks eager translation.
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, applicability = "machine-applicable", code = "{correct}")]
-pub(crate) struct SubdiagnosticWithSuggestion {
-    #[primary_span]
-    span: Span,
-    invalid: String,
-    correct: String,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SubdiagnosticEagerSuggestion {
-    #[subdiagnostic(eager)]
-    //~^ ERROR `#[subdiagnostic(...)]` is not a valid attribute
-    sub: SubdiagnosticWithSuggestion,
-}
-
-/// with a doc comment on the type..
-#[derive(Diagnostic)]
-#[diag(no_crate_example, code = E0123)]
-struct WithDocComment {
-    /// ..and the field
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionsGood {
-    #[suggestion(code("foo", "bar"))]
-    sub: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionsSingleItem {
-    #[suggestion(code("foo"))]
-    sub: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionsNoItem {
-    #[suggestion(code())]
-    //~^ ERROR expected at least one string literal for `code(...)`
-    sub: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionsInvalidItem {
-    #[suggestion(code(foo))]
-    //~^ ERROR `code(...)` must contain only string literals
-    //~| ERROR unexpected token, expected `)`
-    sub: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionsInvalidLiteral {
-    #[suggestion(code = 3)]
-    //~^ ERROR expected string literal
-    sub: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionStyleGood {
-    #[suggestion(code = "", style = "hidden")]
-    sub: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(no_crate_example)]
-struct SuggestionOnVec {
-    #[suggestion(no_crate_suggestion, code = "")]
-    //~^ ERROR `#[suggestion(...)]` is not a valid attribute
-    sub: Vec,
-}
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr
deleted file mode 100644
index 9ecb6c15767e..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive.stderr
+++ /dev/null
@@ -1,634 +0,0 @@
-error: derive(Diagnostic): unsupported type attribute for diagnostic derive enum
-  --> $DIR/diagnostic-derive.rs:49:1
-   |
-LL | #[diag(no_crate_example, code = E0123)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:52:5
-   |
-LL |     Foo,
-   |     ^^^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:54:5
-   |
-LL |     Bar,
-   |     ^^^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: expected parentheses: #[diag(...)]
-  --> $DIR/diagnostic-derive.rs:60:8
-   |
-LL | #[diag = "E0123"]
-   |        ^
-
-error: derive(Diagnostic): `#[nonsense(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:65:1
-   |
-LL | #[nonsense(no_crate_example, code = E0123)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:65:1
-   |
-LL | #[nonsense(no_crate_example, code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:72:1
-   |
-LL | #[diag(code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:82:1
-   |
-LL | #[diag(nonsense("foo"), code = E0123, slug = "foo")]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:87:1
-   |
-LL | #[diag(nonsense = "...", code = E0123, slug = "foo")]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:92:1
-   |
-LL | #[diag(nonsense = 4, code = E0123, slug = "foo")]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): unknown argument
-  --> $DIR/diagnostic-derive.rs:97:40
-   |
-LL | #[diag(no_crate_example, code = E0123, slug = "foo")]
-   |                                        ^^^^
-   |
-   = note: only the `code` parameter is valid after the slug
-
-error: derive(Diagnostic): `#[suggestion = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:104:5
-   |
-LL |     #[suggestion = "bar"]
-   |     ^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive.rs:111:26
-   |
-LL | #[diag(no_crate_example, code = E0456)]
-   |                          ^^^^
-   |
-note: previously specified here
-  --> $DIR/diagnostic-derive.rs:110:26
-   |
-LL | #[diag(no_crate_example, code = E0123)]
-   |                          ^^^^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive.rs:116:40
-   |
-LL | #[diag(no_crate_example, code = E0123, code = E0456)]
-   |                                        ^^^^
-   |
-note: previously specified here
-  --> $DIR/diagnostic-derive.rs:116:26
-   |
-LL | #[diag(no_crate_example, code = E0123, code = E0456)]
-   |                          ^^^^
-
-error: derive(Diagnostic): diagnostic slug must be the first argument
-  --> $DIR/diagnostic-derive.rs:121:26
-   |
-LL | #[diag(no_crate_example, no_crate::example, code = E0123)]
-   |                          ^^^^^^^^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:126:1
-   |
-LL | struct KindNotProvided {}
-   | ^^^^^^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:129:1
-   |
-LL | #[diag(code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): the `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/diagnostic-derive.rs:140:5
-   |
-LL |     #[primary_span]
-   |     ^
-
-error: derive(Diagnostic): `#[nonsense]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:148:5
-   |
-LL |     #[nonsense]
-   |     ^
-
-error: derive(Diagnostic): the `#[label(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/diagnostic-derive.rs:165:5
-   |
-LL |     #[label(no_crate_label)]
-   |     ^
-
-error: derive(Diagnostic): `name` doesn't refer to a field on this type
-  --> $DIR/diagnostic-derive.rs:173:46
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "{name}")]
-   |                                              ^^^^^^^^
-
-error: invalid format string: expected `}` but string was terminated
-  --> $DIR/diagnostic-derive.rs:178:10
-   |
-LL | #[derive(Diagnostic)]
-   |          ^^^^^^^^^^ expected `}` in format string
-   |
-   = note: if you intended to print `{`, you can escape it using `{{`
-   = note: this error originates in the derive macro `Diagnostic` (in Nightly builds, run with -Z macro-backtrace for more info)
-
-error: invalid format string: unmatched `}` found
-  --> $DIR/diagnostic-derive.rs:188:10
-   |
-LL | #[derive(Diagnostic)]
-   |          ^^^^^^^^^^ unmatched `}` in format string
-   |
-   = note: if you intended to print `}`, you can escape it using `}}`
-   = note: this error originates in the derive macro `Diagnostic` (in Nightly builds, run with -Z macro-backtrace for more info)
-
-error: derive(Diagnostic): the `#[label(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/diagnostic-derive.rs:208:5
-   |
-LL |     #[label(no_crate_label)]
-   |     ^
-
-error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive.rs:227:5
-   |
-LL |     #[suggestion(no_crate_suggestion)]
-   |     ^
-
-error: derive(Diagnostic): invalid nested attribute
-  --> $DIR/diagnostic-derive.rs:235:18
-   |
-LL |     #[suggestion(nonsense = "bar")]
-   |                  ^^^^^^^^
-   |
-   = help: only `style`, `code` and `applicability` are valid nested attributes
-
-error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive.rs:235:5
-   |
-LL |     #[suggestion(nonsense = "bar")]
-   |     ^
-
-error: derive(Diagnostic): invalid nested attribute
-  --> $DIR/diagnostic-derive.rs:244:18
-   |
-LL |     #[suggestion(msg = "bar")]
-   |                  ^^^
-   |
-   = help: only `style`, `code` and `applicability` are valid nested attributes
-
-error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive.rs:244:5
-   |
-LL |     #[suggestion(msg = "bar")]
-   |     ^
-
-error: derive(Diagnostic): wrong field type for suggestion
-  --> $DIR/diagnostic-derive.rs:267:5
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "This is suggested code")]
-   |     ^
-   |
-   = help: `#[suggestion(...)]` should be applied to fields of type `Span` or `(Span, Applicability)`
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive.rs:283:24
-   |
-LL |     suggestion: (Span, Span, Applicability),
-   |                        ^^^^
-   |
-note: previously specified here
-  --> $DIR/diagnostic-derive.rs:283:18
-   |
-LL |     suggestion: (Span, Span, Applicability),
-   |                  ^^^^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive.rs:291:33
-   |
-LL |     suggestion: (Applicability, Applicability, Span),
-   |                                 ^^^^^^^^^^^^^
-   |
-note: previously specified here
-  --> $DIR/diagnostic-derive.rs:291:18
-   |
-LL |     suggestion: (Applicability, Applicability, Span),
-   |                  ^^^^^^^^^^^^^
-
-error: derive(Diagnostic): `#[label = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:298:5
-   |
-LL |     #[label = "bar"]
-   |     ^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive.rs:449:5
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "...", applicability = "maybe-incorrect")]
-   |     ^
-   |
-note: previously specified here
-  --> $DIR/diagnostic-derive.rs:451:24
-   |
-LL |     suggestion: (Span, Applicability),
-   |                        ^^^^^^^^^^^^^
-
-error: derive(Diagnostic): invalid applicability
-  --> $DIR/diagnostic-derive.rs:457:69
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "...", applicability = "batman")]
-   |                                                                     ^^^^^^^^
-
-error: derive(Diagnostic): the `#[help(...)]` attribute can only be applied to fields of type `Span`, `MultiSpan`, `bool` or `()`
-  --> $DIR/diagnostic-derive.rs:524:5
-   |
-LL |     #[help(no_crate_help)]
-   |     ^
-
-error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/diagnostic-derive.rs:533:29
-   |
-LL |     #[label(no_crate_label, foo)]
-   |                             ^^^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/diagnostic-derive.rs:541:29
-   |
-LL |     #[label(no_crate_label, foo = "...")]
-   |                             ^^^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/diagnostic-derive.rs:549:29
-   |
-LL |     #[label(no_crate_label, foo("..."))]
-   |                             ^^^
-
-error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:561:5
-   |
-LL |     #[primary_span]
-   |     ^
-   |
-   = help: the `primary_span` field attribute is not valid for lint diagnostics
-
-error: derive(Diagnostic): `#[error(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:581:1
-   |
-LL | #[error(no_crate_example, code = E0123)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:581:1
-   |
-LL | #[error(no_crate_example, code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): `#[warn_(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:588:1
-   |
-LL | #[warn_(no_crate_example, code = E0123)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:588:1
-   |
-LL | #[warn_(no_crate_example, code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): `#[lint(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:595:1
-   |
-LL | #[lint(no_crate_example, code = E0123)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:595:1
-   |
-LL | #[lint(no_crate_example, code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): `#[lint(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:602:1
-   |
-LL | #[lint(no_crate_example, code = E0123)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive.rs:602:1
-   |
-LL | #[lint(no_crate_example, code = E0123)]
-   | ^
-   |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive.rs:611:53
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "...", code = ",,,")]
-   |                                                     ^^^^
-   |
-note: previously specified here
-  --> $DIR/diagnostic-derive.rs:611:39
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "...", code = ",,,")]
-   |                                       ^^^^
-
-error: derive(Diagnostic): wrong types for suggestion
-  --> $DIR/diagnostic-derive.rs:620:24
-   |
-LL |     suggestion: (Span, usize),
-   |                        ^^^^^
-   |
-   = help: `#[suggestion(...)]` on a tuple field must be applied to fields of type `(Span, Applicability)`
-
-error: derive(Diagnostic): wrong types for suggestion
-  --> $DIR/diagnostic-derive.rs:628:17
-   |
-LL |     suggestion: (Span,),
-   |                 ^^^^^^^
-   |
-   = help: `#[suggestion(...)]` on a tuple field must be applied to fields of type `(Span, Applicability)`
-
-error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive.rs:635:5
-   |
-LL |     #[suggestion(no_crate_suggestion)]
-   |     ^
-
-error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:642:1
-   |
-LL | #[multipart_suggestion(no_crate_suggestion)]
-   | ^
-   |
-   = help: consider creating a `Subdiagnostic` instead
-
-error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:645:1
-   |
-LL | #[multipart_suggestion()]
-   | ^
-   |
-   = help: consider creating a `Subdiagnostic` instead
-
-error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:649:5
-   |
-LL |     #[multipart_suggestion(no_crate_suggestion)]
-   |     ^
-   |
-   = help: consider creating a `Subdiagnostic` instead
-
-error: derive(Diagnostic): `#[suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:657:1
-   |
-LL | #[suggestion(no_crate_suggestion, code = "...")]
-   | ^
-   |
-   = help: `#[label]` and `#[suggestion]` can only be applied to fields
-
-error: derive(Diagnostic): `#[label]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:666:1
-   |
-LL | #[label]
-   | ^
-   |
-   = help: `#[label]` and `#[suggestion]` can only be applied to fields
-
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:700:5
-   |
-LL |     #[subdiagnostic(bad)]
-   |     ^
-
-error: derive(Diagnostic): `#[subdiagnostic = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:708:5
-   |
-LL |     #[subdiagnostic = "bad"]
-   |     ^
-
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:716:5
-   |
-LL |     #[subdiagnostic(bad, bad)]
-   |     ^
-
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:724:5
-   |
-LL |     #[subdiagnostic("bad")]
-   |     ^
-
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:732:5
-   |
-LL |     #[subdiagnostic(eager)]
-   |     ^
-
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:740:5
-   |
-LL |     #[subdiagnostic(eager)]
-   |     ^
-
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:761:5
-   |
-LL |     #[subdiagnostic(eager)]
-   |     ^
-
-error: derive(Diagnostic): expected at least one string literal for `code(...)`
-  --> $DIR/diagnostic-derive.rs:792:23
-   |
-LL |     #[suggestion(code())]
-   |                       ^
-
-error: derive(Diagnostic): `code(...)` must contain only string literals
-  --> $DIR/diagnostic-derive.rs:800:23
-   |
-LL |     #[suggestion(code(foo))]
-   |                       ^^^
-
-error: unexpected token, expected `)`
-  --> $DIR/diagnostic-derive.rs:800:23
-   |
-LL |     #[suggestion(code(foo))]
-   |                       ^^^
-
-error: expected string literal
-  --> $DIR/diagnostic-derive.rs:809:25
-   |
-LL |     #[suggestion(code = 3)]
-   |                         ^
-
-error: derive(Diagnostic): `#[suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive.rs:824:5
-   |
-LL |     #[suggestion(no_crate_suggestion, code = "")]
-   |     ^
-   |
-   = note: `#[suggestion(...)]` applied to `Vec` field is ambiguous
-   = help: to show a suggestion consisting of multiple parts, use a `Subdiagnostic` annotated with `#[multipart_suggestion(...)]`
-   = help: to show a variable set of suggestions, use a `Vec` of `Subdiagnostic`s annotated with `#[suggestion(...)]`
-
-error: cannot find attribute `nonsense` in this scope
-  --> $DIR/diagnostic-derive.rs:65:3
-   |
-LL | #[nonsense(no_crate_example, code = E0123)]
-   |   ^^^^^^^^
-
-error: cannot find attribute `nonsense` in this scope
-  --> $DIR/diagnostic-derive.rs:148:7
-   |
-LL |     #[nonsense]
-   |       ^^^^^^^^
-
-error: cannot find attribute `error` in this scope
-  --> $DIR/diagnostic-derive.rs:581:3
-   |
-LL | #[error(no_crate_example, code = E0123)]
-   |   ^^^^^
-   |
-help: `error` is an attribute that can be used by the derive macro `Error`, you might be missing a `derive` attribute
-   |
-LL + #[derive(Error)]
-LL | struct ErrorAttribute {}
-   |
-
-error: cannot find attribute `warn_` in this scope
-  --> $DIR/diagnostic-derive.rs:588:3
-   |
-LL | #[warn_(no_crate_example, code = E0123)]
-   |   ^^^^^
-   |
-help: a built-in attribute with a similar name exists
-   |
-LL - #[warn_(no_crate_example, code = E0123)]
-LL + #[warn(no_crate_example, code = E0123)]
-   |
-
-error: cannot find attribute `lint` in this scope
-  --> $DIR/diagnostic-derive.rs:595:3
-   |
-LL | #[lint(no_crate_example, code = E0123)]
-   |   ^^^^
-   |
-help: a built-in attribute with a similar name exists
-   |
-LL - #[lint(no_crate_example, code = E0123)]
-LL + #[link(no_crate_example, code = E0123)]
-   |
-
-error: cannot find attribute `lint` in this scope
-  --> $DIR/diagnostic-derive.rs:602:3
-   |
-LL | #[lint(no_crate_example, code = E0123)]
-   |   ^^^^
-   |
-help: a built-in attribute with a similar name exists
-   |
-LL - #[lint(no_crate_example, code = E0123)]
-LL + #[link(no_crate_example, code = E0123)]
-   |
-
-error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive.rs:642:3
-   |
-LL | #[multipart_suggestion(no_crate_suggestion)]
-   |   ^^^^^^^^^^^^^^^^^^^^
-   |
-help: `multipart_suggestion` is an attribute that can be used by the derive macro `Subdiagnostic`, you might be missing a `derive` attribute
-   |
-LL + #[derive(Subdiagnostic)]
-LL | struct MultipartSuggestion {
-   |
-
-error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive.rs:645:3
-   |
-LL | #[multipart_suggestion()]
-   |   ^^^^^^^^^^^^^^^^^^^^
-   |
-help: `multipart_suggestion` is an attribute that can be used by the derive macro `Subdiagnostic`, you might be missing a `derive` attribute
-   |
-LL + #[derive(Subdiagnostic)]
-LL | struct MultipartSuggestion {
-   |
-
-error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive.rs:649:7
-   |
-LL |     #[multipart_suggestion(no_crate_suggestion)]
-   |       ^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: `multipart_suggestion` is an attribute that can be used by the derive macro `Subdiagnostic`, you might be missing a `derive` attribute
-
-error[E0425]: cannot find value `nonsense` in module `crate::fluent_generated`
-  --> $DIR/diagnostic-derive.rs:77:8
-   |
-LL | #[diag(nonsense, code = E0123)]
-   |        ^^^^^^^^ not found in `crate::fluent_generated`
-
-error[E0277]: the trait bound `Hello: IntoDiagArg` is not satisfied
-  --> $DIR/diagnostic-derive.rs:347:12
-   |
-LL | #[derive(Diagnostic)]
-   |          ---------- required by a bound introduced by this call
-...
-LL |     other: Hello,
-   |            ^^^^^ unsatisfied trait bound
-   |
-help: the nightly-only, unstable trait `IntoDiagArg` is not implemented for `Hello`
-  --> $DIR/diagnostic-derive.rs:42:1
-   |
-LL | struct Hello {}
-   | ^^^^^^^^^^^^
-   = help: normalized in stderr
-note: required by a bound in `Diag::<'a, G>::arg`
-  --> $COMPILER_DIR/rustc_errors/src/diagnostic.rs:LL:CC
-  ::: $COMPILER_DIR/rustc_errors/src/diagnostic.rs:LL:CC
-   |
-   = note: in this macro invocation
-   = note: this error originates in the macro `with_fn` (in Nightly builds, run with -Z macro-backtrace for more info)
-
-error: aborting due to 80 previous errors
-
-Some errors have detailed explanations: E0277, E0425.
-For more information about an error, try `rustc --explain E0277`.
diff --git a/tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.rs b/tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.rs
deleted file mode 100644
index 6402b00ef0a9..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-//@ rustc-env:CARGO_CRATE_NAME=rustc_dummy
-
-#![feature(rustc_private)]
-#![crate_type = "lib"]
-
-extern crate rustc_span;
-use rustc_span::symbol::Ident;
-use rustc_span::Span;
-
-extern crate rustc_macros;
-use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-
-extern crate rustc_middle;
-use rustc_middle::ty::Ty;
-
-extern crate rustc_errors;
-use rustc_errors::{Applicability, MultiSpan};
-
-extern crate rustc_session;
-
-#[derive(Diagnostic)]
-#[diag(compiletest_example, code = E0123)]
-//~^ ERROR diagnostic slug and crate name do not match
-struct Hello {}
diff --git a/tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.stderr b/tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.stderr
deleted file mode 100644
index 4f54239f0fa3..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/enforce_slug_naming.stderr
+++ /dev/null
@@ -1,11 +0,0 @@
-error: derive(Diagnostic): diagnostic slug and crate name do not match
-  --> $DIR/enforce_slug_naming.rs:22:8
-   |
-LL | #[diag(compiletest_example, code = E0123)]
-   |        ^^^^^^^^^^^^^^^^^^^
-   |
-   = note: slug is `compiletest_example` but the crate name is `rustc_dummy`
-   = help: expected a slug starting with `dummy_...`
-
-error: aborting due to 1 previous error
-
diff --git a/tests/ui-fulldeps/session-diagnostic/example.ftl b/tests/ui-fulldeps/session-diagnostic/example.ftl
deleted file mode 100644
index 1d1ba8e1bd5c..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/example.ftl
+++ /dev/null
@@ -1,7 +0,0 @@
-no_crate_example = this is an example message used in testing
-    .note = with a note
-    .help = with a help
-    .suggestion = with a suggestion
-    .label = with a label
-
-no_crate_bad_reference = {$r} does not exist
diff --git a/tests/ui-fulldeps/session-diagnostic/invalid-variable.rs b/tests/ui-fulldeps/session-diagnostic/invalid-variable.rs
deleted file mode 100644
index cbe9e3f4ef4c..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/invalid-variable.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-//@ run-fail
-//@ compile-flags: --test
-// test that messages referencing non-existent fields cause test failures
-
-#![feature(rustc_private)]
-#![crate_type = "lib"]
-
-extern crate rustc_driver;
-extern crate rustc_fluent_macro;
-extern crate rustc_macros;
-extern crate rustc_errors;
-use rustc_macros::Diagnostic;
-use rustc_errors::{DiagMessage, SubdiagMessage};
-extern crate rustc_session;
-
-rustc_fluent_macro::fluent_messages! { "./example.ftl" }
-
-#[derive(Diagnostic)]
-#[diag(no_crate_bad_reference)]
-struct BadRef;
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.rs b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.rs
deleted file mode 100644
index f42fe1689842..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-//@ check-fail
-// Tests error conditions for specifying subdiagnostics using #[derive(Subdiagnostic)].
-// This test is split off from the main `subdiagnostic-derive`,
-// because this error is generated post-expansion.
-
-// The proc_macro2 crate handles spans differently when on beta/stable release rather than nightly,
-// changing the output of this test. Since Subdiagnostic is strictly internal to the compiler
-// the test is just ignored on stable and beta:
-//@ ignore-stage1
-//@ ignore-beta
-//@ ignore-stable
-
-#![feature(rustc_private)]
-#![crate_type = "lib"]
-
-extern crate rustc_errors;
-extern crate rustc_fluent_macro;
-extern crate rustc_macros;
-extern crate rustc_session;
-extern crate rustc_span;
-extern crate core;
-
-use rustc_errors::{Applicability, DiagMessage, SubdiagMessage};
-use rustc_macros::Subdiagnostic;
-use rustc_span::Span;
-
-rustc_fluent_macro::fluent_messages! { "./example.ftl" }
-
-#[derive(Subdiagnostic)]
-#[label(slug)]
-//~^ ERROR cannot find value `slug` in module `crate::fluent_generated`
-//~^^ NOTE not found in `crate::fluent_generated`
-struct L {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.stderr b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.stderr
deleted file mode 100644
index 37566e39fcd6..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-2.stderr
+++ /dev/null
@@ -1,9 +0,0 @@
-error[E0425]: cannot find value `slug` in module `crate::fluent_generated`
-  --> $DIR/subdiagnostic-derive-2.rs:30:9
-   |
-LL | #[label(slug)]
-   |         ^^^^ not found in `crate::fluent_generated`
-
-error: aborting due to 1 previous error
-
-For more information about this error, try `rustc --explain E0425`.
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.rs b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.rs
deleted file mode 100644
index c06ea451b9bf..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.rs
+++ /dev/null
@@ -1,809 +0,0 @@
-//@ check-fail
-// Tests error conditions for specifying subdiagnostics using #[derive(Subdiagnostic)]
-
-// The proc_macro2 crate handles spans differently when on beta/stable release rather than nightly,
-// changing the output of this test. Since Subdiagnostic is strictly internal to the compiler
-// the test is just ignored on stable and beta:
-//@ ignore-stage1
-//@ ignore-beta
-//@ ignore-stable
-
-#![feature(rustc_private)]
-#![crate_type = "lib"]
-
-extern crate rustc_errors;
-extern crate rustc_fluent_macro;
-extern crate rustc_macros;
-extern crate rustc_session;
-extern crate rustc_span;
-extern crate core;
-
-use rustc_errors::{Applicability, DiagMessage, SubdiagMessage};
-use rustc_macros::Subdiagnostic;
-use rustc_span::Span;
-
-rustc_fluent_macro::fluent_messages! { "./example.ftl" }
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct A {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-enum B {
-    #[label(no_crate_example)]
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-    #[label(no_crate_example)]
-    B {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-//~^ ERROR label without `#[primary_span]` field
-struct C {
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label]
-//~^ ERROR diagnostic slug must be first argument
-struct D {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[foo]
-//~^ ERROR `#[foo]` is not a valid attribute
-//~^^ ERROR cannot find attribute `foo` in this scope
-struct E {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label = "..."]
-//~^ ERROR `#[label = ...]` is not a valid attribute
-struct F {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(bug = "...")]
-//~^ ERROR no nested attribute expected here
-//~| ERROR diagnostic slug must be first argument
-struct G {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(slug = 4)]
-//~^ ERROR no nested attribute expected here
-//~| ERROR diagnostic slug must be first argument
-struct J {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(slug("..."))]
-//~^ ERROR no nested attribute expected here
-//~| ERROR diagnostic slug must be first argument
-struct K {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label()]
-//~^ ERROR diagnostic slug must be first argument of a `#[label(...)]` attribute
-struct M {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example, code = "...")]
-//~^ ERROR no nested attribute expected here
-struct N {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example, applicability = "machine-applicable")]
-//~^ ERROR no nested attribute expected here
-struct O {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[foo]
-//~^ ERROR cannot find attribute `foo` in this scope
-//~^^ ERROR unsupported type attribute for subdiagnostic enum
-enum P {
-    #[label(no_crate_example)]
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum Q {
-    #[bar]
-    //~^ ERROR `#[bar]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum R {
-    #[bar = "..."]
-    //~^ ERROR `#[bar = ...]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum S {
-    #[bar = 4]
-    //~^ ERROR `#[bar = ...]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum T {
-    #[bar("...")]
-    //~^ ERROR `#[bar(...)]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum U {
-    #[label(code = "...")]
-    //~^ ERROR diagnostic slug must be first argument of a `#[label(...)]` attribute
-    //~| ERROR no nested attribute expected here
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum V {
-    #[label(no_crate_example)]
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-    B {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-//~^ ERROR label without `#[primary_span]` field
-struct W {
-    #[primary_span]
-    //~^ ERROR the `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-    span: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct X {
-    #[primary_span]
-    span: Span,
-    #[applicability]
-    //~^ ERROR `#[applicability]` is only valid on suggestions
-    applicability: Applicability,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct Y {
-    #[primary_span]
-    span: Span,
-    #[bar]
-    //~^ ERROR `#[bar]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    bar: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct Z {
-    #[primary_span]
-    span: Span,
-    #[bar = "..."]
-    //~^ ERROR `#[bar = ...]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    bar: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct AA {
-    #[primary_span]
-    span: Span,
-    #[bar("...")]
-    //~^ ERROR `#[bar(...)]` is not a valid attribute
-    //~^^ ERROR cannot find attribute `bar` in this scope
-    bar: String,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct AB {
-    #[primary_span]
-    span: Span,
-    #[skip_arg]
-    z: Z,
-}
-
-#[derive(Subdiagnostic)]
-union AC {
-    //~^ ERROR unexpected unsupported untagged union
-    span: u32,
-    b: u64,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-#[label(no_crate_example)]
-struct AD {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example, no_crate::example)]
-//~^ ERROR a diagnostic slug must be the first argument to the attribute
-struct AE {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct AF {
-    #[primary_span]
-    //~^ NOTE previously specified here
-    span_a: Span,
-    #[primary_span]
-    //~^ ERROR specified multiple times
-    span_b: Span,
-}
-
-#[derive(Subdiagnostic)]
-struct AG {
-    //~^ ERROR subdiagnostic kind not specified
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...")]
-struct AH {
-    #[primary_span]
-    span: Span,
-    #[applicability]
-    applicability: Applicability,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-enum AI {
-    #[suggestion(no_crate_example, code = "...")]
-    A {
-        #[primary_span]
-        span: Span,
-        #[applicability]
-        applicability: Applicability,
-        var: String,
-    },
-    #[suggestion(no_crate_example, code = "...")]
-    B {
-        #[primary_span]
-        span: Span,
-        #[applicability]
-        applicability: Applicability,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...", code = "...")]
-//~^ ERROR specified multiple times
-//~^^ NOTE previously specified here
-struct AJ {
-    #[primary_span]
-    span: Span,
-    #[applicability]
-    applicability: Applicability,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...")]
-struct AK {
-    #[primary_span]
-    span: Span,
-    #[applicability]
-    //~^ NOTE previously specified here
-    applicability_a: Applicability,
-    #[applicability]
-    //~^ ERROR specified multiple times
-    applicability_b: Applicability,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...")]
-struct AL {
-    #[primary_span]
-    span: Span,
-    #[applicability]
-    //~^ ERROR the `#[applicability]` attribute can only be applied to fields of type `Applicability`
-    applicability: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...")]
-struct AM {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example)]
-//~^ ERROR suggestion without `code = "..."`
-struct AN {
-    #[primary_span]
-    span: Span,
-    #[applicability]
-    applicability: Applicability,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...", applicability = "foo")]
-//~^ ERROR invalid applicability
-struct AO {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[help(no_crate_example)]
-struct AP {
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[note(no_crate_example)]
-struct AQ;
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...")]
-//~^ ERROR suggestion without `#[primary_span]` field
-struct AR {
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...", applicability = "machine-applicable")]
-struct AS {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[label]
-//~^ ERROR unsupported type attribute for subdiagnostic enum
-enum AT {
-    #[label(no_crate_example)]
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "{var}", applicability = "machine-applicable")]
-struct AU {
-    #[primary_span]
-    span: Span,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "{var}", applicability = "machine-applicable")]
-//~^ ERROR `var` doesn't refer to a field on this type
-struct AV {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-enum AW {
-    #[suggestion(no_crate_example, code = "{var}", applicability = "machine-applicable")]
-    A {
-        #[primary_span]
-        span: Span,
-        var: String,
-    },
-}
-
-#[derive(Subdiagnostic)]
-enum AX {
-    #[suggestion(no_crate_example, code = "{var}", applicability = "machine-applicable")]
-    //~^ ERROR `var` doesn't refer to a field on this type
-    A {
-        #[primary_span]
-        span: Span,
-    },
-}
-
-#[derive(Subdiagnostic)]
-#[warning(no_crate_example)]
-struct AY {}
-
-#[derive(Subdiagnostic)]
-#[warning(no_crate_example)]
-struct AZ {
-    #[primary_span]
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "...")]
-//~^ ERROR suggestion without `#[primary_span]` field
-struct BA {
-    #[suggestion_part]
-    //~^ ERROR `#[suggestion_part]` is not a valid attribute
-    span: Span,
-    #[suggestion_part(code = "...")]
-    //~^ ERROR `#[suggestion_part(...)]` is not a valid attribute
-    span2: Span,
-    #[applicability]
-    applicability: Applicability,
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, code = "...", applicability = "machine-applicable")]
-//~^ ERROR multipart suggestion without any `#[suggestion_part(...)]` fields
-//~| ERROR invalid nested attribute
-struct BBa {
-    var: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, applicability = "machine-applicable")]
-struct BBb {
-    #[suggestion_part]
-    //~^ ERROR `#[suggestion_part(...)]` attribute without `code = "..."`
-    span1: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, applicability = "machine-applicable")]
-struct BBc {
-    #[suggestion_part()]
-    //~^ ERROR `#[suggestion_part(...)]` attribute without `code = "..."`
-    span1: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-//~^ ERROR multipart suggestion without any `#[suggestion_part(...)]` fields
-struct BC {
-    #[primary_span]
-    //~^ ERROR `#[primary_span]` is not a valid attribute
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BD {
-    #[suggestion_part]
-    //~^ ERROR `#[suggestion_part(...)]` attribute without `code = "..."`
-    span1: Span,
-    #[suggestion_part()]
-    //~^ ERROR `#[suggestion_part(...)]` attribute without `code = "..."`
-    span2: Span,
-    #[suggestion_part(foo = "bar")]
-    //~^ ERROR `code` is the only valid nested attribute
-    //~| ERROR expected `,`
-    span4: Span,
-    #[suggestion_part(code = "...")]
-    //~^ ERROR the `#[suggestion_part(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-    s1: String,
-    #[suggestion_part()]
-    //~^ ERROR the `#[suggestion_part(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-    s2: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, applicability = "machine-applicable")]
-struct BE {
-    #[suggestion_part(code = "...", code = ",,,")]
-    //~^ ERROR specified multiple times
-    //~| NOTE previously specified here
-    span: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, applicability = "machine-applicable")]
-struct BF {
-    #[suggestion_part(code = "(")]
-    first: Span,
-    #[suggestion_part(code = ")")]
-    second: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BG {
-    #[applicability]
-    appl: Applicability,
-    #[suggestion_part(code = "(")]
-    first: Span,
-    #[suggestion_part(code = ")")]
-    second: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, applicability = "machine-applicable")]
-struct BH {
-    #[applicability]
-    //~^ ERROR `#[applicability]` has no effect
-    appl: Applicability,
-    #[suggestion_part(code = "(")]
-    first: Span,
-    #[suggestion_part(code = ")")]
-    second: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example, applicability = "machine-applicable")]
-struct BI {
-    #[suggestion_part(code = "")]
-    spans: Vec,
-}
-
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct BJ {
-    #[primary_span]
-    span: Span,
-    r#type: String,
-}
-
-/// with a doc comment on the type..
-#[derive(Subdiagnostic)]
-#[label(no_crate_example)]
-struct BK {
-    /// ..and the field
-    #[primary_span]
-    span: Span,
-}
-
-/// with a doc comment on the type..
-#[derive(Subdiagnostic)]
-enum BL {
-    /// ..and the variant..
-    #[label(no_crate_example)]
-    Foo {
-        /// ..and the field
-        #[primary_span]
-        span: Span,
-    },
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BM {
-    #[suggestion_part(code("foo"))]
-    //~^ ERROR expected exactly one string literal for `code = ...`
-    //~| ERROR unexpected token, expected `)`
-    span: Span,
-    r#type: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BN {
-    #[suggestion_part(code("foo", "bar"))]
-    //~^ ERROR expected exactly one string literal for `code = ...`
-    //~| ERROR unexpected token, expected `)`
-    span: Span,
-    r#type: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BO {
-    #[suggestion_part(code(3))]
-    //~^ ERROR expected exactly one string literal for `code = ...`
-    //~| ERROR unexpected token, expected `)`
-    span: Span,
-    r#type: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BP {
-    #[suggestion_part(code())]
-    //~^ ERROR expected exactly one string literal for `code = ...`
-    span: Span,
-    r#type: String,
-}
-
-#[derive(Subdiagnostic)]
-#[multipart_suggestion(no_crate_example)]
-struct BQ {
-    #[suggestion_part(code = 3)]
-    //~^ ERROR expected string literal
-    span: Span,
-    r#type: String,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "")]
-struct SuggestionStyleDefault {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = "short")]
-struct SuggestionStyleShort {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = "hidden")]
-struct SuggestionStyleHidden {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = "verbose")]
-struct SuggestionStyleVerbose {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = "tool-only")]
-struct SuggestionStyleToolOnly {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = "hidden", style = "normal")]
-//~^ ERROR specified multiple times
-//~| NOTE previously specified here
-struct SuggestionStyleTwice {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion_hidden(no_crate_example, code = "")]
-//~^ ERROR #[suggestion_hidden(...)]` is not a valid attribute
-struct SuggestionStyleOldSyntax {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion_hidden(no_crate_example, code = "", style = "normal")]
-//~^ ERROR #[suggestion_hidden(...)]` is not a valid attribute
-struct SuggestionStyleOldAndNewSyntax {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = "foo")]
-//~^ ERROR invalid suggestion style
-struct SuggestionStyleInvalid1 {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style = 42)]
-//~^ ERROR expected string literal
-struct SuggestionStyleInvalid2 {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style)]
-//~^ ERROR a diagnostic slug must be the first argument to the attribute
-struct SuggestionStyleInvalid3 {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "", style("foo"))]
-//~^ ERROR expected `=`
-struct SuggestionStyleInvalid4 {
-    #[primary_span]
-    sub: Span,
-}
-
-#[derive(Subdiagnostic)]
-#[suggestion(no_crate_example, code = "")]
-//~^ ERROR suggestion without `#[primary_span]` field
-struct PrimarySpanOnVec {
-    #[primary_span]
-    //~^ ERROR `#[primary_span]` is not a valid attribute
-    //~| NOTE there must be exactly one primary span
-    sub: Vec,
-}
-
-#[derive(Subdiagnostic)]
-struct NestedParent {
-    #[subdiagnostic]
-    single_sub: A,
-    #[subdiagnostic]
-    option_sub: Option,
-    #[subdiagnostic]
-    vec_sub: Vec,
-}
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.stderr b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.stderr
deleted file mode 100644
index 9f18f7ffabcc..000000000000
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive.stderr
+++ /dev/null
@@ -1,549 +0,0 @@
-error: derive(Diagnostic): label without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive.rs:52:1
-   |
-LL | #[label(no_crate_example)]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive.rs:59:1
-   |
-LL | #[label]
-   | ^
-
-error: derive(Diagnostic): `#[foo]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:68:1
-   |
-LL | #[foo]
-   | ^
-
-error: derive(Diagnostic): `#[label = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:78:1
-   |
-LL | #[label = "..."]
-   | ^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive.rs:87:9
-   |
-LL | #[label(bug = "...")]
-   |         ^^^
-
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive.rs:87:1
-   |
-LL | #[label(bug = "...")]
-   | ^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive.rs:97:9
-   |
-LL | #[label(slug = 4)]
-   |         ^^^^
-
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive.rs:97:1
-   |
-LL | #[label(slug = 4)]
-   | ^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive.rs:107:9
-   |
-LL | #[label(slug("..."))]
-   |         ^^^^
-
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive.rs:107:1
-   |
-LL | #[label(slug("..."))]
-   | ^
-
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive.rs:117:1
-   |
-LL | #[label()]
-   | ^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive.rs:126:27
-   |
-LL | #[label(no_crate_example, code = "...")]
-   |                           ^^^^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive.rs:135:27
-   |
-LL | #[label(no_crate_example, applicability = "machine-applicable")]
-   |                           ^^^^^^^^^^^^^
-
-error: derive(Diagnostic): unsupported type attribute for subdiagnostic enum
-  --> $DIR/subdiagnostic-derive.rs:144:1
-   |
-LL | #[foo]
-   | ^
-
-error: derive(Diagnostic): `#[bar]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:158:5
-   |
-LL |     #[bar]
-   |     ^
-
-error: derive(Diagnostic): `#[bar = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:170:5
-   |
-LL |     #[bar = "..."]
-   |     ^
-
-error: derive(Diagnostic): `#[bar = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:182:5
-   |
-LL |     #[bar = 4]
-   |     ^
-
-error: derive(Diagnostic): `#[bar(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:194:5
-   |
-LL |     #[bar("...")]
-   |     ^
-
-error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive.rs:206:13
-   |
-LL |     #[label(code = "...")]
-   |             ^^^^
-
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive.rs:206:5
-   |
-LL |     #[label(code = "...")]
-   |     ^
-
-error: derive(Diagnostic): the `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/subdiagnostic-derive.rs:235:5
-   |
-LL |     #[primary_span]
-   |     ^
-
-error: derive(Diagnostic): label without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive.rs:232:1
-   |
-LL | #[label(no_crate_example)]
-   | ^
-
-error: derive(Diagnostic): `#[applicability]` is only valid on suggestions
-  --> $DIR/subdiagnostic-derive.rs:245:5
-   |
-LL |     #[applicability]
-   |     ^
-
-error: derive(Diagnostic): `#[bar]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:255:5
-   |
-LL |     #[bar]
-   |     ^
-   |
-   = help: only `primary_span`, `applicability` and `skip_arg` are valid field attributes
-
-error: derive(Diagnostic): `#[bar = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:266:5
-   |
-LL |     #[bar = "..."]
-   |     ^
-
-error: derive(Diagnostic): `#[bar(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:277:5
-   |
-LL |     #[bar("...")]
-   |     ^
-   |
-   = help: only `primary_span`, `applicability` and `skip_arg` are valid field attributes
-
-error: unexpected unsupported untagged union
-  --> $DIR/subdiagnostic-derive.rs:293:1
-   |
-LL | / union AC {
-LL | |
-LL | |     span: u32,
-LL | |     b: u64,
-LL | | }
-   | |_^
-
-error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/subdiagnostic-derive.rs:308:27
-   |
-LL | #[label(no_crate_example, no_crate::example)]
-   |                           ^^^^^^^^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive.rs:321:5
-   |
-LL |     #[primary_span]
-   |     ^
-   |
-note: previously specified here
-  --> $DIR/subdiagnostic-derive.rs:318:5
-   |
-LL |     #[primary_span]
-   |     ^
-
-error: derive(Diagnostic): subdiagnostic kind not specified
-  --> $DIR/subdiagnostic-derive.rs:327:8
-   |
-LL | struct AG {
-   |        ^^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive.rs:364:46
-   |
-LL | #[suggestion(no_crate_example, code = "...", code = "...")]
-   |                                              ^^^^
-   |
-note: previously specified here
-  --> $DIR/subdiagnostic-derive.rs:364:32
-   |
-LL | #[suggestion(no_crate_example, code = "...", code = "...")]
-   |                                ^^^^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive.rs:382:5
-   |
-LL |     #[applicability]
-   |     ^
-   |
-note: previously specified here
-  --> $DIR/subdiagnostic-derive.rs:379:5
-   |
-LL |     #[applicability]
-   |     ^
-
-error: derive(Diagnostic): the `#[applicability]` attribute can only be applied to fields of type `Applicability`
-  --> $DIR/subdiagnostic-derive.rs:392:5
-   |
-LL |     #[applicability]
-   |     ^
-
-error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/subdiagnostic-derive.rs:405:1
-   |
-LL | #[suggestion(no_crate_example)]
-   | ^
-
-error: derive(Diagnostic): invalid applicability
-  --> $DIR/subdiagnostic-derive.rs:415:62
-   |
-LL | #[suggestion(no_crate_example, code = "...", applicability = "foo")]
-   |                                                              ^^^^^
-
-error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive.rs:433:1
-   |
-LL | #[suggestion(no_crate_example, code = "...")]
-   | ^
-
-error: derive(Diagnostic): unsupported type attribute for subdiagnostic enum
-  --> $DIR/subdiagnostic-derive.rs:447:1
-   |
-LL | #[label]
-   | ^
-
-error: derive(Diagnostic): `var` doesn't refer to a field on this type
-  --> $DIR/subdiagnostic-derive.rs:467:39
-   |
-LL | #[suggestion(no_crate_example, code = "{var}", applicability = "machine-applicable")]
-   |                                       ^^^^^^^
-
-error: derive(Diagnostic): `var` doesn't refer to a field on this type
-  --> $DIR/subdiagnostic-derive.rs:486:43
-   |
-LL |     #[suggestion(no_crate_example, code = "{var}", applicability = "machine-applicable")]
-   |                                           ^^^^^^^
-
-error: derive(Diagnostic): `#[suggestion_part]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:509:5
-   |
-LL |     #[suggestion_part]
-   |     ^
-   |
-   = help: `#[suggestion_part(...)]` is only valid in multipart suggestions, use `#[primary_span]` instead
-
-error: derive(Diagnostic): `#[suggestion_part(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:512:5
-   |
-LL |     #[suggestion_part(code = "...")]
-   |     ^
-   |
-   = help: `#[suggestion_part(...)]` is only valid in multipart suggestions
-
-error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive.rs:506:1
-   |
-LL | #[suggestion(no_crate_example, code = "...")]
-   | ^
-
-error: derive(Diagnostic): invalid nested attribute
-  --> $DIR/subdiagnostic-derive.rs:521:42
-   |
-LL | #[multipart_suggestion(no_crate_example, code = "...", applicability = "machine-applicable")]
-   |                                          ^^^^
-   |
-   = help: only `style` and `applicability` are valid nested attributes
-
-error: derive(Diagnostic): multipart suggestion without any `#[suggestion_part(...)]` fields
-  --> $DIR/subdiagnostic-derive.rs:521:1
-   |
-LL | #[multipart_suggestion(no_crate_example, code = "...", applicability = "machine-applicable")]
-   | ^
-
-error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive.rs:531:5
-   |
-LL |     #[suggestion_part]
-   |     ^
-
-error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive.rs:539:5
-   |
-LL |     #[suggestion_part()]
-   |     ^
-
-error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:548:5
-   |
-LL |     #[primary_span]
-   |     ^
-   |
-   = help: multipart suggestions use one or more `#[suggestion_part]`s rather than one `#[primary_span]`
-
-error: derive(Diagnostic): multipart suggestion without any `#[suggestion_part(...)]` fields
-  --> $DIR/subdiagnostic-derive.rs:545:1
-   |
-LL | #[multipart_suggestion(no_crate_example)]
-   | ^
-
-error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive.rs:556:5
-   |
-LL |     #[suggestion_part]
-   |     ^
-
-error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive.rs:559:5
-   |
-LL |     #[suggestion_part()]
-   |     ^
-
-error: derive(Diagnostic): `code` is the only valid nested attribute
-  --> $DIR/subdiagnostic-derive.rs:562:23
-   |
-LL |     #[suggestion_part(foo = "bar")]
-   |                       ^^^
-
-error: derive(Diagnostic): the `#[suggestion_part(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/subdiagnostic-derive.rs:566:5
-   |
-LL |     #[suggestion_part(code = "...")]
-   |     ^
-
-error: derive(Diagnostic): the `#[suggestion_part(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/subdiagnostic-derive.rs:569:5
-   |
-LL |     #[suggestion_part()]
-   |     ^
-
-error: expected `,`
-  --> $DIR/subdiagnostic-derive.rs:562:27
-   |
-LL |     #[suggestion_part(foo = "bar")]
-   |                           ^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive.rs:577:37
-   |
-LL |     #[suggestion_part(code = "...", code = ",,,")]
-   |                                     ^^^^
-   |
-note: previously specified here
-  --> $DIR/subdiagnostic-derive.rs:577:23
-   |
-LL |     #[suggestion_part(code = "...", code = ",,,")]
-   |                       ^^^^
-
-error: derive(Diagnostic): `#[applicability]` has no effect if all `#[suggestion]`/`#[multipart_suggestion]` attributes have a static `applicability = "..."`
-  --> $DIR/subdiagnostic-derive.rs:606:5
-   |
-LL |     #[applicability]
-   |     ^
-
-error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive.rs:654:28
-   |
-LL |     #[suggestion_part(code("foo"))]
-   |                            ^^^^^
-
-error: unexpected token, expected `)`
-  --> $DIR/subdiagnostic-derive.rs:654:28
-   |
-LL |     #[suggestion_part(code("foo"))]
-   |                            ^^^^^
-
-error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive.rs:664:28
-   |
-LL |     #[suggestion_part(code("foo", "bar"))]
-   |                            ^^^^^
-
-error: unexpected token, expected `)`
-  --> $DIR/subdiagnostic-derive.rs:664:28
-   |
-LL |     #[suggestion_part(code("foo", "bar"))]
-   |                            ^^^^^
-
-error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive.rs:674:28
-   |
-LL |     #[suggestion_part(code(3))]
-   |                            ^
-
-error: unexpected token, expected `)`
-  --> $DIR/subdiagnostic-derive.rs:674:28
-   |
-LL |     #[suggestion_part(code(3))]
-   |                            ^
-
-error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive.rs:684:28
-   |
-LL |     #[suggestion_part(code())]
-   |                            ^
-
-error: expected string literal
-  --> $DIR/subdiagnostic-derive.rs:693:30
-   |
-LL |     #[suggestion_part(code = 3)]
-   |                              ^
-
-error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive.rs:735:1
-   |
-LL | #[suggestion(no_crate_example, code = "", style = "hidden", style = "normal")]
-   | ^
-   |
-note: previously specified here
-  --> $DIR/subdiagnostic-derive.rs:735:1
-   |
-LL | #[suggestion(no_crate_example, code = "", style = "hidden", style = "normal")]
-   | ^
-
-error: derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:744:1
-   |
-LL | #[suggestion_hidden(no_crate_example, code = "")]
-   | ^
-   |
-   = help: Use `#[suggestion(..., style = "hidden")]` instead
-
-error: derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:752:1
-   |
-LL | #[suggestion_hidden(no_crate_example, code = "", style = "normal")]
-   | ^
-   |
-   = help: Use `#[suggestion(..., style = "hidden")]` instead
-
-error: derive(Diagnostic): invalid suggestion style
-  --> $DIR/subdiagnostic-derive.rs:760:51
-   |
-LL | #[suggestion(no_crate_example, code = "", style = "foo")]
-   |                                                   ^^^^^
-   |
-   = help: valid styles are `normal`, `short`, `hidden`, `verbose` and `tool-only`
-
-error: expected string literal
-  --> $DIR/subdiagnostic-derive.rs:768:51
-   |
-LL | #[suggestion(no_crate_example, code = "", style = 42)]
-   |                                                   ^^
-
-error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/subdiagnostic-derive.rs:776:43
-   |
-LL | #[suggestion(no_crate_example, code = "", style)]
-   |                                           ^^^^^
-
-error: expected `=`
-  --> $DIR/subdiagnostic-derive.rs:784:48
-   |
-LL | #[suggestion(no_crate_example, code = "", style("foo"))]
-   |                                                ^
-
-error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive.rs:795:5
-   |
-LL |     #[primary_span]
-   |     ^
-   |
-   = note: there must be exactly one primary span
-   = help: to create a suggestion with multiple spans, use `#[multipart_suggestion]` instead
-
-error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive.rs:792:1
-   |
-LL | #[suggestion(no_crate_example, code = "")]
-   | ^
-
-error: cannot find attribute `foo` in this scope
-  --> $DIR/subdiagnostic-derive.rs:68:3
-   |
-LL | #[foo]
-   |   ^^^
-
-error: cannot find attribute `foo` in this scope
-  --> $DIR/subdiagnostic-derive.rs:144:3
-   |
-LL | #[foo]
-   |   ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:158:7
-   |
-LL |     #[bar]
-   |       ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:170:7
-   |
-LL |     #[bar = "..."]
-   |       ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:182:7
-   |
-LL |     #[bar = 4]
-   |       ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:194:7
-   |
-LL |     #[bar("...")]
-   |       ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:255:7
-   |
-LL |     #[bar]
-   |       ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:266:7
-   |
-LL |     #[bar = "..."]
-   |       ^^^
-
-error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive.rs:277:7
-   |
-LL |     #[bar("...")]
-   |       ^^^
-
-error: aborting due to 82 previous errors
-

From 580c8d3c20ca3b363ece4ec34a9db1194bb9940d Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sat, 7 Feb 2026 15:09:34 +0100
Subject: [PATCH 1425/1843] Update remaining session-diagnostics tests

---
 .../diagnostic-derive-doc-comment-field.rs    |   7 +-
 ...diagnostic-derive-doc-comment-field.stderr |   8 +-
 .../diagnostic-derive-inline.rs               |   1 -
 .../diagnostic-derive-inline.stderr           | 174 +++++++++---------
 .../subdiagnostic-derive-inline.rs            |   1 -
 .../subdiagnostic-derive-inline.stderr        | 174 +++++++++---------
 6 files changed, 180 insertions(+), 185 deletions(-)

diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs
index 4f08c5327c15..e63ab03cda47 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs
@@ -14,7 +14,6 @@
 #![crate_type = "lib"]
 
 extern crate rustc_errors;
-extern crate rustc_fluent_macro;
 extern crate rustc_macros;
 extern crate rustc_session;
 extern crate rustc_span;
@@ -24,12 +23,10 @@ use rustc_errors::{Applicability, DiagMessage, SubdiagMessage};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::Span;
 
-rustc_fluent_macro::fluent_messages! { "./example.ftl" }
-
 struct NotIntoDiagArg;
 
 #[derive(Diagnostic)]
-#[diag(no_crate_example)]
+#[diag("example message")]
 struct Test {
     #[primary_span]
     span: Span,
@@ -39,7 +36,7 @@ struct Test {
 }
 
 #[derive(Subdiagnostic)]
-#[label(no_crate_example)]
+#[label("example message")]
 struct SubTest {
     #[primary_span]
     span: Span,
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.stderr b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.stderr
index 0b00e098f6d4..2d90b2a96b01 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.stderr
@@ -1,5 +1,5 @@
 error[E0277]: the trait bound `NotIntoDiagArg: IntoDiagArg` is not satisfied
-  --> $DIR/diagnostic-derive-doc-comment-field.rs:37:10
+  --> $DIR/diagnostic-derive-doc-comment-field.rs:34:10
    |
 LL | #[derive(Diagnostic)]
    |          ---------- required by a bound introduced by this call
@@ -8,7 +8,7 @@ LL |     arg: NotIntoDiagArg,
    |          ^^^^^^^^^^^^^^ unsatisfied trait bound
    |
 help: the nightly-only, unstable trait `IntoDiagArg` is not implemented for `NotIntoDiagArg`
-  --> $DIR/diagnostic-derive-doc-comment-field.rs:29:1
+  --> $DIR/diagnostic-derive-doc-comment-field.rs:26:1
    |
 LL | struct NotIntoDiagArg;
    | ^^^^^^^^^^^^^^^^^^^^^
@@ -21,7 +21,7 @@ note: required by a bound in `Diag::<'a, G>::arg`
    = note: this error originates in the macro `with_fn` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error[E0277]: the trait bound `NotIntoDiagArg: IntoDiagArg` is not satisfied
-  --> $DIR/diagnostic-derive-doc-comment-field.rs:47:10
+  --> $DIR/diagnostic-derive-doc-comment-field.rs:44:10
    |
 LL | #[derive(Subdiagnostic)]
    |          ------------- required by a bound introduced by this call
@@ -30,7 +30,7 @@ LL |     arg: NotIntoDiagArg,
    |          ^^^^^^^^^^^^^^ unsatisfied trait bound
    |
 help: the nightly-only, unstable trait `IntoDiagArg` is not implemented for `NotIntoDiagArg`
-  --> $DIR/diagnostic-derive-doc-comment-field.rs:29:1
+  --> $DIR/diagnostic-derive-doc-comment-field.rs:26:1
    |
 LL | struct NotIntoDiagArg;
    | ^^^^^^^^^^^^^^^^^^^^^
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
index babe3813e40b..1d2bfc66aaff 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
@@ -19,7 +19,6 @@ extern crate rustc_span;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
-extern crate rustc_fluent_macro;
 extern crate rustc_macros;
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr
index 2ba307940280..71ed77804a6c 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr
@@ -1,11 +1,11 @@
 error: derive(Diagnostic): unsupported type attribute for diagnostic derive enum
-  --> $DIR/diagnostic-derive-inline.rs:45:1
+  --> $DIR/diagnostic-derive-inline.rs:44:1
    |
 LL | #[diag("this is an example message", code = E0123)]
    | ^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:48:5
+  --> $DIR/diagnostic-derive-inline.rs:47:5
    |
 LL |     Foo,
    |     ^^^
@@ -13,7 +13,7 @@ LL |     Foo,
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:50:5
+  --> $DIR/diagnostic-derive-inline.rs:49:5
    |
 LL |     Bar,
    |     ^^^
@@ -21,19 +21,19 @@ LL |     Bar,
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: expected parentheses: #[diag(...)]
-  --> $DIR/diagnostic-derive-inline.rs:56:8
+  --> $DIR/diagnostic-derive-inline.rs:55:8
    |
 LL | #[diag = "E0123"]
    |        ^
 
 error: derive(Diagnostic): `#[nonsense(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:61:1
+  --> $DIR/diagnostic-derive-inline.rs:60:1
    |
 LL | #[nonsense("this is an example message", code = E0123)]
    | ^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:61:1
+  --> $DIR/diagnostic-derive-inline.rs:60:1
    |
 LL | #[nonsense("this is an example message", code = E0123)]
    | ^
@@ -41,7 +41,7 @@ LL | #[nonsense("this is an example message", code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:68:1
+  --> $DIR/diagnostic-derive-inline.rs:67:1
    |
 LL | #[diag(code = E0123)]
    | ^
@@ -49,7 +49,7 @@ LL | #[diag(code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:73:1
+  --> $DIR/diagnostic-derive-inline.rs:72:1
    |
 LL | #[diag(nonsense("foo"), code = E0123, slug = "foo")]
    | ^
@@ -57,7 +57,7 @@ LL | #[diag(nonsense("foo"), code = E0123, slug = "foo")]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:78:1
+  --> $DIR/diagnostic-derive-inline.rs:77:1
    |
 LL | #[diag(nonsense = "...", code = E0123, slug = "foo")]
    | ^
@@ -65,7 +65,7 @@ LL | #[diag(nonsense = "...", code = E0123, slug = "foo")]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:83:1
+  --> $DIR/diagnostic-derive-inline.rs:82:1
    |
 LL | #[diag(nonsense = 4, code = E0123, slug = "foo")]
    | ^
@@ -73,7 +73,7 @@ LL | #[diag(nonsense = 4, code = E0123, slug = "foo")]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): unknown argument
-  --> $DIR/diagnostic-derive-inline.rs:88:52
+  --> $DIR/diagnostic-derive-inline.rs:87:52
    |
 LL | #[diag("this is an example message", code = E0123, slug = "foo")]
    |                                                    ^^^^
@@ -81,43 +81,43 @@ LL | #[diag("this is an example message", code = E0123, slug = "foo")]
    = note: only the `code` parameter is valid after the slug
 
 error: derive(Diagnostic): `#[suggestion = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:95:5
+  --> $DIR/diagnostic-derive-inline.rs:94:5
    |
 LL |     #[suggestion = "bar"]
    |     ^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:102:38
+  --> $DIR/diagnostic-derive-inline.rs:101:38
    |
 LL | #[diag("this is an example message", code = E0456)]
    |                                      ^^^^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:101:38
+  --> $DIR/diagnostic-derive-inline.rs:100:38
    |
 LL | #[diag("this is an example message", code = E0123)]
    |                                      ^^^^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:107:52
+  --> $DIR/diagnostic-derive-inline.rs:106:52
    |
 LL | #[diag("this is an example message", code = E0123, code = E0456)]
    |                                                    ^^^^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:107:38
+  --> $DIR/diagnostic-derive-inline.rs:106:38
    |
 LL | #[diag("this is an example message", code = E0123, code = E0456)]
    |                                      ^^^^
 
 error: derive(Diagnostic): diagnostic slug must be the first argument
-  --> $DIR/diagnostic-derive-inline.rs:112:38
+  --> $DIR/diagnostic-derive-inline.rs:111:38
    |
 LL | #[diag("this is an example message", no_crate::example, code = E0123)]
    |                                      ^^^^^^^^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:117:1
+  --> $DIR/diagnostic-derive-inline.rs:116:1
    |
 LL | struct KindNotProvided {}
    | ^^^^^^
@@ -125,7 +125,7 @@ LL | struct KindNotProvided {}
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:120:1
+  --> $DIR/diagnostic-derive-inline.rs:119:1
    |
 LL | #[diag(code = E0123)]
    | ^
@@ -133,31 +133,31 @@ LL | #[diag(code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): the `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/diagnostic-derive-inline.rs:131:5
+  --> $DIR/diagnostic-derive-inline.rs:130:5
    |
 LL |     #[primary_span]
    |     ^
 
 error: derive(Diagnostic): `#[nonsense]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:139:5
+  --> $DIR/diagnostic-derive-inline.rs:138:5
    |
 LL |     #[nonsense]
    |     ^
 
 error: derive(Diagnostic): the `#[label(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/diagnostic-derive-inline.rs:156:5
+  --> $DIR/diagnostic-derive-inline.rs:155:5
    |
 LL |     #[label("with a label")]
    |     ^
 
 error: derive(Diagnostic): `name` doesn't refer to a field on this type
-  --> $DIR/diagnostic-derive-inline.rs:164:46
+  --> $DIR/diagnostic-derive-inline.rs:163:46
    |
 LL |     #[suggestion("with a suggestion", code = "{name}")]
    |                                              ^^^^^^^^
 
 error: invalid format string: expected `}` but string was terminated
-  --> $DIR/diagnostic-derive-inline.rs:169:10
+  --> $DIR/diagnostic-derive-inline.rs:168:10
    |
 LL | #[derive(Diagnostic)]
    |          ^^^^^^^^^^ expected `}` in format string
@@ -166,7 +166,7 @@ LL | #[derive(Diagnostic)]
    = note: this error originates in the derive macro `Diagnostic` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: invalid format string: unmatched `}` found
-  --> $DIR/diagnostic-derive-inline.rs:179:10
+  --> $DIR/diagnostic-derive-inline.rs:178:10
    |
 LL | #[derive(Diagnostic)]
    |          ^^^^^^^^^^ unmatched `}` in format string
@@ -175,19 +175,19 @@ LL | #[derive(Diagnostic)]
    = note: this error originates in the derive macro `Diagnostic` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: derive(Diagnostic): the `#[label(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/diagnostic-derive-inline.rs:199:5
+  --> $DIR/diagnostic-derive-inline.rs:198:5
    |
 LL |     #[label("with a label")]
    |     ^
 
 error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive-inline.rs:218:5
+  --> $DIR/diagnostic-derive-inline.rs:217:5
    |
 LL |     #[suggestion("with a suggestion")]
    |     ^
 
 error: derive(Diagnostic): invalid nested attribute
-  --> $DIR/diagnostic-derive-inline.rs:226:39
+  --> $DIR/diagnostic-derive-inline.rs:225:39
    |
 LL |     #[suggestion("with a suggestion", nonsense = "bar")]
    |                                       ^^^^^^^^
@@ -195,13 +195,13 @@ LL |     #[suggestion("with a suggestion", nonsense = "bar")]
    = help: only `style`, `code` and `applicability` are valid nested attributes
 
 error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive-inline.rs:226:5
+  --> $DIR/diagnostic-derive-inline.rs:225:5
    |
 LL |     #[suggestion("with a suggestion", nonsense = "bar")]
    |     ^
 
 error: derive(Diagnostic): invalid nested attribute
-  --> $DIR/diagnostic-derive-inline.rs:235:39
+  --> $DIR/diagnostic-derive-inline.rs:234:39
    |
 LL |     #[suggestion("with a suggestion", msg = "bar")]
    |                                       ^^^
@@ -209,13 +209,13 @@ LL |     #[suggestion("with a suggestion", msg = "bar")]
    = help: only `style`, `code` and `applicability` are valid nested attributes
 
 error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive-inline.rs:235:5
+  --> $DIR/diagnostic-derive-inline.rs:234:5
    |
 LL |     #[suggestion("with a suggestion", msg = "bar")]
    |     ^
 
 error: derive(Diagnostic): wrong field type for suggestion
-  --> $DIR/diagnostic-derive-inline.rs:258:5
+  --> $DIR/diagnostic-derive-inline.rs:257:5
    |
 LL |     #[suggestion("with a suggestion", code = "This is suggested code")]
    |     ^
@@ -223,79 +223,79 @@ LL |     #[suggestion("with a suggestion", code = "This is suggested code")]
    = help: `#[suggestion(...)]` should be applied to fields of type `Span` or `(Span, Applicability)`
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:274:24
+  --> $DIR/diagnostic-derive-inline.rs:273:24
    |
 LL |     suggestion: (Span, Span, Applicability),
    |                        ^^^^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:274:18
+  --> $DIR/diagnostic-derive-inline.rs:273:18
    |
 LL |     suggestion: (Span, Span, Applicability),
    |                  ^^^^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:282:33
+  --> $DIR/diagnostic-derive-inline.rs:281:33
    |
 LL |     suggestion: (Applicability, Applicability, Span),
    |                                 ^^^^^^^^^^^^^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:282:18
+  --> $DIR/diagnostic-derive-inline.rs:281:18
    |
 LL |     suggestion: (Applicability, Applicability, Span),
    |                  ^^^^^^^^^^^^^
 
 error: derive(Diagnostic): `#[label = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:289:5
+  --> $DIR/diagnostic-derive-inline.rs:288:5
    |
 LL |     #[label = "bar"]
    |     ^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:390:5
+  --> $DIR/diagnostic-derive-inline.rs:389:5
    |
 LL |     #[suggestion("with a suggestion", code = "...", applicability = "maybe-incorrect")]
    |     ^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:392:24
+  --> $DIR/diagnostic-derive-inline.rs:391:24
    |
 LL |     suggestion: (Span, Applicability),
    |                        ^^^^^^^^^^^^^
 
 error: derive(Diagnostic): invalid applicability
-  --> $DIR/diagnostic-derive-inline.rs:398:69
+  --> $DIR/diagnostic-derive-inline.rs:397:69
    |
 LL |     #[suggestion("with a suggestion", code = "...", applicability = "batman")]
    |                                                                     ^^^^^^^^
 
 error: derive(Diagnostic): the `#[help(...)]` attribute can only be applied to fields of type `Span`, `MultiSpan`, `bool` or `()`
-  --> $DIR/diagnostic-derive-inline.rs:461:5
+  --> $DIR/diagnostic-derive-inline.rs:460:5
    |
 LL |     #[help("with a help")]
    |     ^
 
 error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/diagnostic-derive-inline.rs:470:29
+  --> $DIR/diagnostic-derive-inline.rs:469:29
    |
 LL |     #[label("with a label", foo)]
    |                             ^^^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/diagnostic-derive-inline.rs:478:29
+  --> $DIR/diagnostic-derive-inline.rs:477:29
    |
 LL |     #[label("with a label", foo = "...")]
    |                             ^^^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/diagnostic-derive-inline.rs:486:29
+  --> $DIR/diagnostic-derive-inline.rs:485:29
    |
 LL |     #[label("with a label", foo("..."))]
    |                             ^^^
 
 error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:498:5
+  --> $DIR/diagnostic-derive-inline.rs:497:5
    |
 LL |     #[primary_span]
    |     ^
@@ -303,13 +303,13 @@ LL |     #[primary_span]
    = help: the `primary_span` field attribute is not valid for lint diagnostics
 
 error: derive(Diagnostic): `#[error(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:518:1
+  --> $DIR/diagnostic-derive-inline.rs:517:1
    |
 LL | #[error("this is an example message", code = E0123)]
    | ^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:518:1
+  --> $DIR/diagnostic-derive-inline.rs:517:1
    |
 LL | #[error("this is an example message", code = E0123)]
    | ^
@@ -317,13 +317,13 @@ LL | #[error("this is an example message", code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): `#[warn_(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:525:1
+  --> $DIR/diagnostic-derive-inline.rs:524:1
    |
 LL | #[warn_("this is an example message", code = E0123)]
    | ^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:525:1
+  --> $DIR/diagnostic-derive-inline.rs:524:1
    |
 LL | #[warn_("this is an example message", code = E0123)]
    | ^
@@ -331,13 +331,13 @@ LL | #[warn_("this is an example message", code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): `#[lint(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:532:1
+  --> $DIR/diagnostic-derive-inline.rs:531:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:532:1
+  --> $DIR/diagnostic-derive-inline.rs:531:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
@@ -345,13 +345,13 @@ LL | #[lint("this is an example message", code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): `#[lint(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:539:1
+  --> $DIR/diagnostic-derive-inline.rs:538:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
 
 error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:539:1
+  --> $DIR/diagnostic-derive-inline.rs:538:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
@@ -359,19 +359,19 @@ LL | #[lint("this is an example message", code = E0123)]
    = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:548:53
+  --> $DIR/diagnostic-derive-inline.rs:547:53
    |
 LL |     #[suggestion("with a suggestion", code = "...", code = ",,,")]
    |                                                     ^^^^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:548:39
+  --> $DIR/diagnostic-derive-inline.rs:547:39
    |
 LL |     #[suggestion("with a suggestion", code = "...", code = ",,,")]
    |                                       ^^^^
 
 error: derive(Diagnostic): wrong types for suggestion
-  --> $DIR/diagnostic-derive-inline.rs:557:24
+  --> $DIR/diagnostic-derive-inline.rs:556:24
    |
 LL |     suggestion: (Span, usize),
    |                        ^^^^^
@@ -379,7 +379,7 @@ LL |     suggestion: (Span, usize),
    = help: `#[suggestion(...)]` on a tuple field must be applied to fields of type `(Span, Applicability)`
 
 error: derive(Diagnostic): wrong types for suggestion
-  --> $DIR/diagnostic-derive-inline.rs:565:17
+  --> $DIR/diagnostic-derive-inline.rs:564:17
    |
 LL |     suggestion: (Span,),
    |                 ^^^^^^^
@@ -387,13 +387,13 @@ LL |     suggestion: (Span,),
    = help: `#[suggestion(...)]` on a tuple field must be applied to fields of type `(Span, Applicability)`
 
 error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive-inline.rs:572:5
+  --> $DIR/diagnostic-derive-inline.rs:571:5
    |
 LL |     #[suggestion("with a suggestion")]
    |     ^
 
 error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:579:1
+  --> $DIR/diagnostic-derive-inline.rs:578:1
    |
 LL | #[multipart_suggestion("with a suggestion")]
    | ^
@@ -401,7 +401,7 @@ LL | #[multipart_suggestion("with a suggestion")]
    = help: consider creating a `Subdiagnostic` instead
 
 error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:582:1
+  --> $DIR/diagnostic-derive-inline.rs:581:1
    |
 LL | #[multipart_suggestion()]
    | ^
@@ -409,7 +409,7 @@ LL | #[multipart_suggestion()]
    = help: consider creating a `Subdiagnostic` instead
 
 error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:586:5
+  --> $DIR/diagnostic-derive-inline.rs:585:5
    |
 LL |     #[multipart_suggestion("with a suggestion")]
    |     ^
@@ -417,7 +417,7 @@ LL |     #[multipart_suggestion("with a suggestion")]
    = help: consider creating a `Subdiagnostic` instead
 
 error: derive(Diagnostic): `#[suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:594:1
+  --> $DIR/diagnostic-derive-inline.rs:593:1
    |
 LL | #[suggestion("with a suggestion", code = "...")]
    | ^
@@ -425,7 +425,7 @@ LL | #[suggestion("with a suggestion", code = "...")]
    = help: `#[label]` and `#[suggestion]` can only be applied to fields
 
 error: derive(Diagnostic): `#[label]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:603:1
+  --> $DIR/diagnostic-derive-inline.rs:602:1
    |
 LL | #[label]
    | ^
@@ -433,73 +433,73 @@ LL | #[label]
    = help: `#[label]` and `#[suggestion]` can only be applied to fields
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:637:5
+  --> $DIR/diagnostic-derive-inline.rs:636:5
    |
 LL |     #[subdiagnostic(bad)]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:645:5
+  --> $DIR/diagnostic-derive-inline.rs:644:5
    |
 LL |     #[subdiagnostic = "bad"]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:653:5
+  --> $DIR/diagnostic-derive-inline.rs:652:5
    |
 LL |     #[subdiagnostic(bad, bad)]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:661:5
+  --> $DIR/diagnostic-derive-inline.rs:660:5
    |
 LL |     #[subdiagnostic("bad")]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:669:5
+  --> $DIR/diagnostic-derive-inline.rs:668:5
    |
 LL |     #[subdiagnostic(eager)]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:677:5
+  --> $DIR/diagnostic-derive-inline.rs:676:5
    |
 LL |     #[subdiagnostic(eager)]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:698:5
+  --> $DIR/diagnostic-derive-inline.rs:697:5
    |
 LL |     #[subdiagnostic(eager)]
    |     ^
 
 error: derive(Diagnostic): expected at least one string literal for `code(...)`
-  --> $DIR/diagnostic-derive-inline.rs:729:44
+  --> $DIR/diagnostic-derive-inline.rs:728:44
    |
 LL |     #[suggestion("with a suggestion", code())]
    |                                            ^
 
 error: derive(Diagnostic): `code(...)` must contain only string literals
-  --> $DIR/diagnostic-derive-inline.rs:737:44
+  --> $DIR/diagnostic-derive-inline.rs:736:44
    |
 LL |     #[suggestion("with a suggestion", code(foo))]
    |                                            ^^^
 
 error: unexpected token, expected `)`
-  --> $DIR/diagnostic-derive-inline.rs:737:44
+  --> $DIR/diagnostic-derive-inline.rs:736:44
    |
 LL |     #[suggestion("with a suggestion", code(foo))]
    |                                            ^^^
 
 error: expected string literal
-  --> $DIR/diagnostic-derive-inline.rs:746:46
+  --> $DIR/diagnostic-derive-inline.rs:745:46
    |
 LL |     #[suggestion("with a suggestion", code = 3)]
    |                                              ^
 
 error: derive(Diagnostic): `#[suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:761:5
+  --> $DIR/diagnostic-derive-inline.rs:760:5
    |
 LL |     #[suggestion("with a suggestion", code = "")]
    |     ^
@@ -509,7 +509,7 @@ LL |     #[suggestion("with a suggestion", code = "")]
    = help: to show a variable set of suggestions, use a `Vec` of `Subdiagnostic`s annotated with `#[suggestion(...)]`
 
 error: derive(Diagnostic): Variable `nosub` not found in diagnostic 
-  --> $DIR/diagnostic-derive-inline.rs:773:8
+  --> $DIR/diagnostic-derive-inline.rs:772:8
    |
 LL | #[diag("does not exist: {$nosub}")]
    |        ^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -517,19 +517,19 @@ LL | #[diag("does not exist: {$nosub}")]
    = help: Available fields: "sub"
 
 error: cannot find attribute `nonsense` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:61:3
+  --> $DIR/diagnostic-derive-inline.rs:60:3
    |
 LL | #[nonsense("this is an example message", code = E0123)]
    |   ^^^^^^^^
 
 error: cannot find attribute `nonsense` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:139:7
+  --> $DIR/diagnostic-derive-inline.rs:138:7
    |
 LL |     #[nonsense]
    |       ^^^^^^^^
 
 error: cannot find attribute `error` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:518:3
+  --> $DIR/diagnostic-derive-inline.rs:517:3
    |
 LL | #[error("this is an example message", code = E0123)]
    |   ^^^^^
@@ -541,7 +541,7 @@ LL | struct ErrorAttribute {}
    |
 
 error: cannot find attribute `warn_` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:525:3
+  --> $DIR/diagnostic-derive-inline.rs:524:3
    |
 LL | #[warn_("this is an example message", code = E0123)]
    |   ^^^^^
@@ -553,7 +553,7 @@ LL + #[warn("this is an example message", code = E0123)]
    |
 
 error: cannot find attribute `lint` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:532:3
+  --> $DIR/diagnostic-derive-inline.rs:531:3
    |
 LL | #[lint("this is an example message", code = E0123)]
    |   ^^^^
@@ -565,7 +565,7 @@ LL + #[link("this is an example message", code = E0123)]
    |
 
 error: cannot find attribute `lint` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:539:3
+  --> $DIR/diagnostic-derive-inline.rs:538:3
    |
 LL | #[lint("this is an example message", code = E0123)]
    |   ^^^^
@@ -577,7 +577,7 @@ LL + #[link("this is an example message", code = E0123)]
    |
 
 error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:579:3
+  --> $DIR/diagnostic-derive-inline.rs:578:3
    |
 LL | #[multipart_suggestion("with a suggestion")]
    |   ^^^^^^^^^^^^^^^^^^^^
@@ -589,7 +589,7 @@ LL | struct MultipartSuggestion {
    |
 
 error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:582:3
+  --> $DIR/diagnostic-derive-inline.rs:581:3
    |
 LL | #[multipart_suggestion()]
    |   ^^^^^^^^^^^^^^^^^^^^
@@ -601,7 +601,7 @@ LL | struct MultipartSuggestion {
    |
 
 error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:586:7
+  --> $DIR/diagnostic-derive-inline.rs:585:7
    |
 LL |     #[multipart_suggestion("with a suggestion")]
    |       ^^^^^^^^^^^^^^^^^^^^
@@ -609,7 +609,7 @@ LL |     #[multipart_suggestion("with a suggestion")]
    = note: `multipart_suggestion` is an attribute that can be used by the derive macro `Subdiagnostic`, you might be missing a `derive` attribute
 
 error[E0277]: the trait bound `Hello: IntoDiagArg` is not satisfied
-  --> $DIR/diagnostic-derive-inline.rs:330:12
+  --> $DIR/diagnostic-derive-inline.rs:329:12
    |
 LL | #[derive(Diagnostic)]
    |          ---------- required by a bound introduced by this call
@@ -618,7 +618,7 @@ LL |     other: Hello,
    |            ^^^^^ unsatisfied trait bound
    |
 help: the nightly-only, unstable trait `IntoDiagArg` is not implemented for `Hello`
-  --> $DIR/diagnostic-derive-inline.rs:42:1
+  --> $DIR/diagnostic-derive-inline.rs:41:1
    |
 LL | struct Hello {}
    | ^^^^^^^^^^^^
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
index eaa681d40be5..079c2a74c4a1 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
@@ -12,7 +12,6 @@
 #![crate_type = "lib"]
 
 extern crate rustc_errors;
-extern crate rustc_fluent_macro;
 extern crate rustc_macros;
 extern crate rustc_session;
 extern crate rustc_span;
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
index 11753b949bc7..8ff5ed00d0ba 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
@@ -1,143 +1,143 @@
 error: derive(Diagnostic): label without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive-inline.rs:50:1
+  --> $DIR/subdiagnostic-derive-inline.rs:49:1
    |
 LL | #[label("example message")]
    | ^
 
 error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:57:1
+  --> $DIR/subdiagnostic-derive-inline.rs:56:1
    |
 LL | #[label]
    | ^
 
 error: derive(Diagnostic): `#[foo]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:66:1
+  --> $DIR/subdiagnostic-derive-inline.rs:65:1
    |
 LL | #[foo]
    | ^
 
 error: derive(Diagnostic): `#[label = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:76:1
+  --> $DIR/subdiagnostic-derive-inline.rs:75:1
    |
 LL | #[label = "..."]
    | ^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive-inline.rs:85:9
+  --> $DIR/subdiagnostic-derive-inline.rs:84:9
    |
 LL | #[label(bug = "...")]
    |         ^^^
 
 error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:85:1
+  --> $DIR/subdiagnostic-derive-inline.rs:84:1
    |
 LL | #[label(bug = "...")]
    | ^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive-inline.rs:95:9
+  --> $DIR/subdiagnostic-derive-inline.rs:94:9
    |
 LL | #[label(slug = 4)]
    |         ^^^^
 
 error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:95:1
+  --> $DIR/subdiagnostic-derive-inline.rs:94:1
    |
 LL | #[label(slug = 4)]
    | ^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive-inline.rs:105:9
+  --> $DIR/subdiagnostic-derive-inline.rs:104:9
    |
 LL | #[label(slug("..."))]
    |         ^^^^
 
 error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:105:1
+  --> $DIR/subdiagnostic-derive-inline.rs:104:1
    |
 LL | #[label(slug("..."))]
    | ^
 
 error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:115:1
+  --> $DIR/subdiagnostic-derive-inline.rs:114:1
    |
 LL | #[label()]
    | ^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive-inline.rs:124:28
+  --> $DIR/subdiagnostic-derive-inline.rs:123:28
    |
 LL | #[label("example message", code = "...")]
    |                            ^^^^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive-inline.rs:133:28
+  --> $DIR/subdiagnostic-derive-inline.rs:132:28
    |
 LL | #[label("example message", applicability = "machine-applicable")]
    |                            ^^^^^^^^^^^^^
 
 error: derive(Diagnostic): unsupported type attribute for subdiagnostic enum
-  --> $DIR/subdiagnostic-derive-inline.rs:142:1
+  --> $DIR/subdiagnostic-derive-inline.rs:141:1
    |
 LL | #[foo]
    | ^
 
 error: derive(Diagnostic): `#[bar]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:156:5
+  --> $DIR/subdiagnostic-derive-inline.rs:155:5
    |
 LL |     #[bar]
    |     ^
 
 error: derive(Diagnostic): `#[bar = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:168:5
+  --> $DIR/subdiagnostic-derive-inline.rs:167:5
    |
 LL |     #[bar = "..."]
    |     ^
 
 error: derive(Diagnostic): `#[bar = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:180:5
+  --> $DIR/subdiagnostic-derive-inline.rs:179:5
    |
 LL |     #[bar = 4]
    |     ^
 
 error: derive(Diagnostic): `#[bar(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:192:5
+  --> $DIR/subdiagnostic-derive-inline.rs:191:5
    |
 LL |     #[bar("...")]
    |     ^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/subdiagnostic-derive-inline.rs:204:13
+  --> $DIR/subdiagnostic-derive-inline.rs:203:13
    |
 LL |     #[label(code = "...")]
    |             ^^^^
 
 error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:204:5
+  --> $DIR/subdiagnostic-derive-inline.rs:203:5
    |
 LL |     #[label(code = "...")]
    |     ^
 
 error: derive(Diagnostic): the `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/subdiagnostic-derive-inline.rs:233:5
+  --> $DIR/subdiagnostic-derive-inline.rs:232:5
    |
 LL |     #[primary_span]
    |     ^
 
 error: derive(Diagnostic): label without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive-inline.rs:230:1
+  --> $DIR/subdiagnostic-derive-inline.rs:229:1
    |
 LL | #[label("example message")]
    | ^
 
 error: derive(Diagnostic): `#[applicability]` is only valid on suggestions
-  --> $DIR/subdiagnostic-derive-inline.rs:243:5
+  --> $DIR/subdiagnostic-derive-inline.rs:242:5
    |
 LL |     #[applicability]
    |     ^
 
 error: derive(Diagnostic): `#[bar]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:253:5
+  --> $DIR/subdiagnostic-derive-inline.rs:252:5
    |
 LL |     #[bar]
    |     ^
@@ -145,13 +145,13 @@ LL |     #[bar]
    = help: only `primary_span`, `applicability` and `skip_arg` are valid field attributes
 
 error: derive(Diagnostic): `#[bar = ...]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:264:5
+  --> $DIR/subdiagnostic-derive-inline.rs:263:5
    |
 LL |     #[bar = "..."]
    |     ^
 
 error: derive(Diagnostic): `#[bar(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:275:5
+  --> $DIR/subdiagnostic-derive-inline.rs:274:5
    |
 LL |     #[bar("...")]
    |     ^
@@ -159,7 +159,7 @@ LL |     #[bar("...")]
    = help: only `primary_span`, `applicability` and `skip_arg` are valid field attributes
 
 error: unexpected unsupported untagged union
-  --> $DIR/subdiagnostic-derive-inline.rs:291:1
+  --> $DIR/subdiagnostic-derive-inline.rs:290:1
    |
 LL | / union AC {
 LL | |
@@ -169,97 +169,97 @@ LL | | }
    | |_^
 
 error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:306:28
+  --> $DIR/subdiagnostic-derive-inline.rs:305:28
    |
 LL | #[label("example message", no_crate::example)]
    |                            ^^^^^^^^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive-inline.rs:319:5
+  --> $DIR/subdiagnostic-derive-inline.rs:318:5
    |
 LL |     #[primary_span]
    |     ^
    |
 note: previously specified here
-  --> $DIR/subdiagnostic-derive-inline.rs:316:5
+  --> $DIR/subdiagnostic-derive-inline.rs:315:5
    |
 LL |     #[primary_span]
    |     ^
 
 error: derive(Diagnostic): subdiagnostic kind not specified
-  --> $DIR/subdiagnostic-derive-inline.rs:325:8
+  --> $DIR/subdiagnostic-derive-inline.rs:324:8
    |
 LL | struct AG {
    |        ^^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive-inline.rs:362:47
+  --> $DIR/subdiagnostic-derive-inline.rs:361:47
    |
 LL | #[suggestion("example message", code = "...", code = "...")]
    |                                               ^^^^
    |
 note: previously specified here
-  --> $DIR/subdiagnostic-derive-inline.rs:362:33
+  --> $DIR/subdiagnostic-derive-inline.rs:361:33
    |
 LL | #[suggestion("example message", code = "...", code = "...")]
    |                                 ^^^^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive-inline.rs:380:5
+  --> $DIR/subdiagnostic-derive-inline.rs:379:5
    |
 LL |     #[applicability]
    |     ^
    |
 note: previously specified here
-  --> $DIR/subdiagnostic-derive-inline.rs:377:5
+  --> $DIR/subdiagnostic-derive-inline.rs:376:5
    |
 LL |     #[applicability]
    |     ^
 
 error: derive(Diagnostic): the `#[applicability]` attribute can only be applied to fields of type `Applicability`
-  --> $DIR/subdiagnostic-derive-inline.rs:390:5
+  --> $DIR/subdiagnostic-derive-inline.rs:389:5
    |
 LL |     #[applicability]
    |     ^
 
 error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/subdiagnostic-derive-inline.rs:403:1
+  --> $DIR/subdiagnostic-derive-inline.rs:402:1
    |
 LL | #[suggestion("example message")]
    | ^
 
 error: derive(Diagnostic): invalid applicability
-  --> $DIR/subdiagnostic-derive-inline.rs:413:63
+  --> $DIR/subdiagnostic-derive-inline.rs:412:63
    |
 LL | #[suggestion("example message", code = "...", applicability = "foo")]
    |                                                               ^^^^^
 
 error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive-inline.rs:431:1
+  --> $DIR/subdiagnostic-derive-inline.rs:430:1
    |
 LL | #[suggestion("example message", code = "...")]
    | ^
 
 error: derive(Diagnostic): unsupported type attribute for subdiagnostic enum
-  --> $DIR/subdiagnostic-derive-inline.rs:445:1
+  --> $DIR/subdiagnostic-derive-inline.rs:444:1
    |
 LL | #[label]
    | ^
 
 error: derive(Diagnostic): `var` doesn't refer to a field on this type
-  --> $DIR/subdiagnostic-derive-inline.rs:465:40
+  --> $DIR/subdiagnostic-derive-inline.rs:464:40
    |
 LL | #[suggestion("example message", code = "{var}", applicability = "machine-applicable")]
    |                                        ^^^^^^^
 
 error: derive(Diagnostic): `var` doesn't refer to a field on this type
-  --> $DIR/subdiagnostic-derive-inline.rs:484:44
+  --> $DIR/subdiagnostic-derive-inline.rs:483:44
    |
 LL |     #[suggestion("example message", code = "{var}", applicability = "machine-applicable")]
    |                                            ^^^^^^^
 
 error: derive(Diagnostic): `#[suggestion_part]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:507:5
+  --> $DIR/subdiagnostic-derive-inline.rs:506:5
    |
 LL |     #[suggestion_part]
    |     ^
@@ -267,7 +267,7 @@ LL |     #[suggestion_part]
    = help: `#[suggestion_part(...)]` is only valid in multipart suggestions, use `#[primary_span]` instead
 
 error: derive(Diagnostic): `#[suggestion_part(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:510:5
+  --> $DIR/subdiagnostic-derive-inline.rs:509:5
    |
 LL |     #[suggestion_part(code = "...")]
    |     ^
@@ -275,13 +275,13 @@ LL |     #[suggestion_part(code = "...")]
    = help: `#[suggestion_part(...)]` is only valid in multipart suggestions
 
 error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive-inline.rs:504:1
+  --> $DIR/subdiagnostic-derive-inline.rs:503:1
    |
 LL | #[suggestion("example message", code = "...")]
    | ^
 
 error: derive(Diagnostic): invalid nested attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:519:43
+  --> $DIR/subdiagnostic-derive-inline.rs:518:43
    |
 LL | #[multipart_suggestion("example message", code = "...", applicability = "machine-applicable")]
    |                                           ^^^^
@@ -289,25 +289,25 @@ LL | #[multipart_suggestion("example message", code = "...", applicability = "ma
    = help: only `style` and `applicability` are valid nested attributes
 
 error: derive(Diagnostic): multipart suggestion without any `#[suggestion_part(...)]` fields
-  --> $DIR/subdiagnostic-derive-inline.rs:519:1
+  --> $DIR/subdiagnostic-derive-inline.rs:518:1
    |
 LL | #[multipart_suggestion("example message", code = "...", applicability = "machine-applicable")]
    | ^
 
 error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive-inline.rs:529:5
+  --> $DIR/subdiagnostic-derive-inline.rs:528:5
    |
 LL |     #[suggestion_part]
    |     ^
 
 error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive-inline.rs:537:5
+  --> $DIR/subdiagnostic-derive-inline.rs:536:5
    |
 LL |     #[suggestion_part()]
    |     ^
 
 error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:546:5
+  --> $DIR/subdiagnostic-derive-inline.rs:545:5
    |
 LL |     #[primary_span]
    |     ^
@@ -315,127 +315,127 @@ LL |     #[primary_span]
    = help: multipart suggestions use one or more `#[suggestion_part]`s rather than one `#[primary_span]`
 
 error: derive(Diagnostic): multipart suggestion without any `#[suggestion_part(...)]` fields
-  --> $DIR/subdiagnostic-derive-inline.rs:543:1
+  --> $DIR/subdiagnostic-derive-inline.rs:542:1
    |
 LL | #[multipart_suggestion("example message")]
    | ^
 
 error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive-inline.rs:554:5
+  --> $DIR/subdiagnostic-derive-inline.rs:553:5
    |
 LL |     #[suggestion_part]
    |     ^
 
 error: derive(Diagnostic): `#[suggestion_part(...)]` attribute without `code = "..."`
-  --> $DIR/subdiagnostic-derive-inline.rs:557:5
+  --> $DIR/subdiagnostic-derive-inline.rs:556:5
    |
 LL |     #[suggestion_part()]
    |     ^
 
 error: derive(Diagnostic): `code` is the only valid nested attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:560:23
+  --> $DIR/subdiagnostic-derive-inline.rs:559:23
    |
 LL |     #[suggestion_part(foo = "bar")]
    |                       ^^^
 
 error: derive(Diagnostic): the `#[suggestion_part(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/subdiagnostic-derive-inline.rs:564:5
+  --> $DIR/subdiagnostic-derive-inline.rs:563:5
    |
 LL |     #[suggestion_part(code = "...")]
    |     ^
 
 error: derive(Diagnostic): the `#[suggestion_part(...)]` attribute can only be applied to fields of type `Span` or `MultiSpan`
-  --> $DIR/subdiagnostic-derive-inline.rs:567:5
+  --> $DIR/subdiagnostic-derive-inline.rs:566:5
    |
 LL |     #[suggestion_part()]
    |     ^
 
 error: expected `,`
-  --> $DIR/subdiagnostic-derive-inline.rs:560:27
+  --> $DIR/subdiagnostic-derive-inline.rs:559:27
    |
 LL |     #[suggestion_part(foo = "bar")]
    |                           ^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive-inline.rs:575:37
+  --> $DIR/subdiagnostic-derive-inline.rs:574:37
    |
 LL |     #[suggestion_part(code = "...", code = ",,,")]
    |                                     ^^^^
    |
 note: previously specified here
-  --> $DIR/subdiagnostic-derive-inline.rs:575:23
+  --> $DIR/subdiagnostic-derive-inline.rs:574:23
    |
 LL |     #[suggestion_part(code = "...", code = ",,,")]
    |                       ^^^^
 
 error: derive(Diagnostic): `#[applicability]` has no effect if all `#[suggestion]`/`#[multipart_suggestion]` attributes have a static `applicability = "..."`
-  --> $DIR/subdiagnostic-derive-inline.rs:604:5
+  --> $DIR/subdiagnostic-derive-inline.rs:603:5
    |
 LL |     #[applicability]
    |     ^
 
 error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive-inline.rs:652:28
+  --> $DIR/subdiagnostic-derive-inline.rs:651:28
    |
 LL |     #[suggestion_part(code("foo"))]
    |                            ^^^^^
 
 error: unexpected token, expected `)`
-  --> $DIR/subdiagnostic-derive-inline.rs:652:28
+  --> $DIR/subdiagnostic-derive-inline.rs:651:28
    |
 LL |     #[suggestion_part(code("foo"))]
    |                            ^^^^^
 
 error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive-inline.rs:662:28
+  --> $DIR/subdiagnostic-derive-inline.rs:661:28
    |
 LL |     #[suggestion_part(code("foo", "bar"))]
    |                            ^^^^^
 
 error: unexpected token, expected `)`
-  --> $DIR/subdiagnostic-derive-inline.rs:662:28
+  --> $DIR/subdiagnostic-derive-inline.rs:661:28
    |
 LL |     #[suggestion_part(code("foo", "bar"))]
    |                            ^^^^^
 
 error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive-inline.rs:672:28
+  --> $DIR/subdiagnostic-derive-inline.rs:671:28
    |
 LL |     #[suggestion_part(code(3))]
    |                            ^
 
 error: unexpected token, expected `)`
-  --> $DIR/subdiagnostic-derive-inline.rs:672:28
+  --> $DIR/subdiagnostic-derive-inline.rs:671:28
    |
 LL |     #[suggestion_part(code(3))]
    |                            ^
 
 error: derive(Diagnostic): expected exactly one string literal for `code = ...`
-  --> $DIR/subdiagnostic-derive-inline.rs:682:28
+  --> $DIR/subdiagnostic-derive-inline.rs:681:28
    |
 LL |     #[suggestion_part(code())]
    |                            ^
 
 error: expected string literal
-  --> $DIR/subdiagnostic-derive-inline.rs:691:30
+  --> $DIR/subdiagnostic-derive-inline.rs:690:30
    |
 LL |     #[suggestion_part(code = 3)]
    |                              ^
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/subdiagnostic-derive-inline.rs:733:1
+  --> $DIR/subdiagnostic-derive-inline.rs:732:1
    |
 LL | #[suggestion("example message", code = "", style = "hidden", style = "normal")]
    | ^
    |
 note: previously specified here
-  --> $DIR/subdiagnostic-derive-inline.rs:733:1
+  --> $DIR/subdiagnostic-derive-inline.rs:732:1
    |
 LL | #[suggestion("example message", code = "", style = "hidden", style = "normal")]
    | ^
 
 error: derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:742:1
+  --> $DIR/subdiagnostic-derive-inline.rs:741:1
    |
 LL | #[suggestion_hidden("example message", code = "")]
    | ^
@@ -443,7 +443,7 @@ LL | #[suggestion_hidden("example message", code = "")]
    = help: Use `#[suggestion(..., style = "hidden")]` instead
 
 error: derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:750:1
+  --> $DIR/subdiagnostic-derive-inline.rs:749:1
    |
 LL | #[suggestion_hidden("example message", code = "", style = "normal")]
    | ^
@@ -451,7 +451,7 @@ LL | #[suggestion_hidden("example message", code = "", style = "normal")]
    = help: Use `#[suggestion(..., style = "hidden")]` instead
 
 error: derive(Diagnostic): invalid suggestion style
-  --> $DIR/subdiagnostic-derive-inline.rs:758:52
+  --> $DIR/subdiagnostic-derive-inline.rs:757:52
    |
 LL | #[suggestion("example message", code = "", style = "foo")]
    |                                                    ^^^^^
@@ -459,25 +459,25 @@ LL | #[suggestion("example message", code = "", style = "foo")]
    = help: valid styles are `normal`, `short`, `hidden`, `verbose` and `tool-only`
 
 error: expected string literal
-  --> $DIR/subdiagnostic-derive-inline.rs:766:52
+  --> $DIR/subdiagnostic-derive-inline.rs:765:52
    |
 LL | #[suggestion("example message", code = "", style = 42)]
    |                                                    ^^
 
 error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:774:44
+  --> $DIR/subdiagnostic-derive-inline.rs:773:44
    |
 LL | #[suggestion("example message", code = "", style)]
    |                                            ^^^^^
 
 error: expected `=`
-  --> $DIR/subdiagnostic-derive-inline.rs:782:49
+  --> $DIR/subdiagnostic-derive-inline.rs:781:49
    |
 LL | #[suggestion("example message", code = "", style("foo"))]
    |                                                 ^
 
 error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:793:5
+  --> $DIR/subdiagnostic-derive-inline.rs:792:5
    |
 LL |     #[primary_span]
    |     ^
@@ -486,61 +486,61 @@ LL |     #[primary_span]
    = help: to create a suggestion with multiple spans, use `#[multipart_suggestion]` instead
 
 error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive-inline.rs:790:1
+  --> $DIR/subdiagnostic-derive-inline.rs:789:1
    |
 LL | #[suggestion("example message", code = "")]
    | ^
 
 error: cannot find attribute `foo` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:66:3
+  --> $DIR/subdiagnostic-derive-inline.rs:65:3
    |
 LL | #[foo]
    |   ^^^
 
 error: cannot find attribute `foo` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:142:3
+  --> $DIR/subdiagnostic-derive-inline.rs:141:3
    |
 LL | #[foo]
    |   ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:156:7
+  --> $DIR/subdiagnostic-derive-inline.rs:155:7
    |
 LL |     #[bar]
    |       ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:168:7
+  --> $DIR/subdiagnostic-derive-inline.rs:167:7
    |
 LL |     #[bar = "..."]
    |       ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:180:7
+  --> $DIR/subdiagnostic-derive-inline.rs:179:7
    |
 LL |     #[bar = 4]
    |       ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:192:7
+  --> $DIR/subdiagnostic-derive-inline.rs:191:7
    |
 LL |     #[bar("...")]
    |       ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:253:7
+  --> $DIR/subdiagnostic-derive-inline.rs:252:7
    |
 LL |     #[bar]
    |       ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:264:7
+  --> $DIR/subdiagnostic-derive-inline.rs:263:7
    |
 LL |     #[bar = "..."]
    |       ^^^
 
 error: cannot find attribute `bar` in this scope
-  --> $DIR/subdiagnostic-derive-inline.rs:275:7
+  --> $DIR/subdiagnostic-derive-inline.rs:274:7
    |
 LL |     #[bar("...")]
    |       ^^^

From 236dac8d0e41ae9d4e31908c9a0a3fdf2789f5f6 Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan 
Date: Sat, 7 Feb 2026 14:02:33 -0500
Subject: [PATCH 1426/1843] fix: remove unnecessary trailing commas in format
 strings

This PR makes the new comma-removing lint PR less noisy, allowing it focus only on the new functionality
---
 clippy_dev/src/new_lint.rs                         | 4 ++--
 clippy_lints/src/cargo/lint_groups_priority.rs     | 2 +-
 clippy_lints/src/casts/cast_possible_truncation.rs | 2 +-
 clippy_lints/src/doc/doc_suspicious_footnotes.rs   | 2 +-
 clippy_lints/src/formatting.rs                     | 2 +-
 clippy_lints/src/implicit_hasher.rs                | 4 ++--
 clippy_lints/src/matches/manual_unwrap_or.rs       | 2 +-
 clippy_lints/src/matches/match_like_matches.rs     | 2 +-
 clippy_lints/src/methods/bytes_nth.rs              | 2 +-
 clippy_lints/src/methods/into_iter_on_ref.rs       | 2 +-
 clippy_lints/src/methods/manual_try_fold.rs        | 2 +-
 clippy_lints/src/methods/unit_hash.rs              | 2 +-
 clippy_lints/src/missing_asserts_for_indexing.rs   | 6 +++---
 clippy_lints/src/missing_enforced_import_rename.rs | 2 +-
 clippy_lints/src/ptr/cmp_null.rs                   | 2 +-
 clippy_lints/src/toplevel_ref_arg.rs               | 2 +-
 clippy_lints/src/zero_div_zero.rs                  | 2 +-
 clippy_lints/src/zero_repeat_side_effects.rs       | 2 +-
 rustc_tools_util/src/lib.rs                        | 2 +-
 19 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/clippy_dev/src/new_lint.rs b/clippy_dev/src/new_lint.rs
index 0b6d702d7721..72f281ca4d9d 100644
--- a/clippy_dev/src/new_lint.rs
+++ b/clippy_dev/src/new_lint.rs
@@ -167,9 +167,9 @@ fn add_lint(lint: &LintData<'_>, enable_msrv: bool) -> io::Result<()> {
     let camel_name = to_camel_case(lint.name);
 
     let new_lint = if enable_msrv {
-        format!("Box::new(move |{ctor_arg}| Box::new({module_name}::{camel_name}::new(conf))),\n        ",)
+        format!("Box::new(move |{ctor_arg}| Box::new({module_name}::{camel_name}::new(conf))),\n        ")
     } else {
-        format!("Box::new(|{ctor_arg}| Box::new({module_name}::{camel_name})),\n        ",)
+        format!("Box::new(|{ctor_arg}| Box::new({module_name}::{camel_name})),\n        ")
     };
 
     lib_rs.insert_str(comment_start, &new_lint);
diff --git a/clippy_lints/src/cargo/lint_groups_priority.rs b/clippy_lints/src/cargo/lint_groups_priority.rs
index 14c5e22fb9cd..f937f065d6e0 100644
--- a/clippy_lints/src/cargo/lint_groups_priority.rs
+++ b/clippy_lints/src/cargo/lint_groups_priority.rs
@@ -100,7 +100,7 @@ fn check_table(cx: &LateContext<'_>, table: &DeTable<'_>, known_groups: &FxHashS
                             "to have lints override the group set `{}` to a lower priority",
                             group.as_ref()
                         ),
-                        format!("{{ level = {:?}, priority = {low_priority} }}", group_config.level,),
+                        format!("{{ level = {:?}, priority = {low_priority} }}", group_config.level),
                         Applicability::MaybeIncorrect,
                     );
                 },
diff --git a/clippy_lints/src/casts/cast_possible_truncation.rs b/clippy_lints/src/casts/cast_possible_truncation.rs
index 2eebe8492327..0591afaa6c58 100644
--- a/clippy_lints/src/casts/cast_possible_truncation.rs
+++ b/clippy_lints/src/casts/cast_possible_truncation.rs
@@ -117,7 +117,7 @@ pub(super) fn check(
                 return;
             }
 
-            format!("casting `{cast_from}` to `{cast_to}` may truncate the value{suffix}",)
+            format!("casting `{cast_from}` to `{cast_to}` may truncate the value{suffix}")
         },
 
         (ty::Adt(def, _), Some(to_nbits)) if def.is_enum() => {
diff --git a/clippy_lints/src/doc/doc_suspicious_footnotes.rs b/clippy_lints/src/doc/doc_suspicious_footnotes.rs
index deca29a1885f..dfa6c9637864 100644
--- a/clippy_lints/src/doc/doc_suspicious_footnotes.rs
+++ b/clippy_lints/src/doc/doc_suspicious_footnotes.rs
@@ -91,7 +91,7 @@ pub fn check(cx: &LateContext<'_>, doc: &str, range: Range, fragments: &F
                             diag.span_suggestion_verbose(
                                 this_fragment.span.shrink_to_hi(),
                                 "add footnote definition",
-                                format!("\n\n{label}: ", label = &doc[start..end],),
+                                format!("\n\n{label}: ", label = &doc[start..end]),
                                 Applicability::HasPlaceholders,
                             );
                         } else {
diff --git a/clippy_lints/src/formatting.rs b/clippy_lints/src/formatting.rs
index 3b9c70e23e20..7a64d3135fa5 100644
--- a/clippy_lints/src/formatting.rs
+++ b/clippy_lints/src/formatting.rs
@@ -337,7 +337,7 @@ fn check_missing_else(cx: &EarlyContext<'_>, first: &Expr, second: &Expr) {
             else_span,
             format!("this looks like {looks_like} but the `else` is missing"),
             None,
-            format!("to remove this lint, add the missing `else` or add a new line before {next_thing}",),
+            format!("to remove this lint, add the missing `else` or add a new line before {next_thing}"),
         );
     }
 }
diff --git a/clippy_lints/src/implicit_hasher.rs b/clippy_lints/src/implicit_hasher.rs
index 9dc74a157cbf..70176c62772b 100644
--- a/clippy_lints/src/implicit_hasher.rs
+++ b/clippy_lints/src/implicit_hasher.rs
@@ -92,7 +92,7 @@ impl<'tcx> LateLintPass<'tcx> for ImplicitHasher {
                 ),
                 (
                     target.span(),
-                    format!("{}<{}, S>", target.type_name(), target.type_arguments(),),
+                    format!("{}<{}, S>", target.type_name(), target.type_arguments()),
                 ),
             ];
             suggestions.extend(vis.suggestions);
@@ -352,7 +352,7 @@ impl<'tcx> Visitor<'tcx> for ImplicitHasherConstructorVisitor<'_, '_, 'tcx> {
                     );
                     self.suggestions.insert(
                         e.span,
-                        format!("{container_name}::with_capacity_and_hasher({arg_snippet}, Default::default())",),
+                        format!("{container_name}::with_capacity_and_hasher({arg_snippet}, Default::default())"),
                     );
                 },
                 _ => {},
diff --git a/clippy_lints/src/matches/manual_unwrap_or.rs b/clippy_lints/src/matches/manual_unwrap_or.rs
index abbc43d8e9b0..421c6064284d 100644
--- a/clippy_lints/src/matches/manual_unwrap_or.rs
+++ b/clippy_lints/src/matches/manual_unwrap_or.rs
@@ -173,7 +173,7 @@ fn handle(
                 expr.span,
                 format!("this pattern reimplements `{ty_name}::unwrap_or`"),
                 "replace with",
-                format!("{suggestion}.unwrap_or({reindented_or_body})",),
+                format!("{suggestion}.unwrap_or({reindented_or_body})"),
                 app,
             );
         }
diff --git a/clippy_lints/src/matches/match_like_matches.rs b/clippy_lints/src/matches/match_like_matches.rs
index c26b2dbde7fc..347560b14eea 100644
--- a/clippy_lints/src/matches/match_like_matches.rs
+++ b/clippy_lints/src/matches/match_like_matches.rs
@@ -187,7 +187,7 @@ pub(super) fn check_match<'tcx>(
                 diag.span_suggestion_verbose(
                     e.span,
                     "use `matches!` directly",
-                    format!("{}matches!({snippet}, {pat_and_guard})", if b0 { "" } else { "!" },),
+                    format!("{}matches!({snippet}, {pat_and_guard})", if b0 { "" } else { "!" }),
                     applicability,
                 );
             },
diff --git a/clippy_lints/src/methods/bytes_nth.rs b/clippy_lints/src/methods/bytes_nth.rs
index 40d521d61c11..0a3ea3005e72 100644
--- a/clippy_lints/src/methods/bytes_nth.rs
+++ b/clippy_lints/src/methods/bytes_nth.rs
@@ -34,7 +34,7 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &Expr<'_>, recv: &'tcx E
             parent.span,
             format!("called `.bytes().nth().unwrap()` on a `{caller_type}`"),
             "try",
-            format!("{receiver}.as_bytes()[{n}]",),
+            format!("{receiver}.as_bytes()[{n}]"),
             applicability,
         );
     } else {
diff --git a/clippy_lints/src/methods/into_iter_on_ref.rs b/clippy_lints/src/methods/into_iter_on_ref.rs
index c4b116af4871..5a062732721e 100644
--- a/clippy_lints/src/methods/into_iter_on_ref.rs
+++ b/clippy_lints/src/methods/into_iter_on_ref.rs
@@ -20,7 +20,7 @@ pub(super) fn check(cx: &LateContext<'_>, expr: &hir::Expr<'_>, method_span: Spa
             cx,
             INTO_ITER_ON_REF,
             method_span,
-            format!("this `.into_iter()` call is equivalent to `.{method_name}()` and will not consume the `{kind}`",),
+            format!("this `.into_iter()` call is equivalent to `.{method_name}()` and will not consume the `{kind}`"),
             "call directly",
             method_name.to_string(),
             Applicability::MachineApplicable,
diff --git a/clippy_lints/src/methods/manual_try_fold.rs b/clippy_lints/src/methods/manual_try_fold.rs
index f2e127bedde5..5f5944d5d423 100644
--- a/clippy_lints/src/methods/manual_try_fold.rs
+++ b/clippy_lints/src/methods/manual_try_fold.rs
@@ -47,7 +47,7 @@ pub(super) fn check<'tcx>(
             fold_span,
             "usage of `Iterator::fold` on a type that implements `Try`",
             "use `try_fold` instead",
-            format!("try_fold({init_snip}, {args_snip} ...)",),
+            format!("try_fold({init_snip}, {args_snip} ...)"),
             Applicability::HasPlaceholders,
         );
     }
diff --git a/clippy_lints/src/methods/unit_hash.rs b/clippy_lints/src/methods/unit_hash.rs
index 9defd5626eb4..fb447a99abdc 100644
--- a/clippy_lints/src/methods/unit_hash.rs
+++ b/clippy_lints/src/methods/unit_hash.rs
@@ -19,7 +19,7 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, recv: &'
                 diag.span_suggestion(
                     expr.span,
                     "remove the call to `hash` or consider using",
-                    format!("0_u8.hash({})", snippet(cx, arg.span, ".."),),
+                    format!("0_u8.hash({})", snippet(cx, arg.span, "..")),
                     Applicability::MaybeIncorrect,
                 );
                 diag.note("the implementation of `Hash` for `()` is a no-op");
diff --git a/clippy_lints/src/missing_asserts_for_indexing.rs b/clippy_lints/src/missing_asserts_for_indexing.rs
index 87ee164a1760..651382fb4bfe 100644
--- a/clippy_lints/src/missing_asserts_for_indexing.rs
+++ b/clippy_lints/src/missing_asserts_for_indexing.rs
@@ -364,15 +364,15 @@ fn report_indexes(cx: &LateContext<'_>, map: UnindexMap>
                         // `v.len() < 5` and `v.len() <= 5` does nothing in terms of bounds checks.
                         // The user probably meant `v.len() > 5`
                         LengthComparison::LengthLessThanInt | LengthComparison::LengthLessThanOrEqualInt => {
-                            Some(format!("assert!({slice_str}.len() > {highest_index})",))
+                            Some(format!("assert!({slice_str}.len() > {highest_index})"))
                         },
                         // `5 < v.len()` == `v.len() > 5`
                         LengthComparison::IntLessThanLength if asserted_len < highest_index => {
-                            Some(format!("assert!({slice_str}.len() > {highest_index})",))
+                            Some(format!("assert!({slice_str}.len() > {highest_index})"))
                         },
                         // `5 <= v.len() == `v.len() >= 5`
                         LengthComparison::IntLessThanOrEqualLength if asserted_len <= highest_index => {
-                            Some(format!("assert!({slice_str}.len() > {highest_index})",))
+                            Some(format!("assert!({slice_str}.len() > {highest_index})"))
                         },
                         // `highest_index` here is rather a length, so we need to add 1 to it
                         LengthComparison::LengthEqualInt if asserted_len < highest_index + 1 => match macro_call {
diff --git a/clippy_lints/src/missing_enforced_import_rename.rs b/clippy_lints/src/missing_enforced_import_rename.rs
index 5dd38cf059c2..1f9652a2bd38 100644
--- a/clippy_lints/src/missing_enforced_import_rename.rs
+++ b/clippy_lints/src/missing_enforced_import_rename.rs
@@ -97,7 +97,7 @@ impl LateLintPass<'_> for ImportRename {
                         span_without_semi,
                         "this import should be renamed",
                         "try",
-                        format!("{import} as {name}",),
+                        format!("{import} as {name}"),
                         Applicability::MachineApplicable,
                     );
                 }
diff --git a/clippy_lints/src/ptr/cmp_null.rs b/clippy_lints/src/ptr/cmp_null.rs
index f2d1c855eddd..5e1c62316b7d 100644
--- a/clippy_lints/src/ptr/cmp_null.rs
+++ b/clippy_lints/src/ptr/cmp_null.rs
@@ -32,7 +32,7 @@ pub(super) fn check<'tcx>(
         expr.span,
         "comparing with null is better expressed by the `.is_null()` method",
         "try",
-        format!("{invert}{non_null_path_snippet}.is_null()",),
+        format!("{invert}{non_null_path_snippet}.is_null()"),
         applicability,
     );
     true
diff --git a/clippy_lints/src/toplevel_ref_arg.rs b/clippy_lints/src/toplevel_ref_arg.rs
index 250c277ab5e1..bce2ede6589b 100644
--- a/clippy_lints/src/toplevel_ref_arg.rs
+++ b/clippy_lints/src/toplevel_ref_arg.rs
@@ -109,7 +109,7 @@ impl<'tcx> LateLintPass<'tcx> for ToplevelRefArg {
                     diag.span_suggestion(
                         stmt.span,
                         "try",
-                        format!("let {name}{tyopt} = {initref};", name = snippet(cx, name.span, ".."),),
+                        format!("let {name}{tyopt} = {initref};", name = snippet(cx, name.span, "..")),
                         app,
                     );
                 },
diff --git a/clippy_lints/src/zero_div_zero.rs b/clippy_lints/src/zero_div_zero.rs
index bb0cab3a3075..e44c4dc9776e 100644
--- a/clippy_lints/src/zero_div_zero.rs
+++ b/clippy_lints/src/zero_div_zero.rs
@@ -56,7 +56,7 @@ impl<'tcx> LateLintPass<'tcx> for ZeroDiv {
                 expr.span,
                 "constant division of `0.0` with `0.0` will always result in NaN",
                 None,
-                format!("consider using `{float_type}::NAN` if you would like a constant representing NaN",),
+                format!("consider using `{float_type}::NAN` if you would like a constant representing NaN"),
             );
         }
     }
diff --git a/clippy_lints/src/zero_repeat_side_effects.rs b/clippy_lints/src/zero_repeat_side_effects.rs
index 95085161c09c..cb254cc15629 100644
--- a/clippy_lints/src/zero_repeat_side_effects.rs
+++ b/clippy_lints/src/zero_repeat_side_effects.rs
@@ -168,7 +168,7 @@ fn assign_expr_suggestion(
     let indent = snippet_indent(cx, outer_expr.span).unwrap_or_default();
     let var_name = snippet(cx, assign_expr_span.source_callsite(), "..");
     if needs_curly {
-        format!("{{\n    {indent}{inner_expr};\n    {indent}{var_name} = {vec_str}[] as {return_type}\n{indent}}}",)
+        format!("{{\n    {indent}{inner_expr};\n    {indent}{var_name} = {vec_str}[] as {return_type}\n{indent}}}")
     } else {
         format!("{inner_expr};\n{indent}{var_name} = {vec_str}[] as {return_type}")
     }
diff --git a/rustc_tools_util/src/lib.rs b/rustc_tools_util/src/lib.rs
index 194ed84d04c2..3b7d2d4085d9 100644
--- a/rustc_tools_util/src/lib.rs
+++ b/rustc_tools_util/src/lib.rs
@@ -91,7 +91,7 @@ impl std::fmt::Debug for VersionInfo {
             self.crate_name, self.major, self.minor, self.patch,
         )?;
         if let Some(ref commit_hash) = self.commit_hash {
-            write!(f, ", commit_hash: \"{}\"", commit_hash.trim(),)?;
+            write!(f, ", commit_hash: \"{}\"", commit_hash.trim())?;
         }
         if let Some(ref commit_date) = self.commit_date {
             write!(f, ", commit_date: \"{}\"", commit_date.trim())?;

From d53133eec22e1e7fc6c25dc595708500f36e9ca3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Sat, 7 Feb 2026 22:03:40 +0100
Subject: [PATCH 1427/1843] Remove rustdoc adhoc group

---
 triagebot.toml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index 0ee3d0853034..4a4054391ce7 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1488,11 +1488,6 @@ infra-ci = [
     "@jdno",
     "@jieyouxu",
 ]
-rustdoc = [
-    "@GuillaumeGomez",
-    "@notriddle",
-    "@fmease",
-]
 docs = [
     "@ehuss",
     "@GuillaumeGomez",

From d5c6866dbadbf4c410faf6ab8b01c0f88631e171 Mon Sep 17 00:00:00 2001
From: ia0 
Date: Sat, 7 Feb 2026 22:06:55 +0100
Subject: [PATCH 1428/1843] improve tests

---
 src/tools/rustfmt/tests/source/try_block.rs                | 1 +
 src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs | 5 +++++
 src/tools/rustfmt/tests/target/try_block.rs                | 1 +
 src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs | 5 +++++
 4 files changed, 12 insertions(+)

diff --git a/src/tools/rustfmt/tests/source/try_block.rs b/src/tools/rustfmt/tests/source/try_block.rs
index 2e8d61f7e66a..e324a1331758 100644
--- a/src/tools/rustfmt/tests/source/try_block.rs
+++ b/src/tools/rustfmt/tests/source/try_block.rs
@@ -1,4 +1,5 @@
 // rustfmt-edition: 2018
+#![feature(try_blocks)]
 
 fn main() -> Result<(), !> {
     let _x: Option<_> = try {
diff --git a/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs b/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs
index bae0060374d8..7a1135cfbc76 100644
--- a/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs
+++ b/src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs
@@ -1,4 +1,5 @@
 // rustfmt-edition: 2018
+#![feature(try_blocks_heterogeneous)]
 
 fn main() -> Result<(), !> {
     let _x = try bikeshed Option<_> {
@@ -24,6 +25,10 @@ fn baz() -> Option {
         6
     }; // comment
 
+    let x = try /* Invisible comment */ bikeshed Option<()> {};
+    let x = try bikeshed /* Invisible comment */ Option<()> {};
+    let x = try bikeshed Option<()> /* Invisible comment */ {};
+
     let x = try bikeshed Option { baz()?; baz()?; baz()?; 7 };
 
     let x = try bikeshed Foo { 1 + 1 + 1 };
diff --git a/src/tools/rustfmt/tests/target/try_block.rs b/src/tools/rustfmt/tests/target/try_block.rs
index 19a3f3e14876..61da123b735d 100644
--- a/src/tools/rustfmt/tests/target/try_block.rs
+++ b/src/tools/rustfmt/tests/target/try_block.rs
@@ -1,4 +1,5 @@
 // rustfmt-edition: 2018
+#![feature(try_blocks)]
 
 fn main() -> Result<(), !> {
     let _x: Option<_> = try { 4 };
diff --git a/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs b/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs
index 6f910ed5d051..018d53ed35e4 100644
--- a/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs
+++ b/src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs
@@ -1,4 +1,5 @@
 // rustfmt-edition: 2018
+#![feature(try_blocks_heterogeneous)]
 
 fn main() -> Result<(), !> {
     let _x = try bikeshed Option<_> { 4 };
@@ -18,6 +19,10 @@ fn baz() -> Option {
 
     let y = try bikeshed Option { 6 }; // comment
 
+    let x = try /* Invisible comment */ bikeshed Option<()> {};
+    let x = try bikeshed /* Invisible comment */ Option<()> {};
+    let x = try bikeshed Option<()> /* Invisible comment */ {};
+
     let x = try bikeshed Option {
         baz()?;
         baz()?;

From d0b3a0bdc02924cdd28e37f6209ddea0b6bd4727 Mon Sep 17 00:00:00 2001
From: Jules Bertholet 
Date: Mon, 2 Feb 2026 22:01:10 -0500
Subject: [PATCH 1429/1843] Add some conversion trait impls

- `impl From<[MaybeUninit; N]> for MaybeUninit<[T; N]>`
- `impl AsRef<[MaybeUninit; N]> for MaybeUninit<[T; N]>`
- `impl AsRef<[MaybeUninit]> for MaybeUninit<[T; N]>`
- `impl AsMut<[MaybeUninit; N]> for MaybeUninit<[T; N]>`
- `impl AsMut<[MaybeUninit]> for MaybeUninit<[T; N]>`
- `impl From> for [MaybeUninit; N]`
- `impl AsRef<[Cell; N]> for Cell<[T; N]>`
- `impl AsRef<[Cell]> for Cell<[T; N]>`
- `impl AsRef<[Cell]> for Cell<[T]>`
---
 library/core/src/cell.rs                      | 24 +++++++++
 library/core/src/mem/maybe_uninit.rs          | 50 +++++++++++++++++++
 .../issue-71394-no-from-impl.stderr           |  2 +-
 3 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs
index 661ea4ab6a27..a9e7c49515c7 100644
--- a/library/core/src/cell.rs
+++ b/library/core/src/cell.rs
@@ -689,6 +689,30 @@ impl, U> CoerceUnsized> for Cell {}
 #[unstable(feature = "dispatch_from_dyn", issue = "none")]
 impl, U> DispatchFromDyn> for Cell {}
 
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsRef<[Cell; N]> for Cell<[T; N]> {
+    #[inline]
+    fn as_ref(&self) -> &[Cell; N] {
+        self.as_array_of_cells()
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsRef<[Cell]> for Cell<[T; N]> {
+    #[inline]
+    fn as_ref(&self) -> &[Cell] {
+        &*self.as_array_of_cells()
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsRef<[Cell]> for Cell<[T]> {
+    #[inline]
+    fn as_ref(&self) -> &[Cell] {
+        self.as_slice_of_cells()
+    }
+}
+
 impl Cell<[T]> {
     /// Returns a `&[Cell]` from a `&Cell<[T]>`
     ///
diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs
index 320eb97f83a4..594147720193 100644
--- a/library/core/src/mem/maybe_uninit.rs
+++ b/library/core/src/mem/maybe_uninit.rs
@@ -1532,6 +1532,56 @@ impl MaybeUninit<[T; N]> {
     }
 }
 
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl From<[MaybeUninit; N]> for MaybeUninit<[T; N]> {
+    #[inline]
+    fn from(arr: [MaybeUninit; N]) -> Self {
+        arr.transpose()
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsRef<[MaybeUninit; N]> for MaybeUninit<[T; N]> {
+    #[inline]
+    fn as_ref(&self) -> &[MaybeUninit; N] {
+        // SAFETY: T and MaybeUninit have the same layout
+        unsafe { &*ptr::from_ref(self).cast() }
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsRef<[MaybeUninit]> for MaybeUninit<[T; N]> {
+    #[inline]
+    fn as_ref(&self) -> &[MaybeUninit] {
+        &*AsRef::<[MaybeUninit; N]>::as_ref(self)
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsMut<[MaybeUninit; N]> for MaybeUninit<[T; N]> {
+    #[inline]
+    fn as_mut(&mut self) -> &mut [MaybeUninit; N] {
+        // SAFETY: T and MaybeUninit have the same layout
+        unsafe { &mut *ptr::from_mut(self).cast() }
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl AsMut<[MaybeUninit]> for MaybeUninit<[T; N]> {
+    #[inline]
+    fn as_mut(&mut self) -> &mut [MaybeUninit] {
+        &mut *AsMut::<[MaybeUninit; N]>::as_mut(self)
+    }
+}
+
+#[stable(feature = "more_conversion_trait_impls", since = "CURRENT_RUSTC_VERSION")]
+impl From> for [MaybeUninit; N] {
+    #[inline]
+    fn from(arr: MaybeUninit<[T; N]>) -> Self {
+        arr.transpose()
+    }
+}
+
 impl [MaybeUninit; N] {
     /// Transposes a `[MaybeUninit; N]` into a `MaybeUninit<[T; N]>`.
     ///
diff --git a/tests/ui/suggestions/issue-71394-no-from-impl.stderr b/tests/ui/suggestions/issue-71394-no-from-impl.stderr
index 9e068c311ae0..b8a240f7f07c 100644
--- a/tests/ui/suggestions/issue-71394-no-from-impl.stderr
+++ b/tests/ui/suggestions/issue-71394-no-from-impl.stderr
@@ -13,7 +13,7 @@ LL |     let _: &[i8] = data.into();
              `[T; 6]` implements `From<(T, T, T, T, T, T)>`
              `[T; 7]` implements `From<(T, T, T, T, T, T, T)>`
              `[T; 8]` implements `From<(T, T, T, T, T, T, T, T)>`
-           and 6 others
+           and 7 others
    = note: required for `&[u8]` to implement `Into<&[i8]>`
 
 error: aborting due to 1 previous error

From 6de0591c0b62f23ebe8413de465540771ed80074 Mon Sep 17 00:00:00 2001
From: Manuel Drehwald 
Date: Sat, 7 Feb 2026 17:34:39 -0800
Subject: [PATCH 1430/1843] Split ol mapper into more specific to/kernel/from
 mapper and move init_all_rtls into global ctor

---
 .../src/builder/gpu_offload.rs                | 88 ++++++++++++-------
 .../codegen-llvm/gpu_offload/control_flow.rs  |  4 +-
 tests/codegen-llvm/gpu_offload/gpu_host.rs    | 53 ++++++-----
 3 files changed, 92 insertions(+), 53 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
index 16ab18a568e8..a60e2d1f0ab0 100644
--- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
+++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
@@ -1,12 +1,13 @@
 use std::ffi::CString;
 
+use bitflags::Flags;
 use llvm::Linkage::*;
 use rustc_abi::Align;
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods};
 use rustc_middle::bug;
-use rustc_middle::ty::offload_meta::OffloadMetadata;
+use rustc_middle::ty::offload_meta::{MappingFlags, OffloadMetadata};
 
 use crate::builder::Builder;
 use crate::common::CodegenCx;
@@ -28,10 +29,6 @@ pub(crate) struct OffloadGlobals<'ll> {
     pub mapper_fn_ty: &'ll llvm::Type,
 
     pub ident_t_global: &'ll llvm::Value,
-
-    // FIXME(offload): Drop this, once we fully automated our offload compilation pipeline, since
-    // LLVM will initialize them for us if it sees gpu kernels being registered.
-    pub init_rtls: &'ll llvm::Value,
 }
 
 impl<'ll> OffloadGlobals<'ll> {
@@ -42,9 +39,6 @@ impl<'ll> OffloadGlobals<'ll> {
         let (begin_mapper, _, end_mapper, mapper_fn_ty) = gen_tgt_data_mappers(cx);
         let ident_t_global = generate_at_one(cx);
 
-        let init_ty = cx.type_func(&[], cx.type_void());
-        let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty);
-
         // We want LLVM's openmp-opt pass to pick up and optimize this module, since it covers both
         // openmp and offload optimizations.
         llvm::add_module_flag_u32(cx.llmod(), llvm::ModuleFlagMergeBehavior::Max, "openmp", 51);
@@ -58,7 +52,6 @@ impl<'ll> OffloadGlobals<'ll> {
             end_mapper,
             mapper_fn_ty,
             ident_t_global,
-            init_rtls,
         }
     }
 }
@@ -91,6 +84,11 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) {
     let atexit = cx.type_func(&[cx.type_ptr()], cx.type_i32());
     let atexit_fn = declare_offload_fn(cx, "atexit", atexit);
 
+    // FIXME(offload): Drop this, once we fully automated our offload compilation pipeline, since
+    // LLVM will initialize them for us if it sees gpu kernels being registered.
+    let init_ty = cx.type_func(&[], cx.type_void());
+    let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty);
+
     let desc_ty = cx.type_func(&[], cx.type_void());
     let reg_name = ".omp_offloading.descriptor_reg";
     let unreg_name = ".omp_offloading.descriptor_unreg";
@@ -104,12 +102,14 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) {
     // define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
     // entry:
     //   call void @__tgt_register_lib(ptr @.omp_offloading.descriptor)
+    //   call void @__tgt_init_all_rtls()
     //   %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
     //   ret void
     // }
     let bb = Builder::append_block(cx, desc_reg_fn, "entry");
     let mut a = Builder::build(cx, bb);
     a.call(reg_lib_decl, None, None, register_lib, &[omp_descriptor], None, None);
+    a.call(init_ty, None, None, init_rtls, &[], None, None);
     a.call(atexit, None, None, atexit_fn, &[desc_unreg_fn], None, None);
     a.ret_void();
 
@@ -345,7 +345,9 @@ impl KernelArgsTy {
 #[derive(Copy, Clone)]
 pub(crate) struct OffloadKernelGlobals<'ll> {
     pub offload_sizes: &'ll llvm::Value,
-    pub memtransfer_types: &'ll llvm::Value,
+    pub memtransfer_begin: &'ll llvm::Value,
+    pub memtransfer_kernel: &'ll llvm::Value,
+    pub memtransfer_end: &'ll llvm::Value,
     pub region_id: &'ll llvm::Value,
 }
 
@@ -423,18 +425,38 @@ pub(crate) fn gen_define_handling<'ll>(
 
     let offload_entry_ty = offload_globals.offload_entry_ty;
 
-    // FIXME(Sa4dUs): add `OMP_MAP_TARGET_PARAM = 0x20` only if necessary
     let (sizes, transfer): (Vec<_>, Vec<_>) =
-        metadata.iter().map(|m| (m.payload_size, m.mode.bits() | 0x20)).unzip();
+        metadata.iter().map(|m| (m.payload_size, m.mode)).unzip();
+    // Our begin mapper should only see simplified information about which args have to be
+    // transferred to the device, the end mapper only about which args should be transferred back.
+    // Any information beyond that makes it harder for LLVM's opt pass to evaluate whether it can
+    // safely move (=optimize) the LLVM-IR location of this data transfer. Only the mapping types
+    // mentioned below are handled, so make sure that we don't generate any other ones.
+    let handled_mappings = MappingFlags::TO
+        | MappingFlags::FROM
+        | MappingFlags::TARGET_PARAM
+        | MappingFlags::LITERAL
+        | MappingFlags::IMPLICIT;
+    for arg in &transfer {
+        debug_assert!(!arg.contains_unknown_bits());
+        debug_assert!(handled_mappings.contains(*arg));
+    }
+
+    let valid_begin_mappings = MappingFlags::TO | MappingFlags::LITERAL | MappingFlags::IMPLICIT;
+    let transfer_to: Vec =
+        transfer.iter().map(|m| m.intersection(valid_begin_mappings).bits()).collect();
+    let transfer_from: Vec =
+        transfer.iter().map(|m| m.intersection(MappingFlags::FROM).bits()).collect();
+    // FIXME(offload): add `OMP_MAP_TARGET_PARAM = 0x20` only if necessary
+    let transfer_kernel = vec![MappingFlags::TARGET_PARAM.bits(); transfer_to.len()];
 
     let offload_sizes = add_priv_unnamed_arr(&cx, &format!(".offload_sizes.{symbol}"), &sizes);
-    // Here we figure out whether something needs to be copied to the gpu (=1), from the gpu (=2),
-    // or both to and from the gpu (=3). Other values shouldn't affect us for now.
-    // A non-mutable reference or pointer will be 1, an array that's not read, but fully overwritten
-    // will be 2. For now, everything is 3, until we have our frontend set up.
-    // 1+2+32: 1 (MapTo), 2 (MapFrom), 32 (Add one extra input ptr per function, to be used later).
-    let memtransfer_types =
-        add_priv_unnamed_arr(&cx, &format!(".offload_maptypes.{symbol}"), &transfer);
+    let memtransfer_begin =
+        add_priv_unnamed_arr(&cx, &format!(".offload_maptypes.{symbol}.begin"), &transfer_to);
+    let memtransfer_kernel =
+        add_priv_unnamed_arr(&cx, &format!(".offload_maptypes.{symbol}.kernel"), &transfer_kernel);
+    let memtransfer_end =
+        add_priv_unnamed_arr(&cx, &format!(".offload_maptypes.{symbol}.end"), &transfer_from);
 
     // Next: For each function, generate these three entries. A weak constant,
     // the llvm.rodata entry name, and  the llvm_offload_entries value
@@ -469,7 +491,13 @@ pub(crate) fn gen_define_handling<'ll>(
 
     cx.add_compiler_used_global(offload_entry);
 
-    let result = OffloadKernelGlobals { offload_sizes, memtransfer_types, region_id };
+    let result = OffloadKernelGlobals {
+        offload_sizes,
+        memtransfer_begin,
+        memtransfer_kernel,
+        memtransfer_end,
+        region_id,
+    };
 
     // FIXME(Sa4dUs): use this global for constant offload sizes
     cx.add_compiler_used_global(result.offload_sizes);
@@ -535,7 +563,13 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
     offload_dims: &OffloadKernelDims<'ll>,
 ) {
     let cx = builder.cx;
-    let OffloadKernelGlobals { memtransfer_types, region_id, .. } = offload_data;
+    let OffloadKernelGlobals {
+        memtransfer_begin,
+        memtransfer_kernel,
+        memtransfer_end,
+        region_id,
+        ..
+    } = offload_data;
     let OffloadKernelDims { num_workgroups, threads_per_block, workgroup_dims, thread_dims } =
         offload_dims;
 
@@ -608,12 +642,6 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
         geps.push(gep);
     }
 
-    let init_ty = cx.type_func(&[], cx.type_void());
-    let init_rtls_decl = offload_globals.init_rtls;
-
-    // call void @__tgt_init_all_rtls()
-    builder.call(init_ty, None, None, init_rtls_decl, &[], None, None);
-
     for i in 0..num_args {
         let idx = cx.get_const_i32(i);
         let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, idx]);
@@ -668,14 +696,14 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
     generate_mapper_call(
         builder,
         geps,
-        memtransfer_types,
+        memtransfer_begin,
         begin_mapper_decl,
         fn_ty,
         num_args,
         s_ident_t,
     );
     let values =
-        KernelArgsTy::new(&cx, num_args, memtransfer_types, geps, workgroup_dims, thread_dims);
+        KernelArgsTy::new(&cx, num_args, memtransfer_kernel, geps, workgroup_dims, thread_dims);
 
     // Step 3)
     // Here we fill the KernelArgsTy, see the documentation above
@@ -701,7 +729,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
     generate_mapper_call(
         builder,
         geps,
-        memtransfer_types,
+        memtransfer_end,
         end_mapper_decl,
         fn_ty,
         num_args,
diff --git a/tests/codegen-llvm/gpu_offload/control_flow.rs b/tests/codegen-llvm/gpu_offload/control_flow.rs
index 1a3d3cd7a778..fb483db667b2 100644
--- a/tests/codegen-llvm/gpu_offload/control_flow.rs
+++ b/tests/codegen-llvm/gpu_offload/control_flow.rs
@@ -19,9 +19,9 @@
 // CHECK: br label %bb3
 // CHECK-NOT define
 // CHECK: bb3
-// CHECK: call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.foo, ptr null, ptr null)
+// CHECK: call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.foo.begin, ptr null, ptr null)
 // CHECK: %10 = call i32 @__tgt_target_kernel(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 256, i32 32, ptr nonnull @.foo.region_id, ptr nonnull %kernel_args)
-// CHECK-NEXT: call void @__tgt_target_data_end_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.foo, ptr null, ptr null)
+// CHECK-NEXT: call void @__tgt_target_data_end_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.foo.end, ptr null, ptr null)
 #[unsafe(no_mangle)]
 unsafe fn main() {
     let A = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0];
diff --git a/tests/codegen-llvm/gpu_offload/gpu_host.rs b/tests/codegen-llvm/gpu_offload/gpu_host.rs
index e12d7c792ae7..f25ba679abbd 100644
--- a/tests/codegen-llvm/gpu_offload/gpu_host.rs
+++ b/tests/codegen-llvm/gpu_offload/gpu_host.rs
@@ -14,19 +14,20 @@
 #[unsafe(no_mangle)]
 fn main() {
     let mut x = [3.0; 256];
-    kernel_1(&mut x);
+    let y = [1.0; 256];
+    kernel_1(&mut x, &y);
     core::hint::black_box(&x);
+    core::hint::black_box(&y);
 }
 
-pub fn kernel_1(x: &mut [f32; 256]) {
-    core::intrinsics::offload(kernel_1, [256, 1, 1], [32, 1, 1], (x,))
+pub fn kernel_1(x: &mut [f32; 256], y: &[f32; 256]) {
+    core::intrinsics::offload(_kernel_1, [256, 1, 1], [32, 1, 1], (x, y))
 }
 
-#[unsafe(no_mangle)]
 #[inline(never)]
-pub fn _kernel_1(x: &mut [f32; 256]) {
+pub fn _kernel_1(x: &mut [f32; 256], y: &[f32; 256]) {
     for i in 0..256 {
-        x[i] = 21.0;
+        x[i] = 21.0 + y[i];
     }
 }
 
@@ -39,8 +40,10 @@ pub fn _kernel_1(x: &mut [f32; 256]) {
 
 // CHECK-DAG: @.omp_offloading.descriptor = internal constant { i32, ptr, ptr, ptr } zeroinitializer
 // CHECK-DAG: @llvm.global_ctors = appending constant [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }]
-// CHECK-DAG: @.offload_sizes.[[K:[^ ]*kernel_1]] = private unnamed_addr constant [1 x i64] [i64 1024]
-// CHECK-DAG: @.offload_maptypes.[[K]] = private unnamed_addr constant [1 x i64] [i64 35]
+// CHECK-DAG: @.offload_sizes.[[K:[^ ]*kernel_1]] = private unnamed_addr constant [2 x i64] [i64 1024, i64 1024]
+// CHECK-DAG: @.offload_maptypes.[[K]].begin = private unnamed_addr constant [2 x i64] [i64 1, i64 1]
+// CHECK-DAG: @.offload_maptypes.[[K]].kernel = private unnamed_addr constant [2 x i64] [i64 32, i64 32]
+// CHECK-DAG: @.offload_maptypes.[[K]].end = private unnamed_addr constant [2 x i64] [i64 2, i64 0]
 // CHECK-DAG: @.[[K]].region_id = internal constant i8 0
 // CHECK-DAG: @.offloading.entry_name.[[K]] = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[K]]{{\\00}}", section ".llvm.rodata.offloading", align 1
 // CHECK-DAG: @.offloading.entry.[[K]] = internal constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @.[[K]].region_id, ptr @.offloading.entry_name.[[K]], i64 0, i64 0, ptr null }, section "llvm_offload_entries", align 8
@@ -49,20 +52,27 @@ pub fn _kernel_1(x: &mut [f32; 256]) {
 
 // CHECK-LABEL: define{{( dso_local)?}} void @main()
 // CHECK-NEXT: start:
-// CHECK-NEXT:  %0 = alloca [8 x i8], align 8
-// CHECK-NEXT:  %x = alloca [1024 x i8], align 16
-// CHECK-NEXT:   %.offload_baseptrs = alloca [1 x ptr], align 8
-// CHECK-NEXT:   %.offload_ptrs = alloca [1 x ptr], align 8
-// CHECK-NEXT:   %.offload_sizes = alloca [1 x i64], align 8
+// CHECK-NEXT:   %0 = alloca [8 x i8], align 8
+// CHECK-NEXT:   %1 = alloca [8 x i8], align 8
+// CHECK-NEXT:   %y = alloca [1024 x i8], align 16
+// CHECK-NEXT:   %x = alloca [1024 x i8], align 16
+// CHECK-NEXT:   %.offload_baseptrs = alloca [2 x ptr], align 8
+// CHECK-NEXT:   %.offload_ptrs = alloca [2 x ptr], align 8
+// CHECK-NEXT:   %.offload_sizes = alloca [2 x i64], align 8
 // CHECK-NEXT:   %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
-// CHECK:        call void @__tgt_init_all_rtls()
-// CHECK-NEXT:   store ptr %x, ptr %.offload_baseptrs, align 8
+// CHECK:   store ptr %x, ptr %.offload_baseptrs, align 8
 // CHECK-NEXT:   store ptr %x, ptr %.offload_ptrs, align 8
 // CHECK-NEXT:   store i64 1024, ptr %.offload_sizes, align 8
-// CHECK-NEXT:   call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.[[K]], ptr null, ptr null)
+// CHECK-NEXT:   [[BPTRS_1:%.*]] = getelementptr inbounds nuw i8, ptr %.offload_baseptrs, i64 8
+// CHECK-NEXT:   store ptr %y, ptr [[BPTRS_1]], align 8
+// CHECK-NEXT:   [[PTRS_1:%.*]] = getelementptr inbounds nuw i8, ptr %.offload_ptrs, i64 8
+// CHECK-NEXT:   store ptr %y, ptr [[PTRS_1]], align 8
+// CHECK-NEXT:   [[SIZES_1:%.*]] = getelementptr inbounds nuw i8, ptr %.offload_sizes, i64 8
+// CHECK-NEXT:   store i64 1024, ptr [[SIZES_1]], align 8
+// CHECK-NEXT:   call void @__tgt_target_data_begin_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 2, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.[[K]].begin, ptr null, ptr null)
 // CHECK-NEXT:   store i32 3, ptr %kernel_args, align 8
 // CHECK-NEXT:   [[P4:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 4
-// CHECK-NEXT:   store i32 1, ptr [[P4]], align 4
+// CHECK-NEXT:   store i32 2, ptr [[P4]], align 4
 // CHECK-NEXT:   [[P8:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 8
 // CHECK-NEXT:   store ptr %.offload_baseptrs, ptr [[P8]], align 8
 // CHECK-NEXT:   [[P16:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 16
@@ -70,7 +80,7 @@ pub fn _kernel_1(x: &mut [f32; 256]) {
 // CHECK-NEXT:   [[P24:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 24
 // CHECK-NEXT:   store ptr %.offload_sizes, ptr [[P24]], align 8
 // CHECK-NEXT:   [[P32:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 32
-// CHECK-NEXT:   store ptr @.offload_maptypes.[[K]], ptr [[P32]], align 8
+// CHECK-NEXT:   store ptr @.offload_maptypes.[[K]].kernel, ptr [[P32]], align 8
 // CHECK-NEXT:   [[P40:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 40
 // CHECK-NEXT:   [[P72:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 72
 // CHECK-NEXT:   call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[P40]], i8 0, i64 32, i1 false)
@@ -81,9 +91,9 @@ pub fn _kernel_1(x: &mut [f32; 256]) {
 // CHECK-NEXT:   store i32 1, ptr [[P92]], align 4
 // CHECK-NEXT:   [[P96:%[^ ]+]] = getelementptr inbounds nuw i8, ptr %kernel_args, i64 96
 // CHECK-NEXT:   store i32 0, ptr [[P96]], align 8
-// CHECK-NEXT:   {{%[^ ]+}} = call i32 @__tgt_target_kernel(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 256, i32 32, ptr nonnull @.[[K]].region_id, ptr nonnull %kernel_args)
-// CHECK-NEXT:   call void @__tgt_target_data_end_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 1, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.[[K]], ptr null, ptr null)
-// CHECK:   ret void
+// CHECK-NEXT:   [[TGT_RET:%.*]] = call i32 @__tgt_target_kernel(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 256, i32 32, ptr nonnull @.[[K]].region_id, ptr nonnull %kernel_args)
+// CHECK-NEXT:   call void @__tgt_target_data_end_mapper(ptr nonnull @anon.{{.*}}.1, i64 -1, i32 2, ptr nonnull %.offload_baseptrs, ptr nonnull %.offload_ptrs, ptr nonnull %.offload_sizes, ptr nonnull @.offload_maptypes.[[K]].end, ptr null, ptr null)
+// CHECK:  ret void
 // CHECK-NEXT: }
 
 // CHECK: declare void @__tgt_register_lib(ptr) local_unnamed_addr
@@ -92,6 +102,7 @@ pub fn _kernel_1(x: &mut [f32; 256]) {
 // CHECK-LABEL: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   call void @__tgt_register_lib(ptr nonnull @.omp_offloading.descriptor)
+// CHECK-NEXT:   call void @__tgt_init_all_rtls()
 // CHECK-NEXT:   %0 = {{tail }}call i32 @atexit(ptr nonnull @.omp_offloading.descriptor_unreg)
 // CHECK-NEXT:   ret void
 // CHECK-NEXT: }

From 7c5ea7ffd648cb1170510a7b50b0294a30891ffd Mon Sep 17 00:00:00 2001
From: Ruiyang Wang 
Date: Sat, 7 Feb 2026 17:57:53 +0000
Subject: [PATCH 1431/1843] Fix bound var resolution for trait aliases

Handle DefKind::TraitAlias in resolve_bound_vars so that associated
item constraints and return type notation work through trait aliases.
---
 .../src/collect/resolve_bound_vars.rs              |  5 +++--
 .../return-type-notation/trait-alias.rs            | 13 +++++++++++++
 .../trait-alias-bound-vars.rs                      | 14 ++++++++++++++
 3 files changed, 30 insertions(+), 2 deletions(-)
 create mode 100644 tests/ui/associated-type-bounds/return-type-notation/trait-alias.rs
 create mode 100644 tests/ui/associated-type-bounds/trait-alias-bound-vars.rs

diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
index 26f79d374075..d4a01d26a1fc 100644
--- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
+++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
@@ -1695,7 +1695,8 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                 | DefKind::Union
                 | DefKind::Enum
                 | DefKind::TyAlias
-                | DefKind::Trait,
+                | DefKind::Trait
+                | DefKind::TraitAlias,
                 def_id,
             ) if depth == 0 => Some(def_id),
             _ => None,
@@ -1865,7 +1866,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
             if constraint.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation
             {
                 let bound_vars = if let Some(type_def_id) = type_def_id
-                    && self.tcx.def_kind(type_def_id) == DefKind::Trait
+                    && let DefKind::Trait | DefKind::TraitAlias = self.tcx.def_kind(type_def_id)
                     && let Some((mut bound_vars, assoc_fn)) = BoundVarContext::supertrait_hrtb_vars(
                         self.tcx,
                         type_def_id,
diff --git a/tests/ui/associated-type-bounds/return-type-notation/trait-alias.rs b/tests/ui/associated-type-bounds/return-type-notation/trait-alias.rs
new file mode 100644
index 000000000000..88f60a72afa1
--- /dev/null
+++ b/tests/ui/associated-type-bounds/return-type-notation/trait-alias.rs
@@ -0,0 +1,13 @@
+// Regression test for .
+//@ check-pass
+#![feature(return_type_notation, trait_alias)]
+
+trait Tr {
+    fn f() -> impl Sized;
+}
+
+trait Al = Tr;
+
+fn f>() {}
+
+fn main() {}
diff --git a/tests/ui/associated-type-bounds/trait-alias-bound-vars.rs b/tests/ui/associated-type-bounds/trait-alias-bound-vars.rs
new file mode 100644
index 000000000000..4644cee21008
--- /dev/null
+++ b/tests/ui/associated-type-bounds/trait-alias-bound-vars.rs
@@ -0,0 +1,14 @@
+// Check that we're successfully collecting bound vars behind trait aliases.
+// Regression test for .
+//@ check-pass
+//@ needs-rustc-debug-assertions
+#![feature(trait_alias)]
+
+trait A<'a> { type X; }
+trait B: for<'a> A<'a> {}
+trait C = B;
+
+fn f() where T: C {}
+fn g() where T: C A<'r>> {}
+
+fn main() {}

From 58e4a341b634f21cf164e7636f4b8d63432dae7b Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Thu, 5 Feb 2026 14:24:39 +1100
Subject: [PATCH 1432/1843] Move `try_get_cached` from `rustc_query_system` to
 `rustc_middle`.

That's the only place it's used, so it no longer needs to be `pub`.
---
 compiler/rustc_middle/src/query/inner.rs      | 21 ++++++++++++++++++-
 .../rustc_query_system/src/query/plumbing.rs  | 20 ------------------
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs
index 32fe0737ec12..594286bfa574 100644
--- a/compiler/rustc_middle/src/query/inner.rs
+++ b/compiler/rustc_middle/src/query/inner.rs
@@ -2,7 +2,7 @@
 //! `tcx.$query(..)` and its variations.
 
 use rustc_query_system::dep_graph::{DepKind, DepNodeKey};
-use rustc_query_system::query::{QueryCache, QueryMode, try_get_cached};
+use rustc_query_system::query::{QueryCache, QueryMode};
 use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
 
 use crate::dep_graph;
@@ -10,6 +10,25 @@ use crate::query::erase::{self, Erasable, Erased};
 use crate::query::plumbing::QueryVTable;
 use crate::ty::TyCtxt;
 
+/// Checks whether there is already a value for this key in the in-memory
+/// query cache, returning that value if present.
+///
+/// (Also performs some associated bookkeeping, if a value was found.)
+#[inline(always)]
+fn try_get_cached<'tcx, C>(tcx: TyCtxt<'tcx>, cache: &C, key: &C::Key) -> Option
+where
+    C: QueryCache,
+{
+    match cache.lookup(key) {
+        Some((value, index)) => {
+            tcx.prof.query_cache_hit(index.into());
+            tcx.dep_graph.read_index(index);
+            Some(value)
+        }
+        None => None,
+    }
+}
+
 /// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)`
 /// for all queries.
 #[inline(always)]
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 32c4f6e1c847..795d36c7b808 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -266,26 +266,6 @@ impl<'tcx> CycleError> {
     }
 }
 
-/// Checks whether there is already a value for this key in the in-memory
-/// query cache, returning that value if present.
-///
-/// (Also performs some associated bookkeeping, if a value was found.)
-#[inline(always)]
-pub fn try_get_cached(tcx: Tcx, cache: &C, key: &C::Key) -> Option
-where
-    C: QueryCache,
-    Tcx: DepContext,
-{
-    match cache.lookup(key) {
-        Some((value, index)) => {
-            tcx.profiler().query_cache_hit(index.into());
-            tcx.dep_graph().read_index(index);
-            Some(value)
-        }
-        None => None,
-    }
-}
-
 #[cold]
 #[inline(never)]
 fn cycle_error<'tcx, Q>(

From 25df79ea5d5a9cc03d13018d7a3f86190c35dc2c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Thu, 5 Feb 2026 15:15:13 +1100
Subject: [PATCH 1433/1843] Derive `Clone` for `QueryJob`.

Instead of writing it by hand.
---
 compiler/rustc_query_system/src/query/job.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index 50cb58f0b4d5..0fd50128b544 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -65,7 +65,7 @@ pub struct QueryJobInfo<'tcx> {
 }
 
 /// Represents an active query job.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct QueryJob<'tcx> {
     pub id: QueryJobId,
 
@@ -79,12 +79,6 @@ pub struct QueryJob<'tcx> {
     latch: Option>,
 }
 
-impl<'tcx> Clone for QueryJob<'tcx> {
-    fn clone(&self) -> Self {
-        Self { id: self.id, span: self.span, parent: self.parent, latch: self.latch.clone() }
-    }
-}
-
 impl<'tcx> QueryJob<'tcx> {
     /// Creates a new query job.
     #[inline]

From d3ec6a351a027a71bdfbd36429288c1357ccdbea Mon Sep 17 00:00:00 2001
From: Weixie Cui 
Date: Sun, 8 Feb 2026 11:40:56 +0800
Subject: [PATCH 1434/1843] fix: add continue

---
 compiler/rustc_borrowck/src/diagnostics/region_errors.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index ae389d1a6e10..6e35cf0ea28b 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -872,6 +872,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             for alias_ty in alias_tys {
                 if alias_ty.span.desugaring_kind().is_some() {
                     // Skip `async` desugaring `impl Future`.
+                    continue;
                 }
                 if let TyKind::TraitObject(_, lt) = alias_ty.kind {
                     if lt.kind == hir::LifetimeKind::ImplicitObjectLifetimeDefault {

From 6527b3404c7ce641d50f4bda7711903f30b3e34c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Sun, 8 Feb 2026 15:14:17 +1100
Subject: [PATCH 1435/1843] Move a lot of `rustc_query_system::plumbing` to
 `rustc_query_impl::execution` (part 1).

[Note: this commit conceptually moves 75% of a file to another location,
and leaves 25% of it behind. It's impossible to preserve all the git
history. To preserve git history of the moved 75%, in this commit we
rename the file and remove the 25% from it, leaving the code in an
incomplete (uncompilable) state. In the next commit we add back the
25% in the old location.]

We are in the process of eliminating `rustc_query_system`. Chunks of it
are unused by `rustc_middle`, and so can be moved into
`rustc_query_impl`. This commit does some of that.

Mostly it's just moving code from one file to a new file. There are a
couple of non-trivial changes.

- `QueryState` and `ActiveKeyStatus` must remain in `rustc_query_system`
  because they are used by `rustc_middle`. But their inherent methods
  are not used by `rustc_middle`. So these methods are moved and
  converted to free functions.

- The visibility of some things must increase. This includes `DepGraphData`
  and some of its methods, which are now used in `rustc_query_impl`.
  This is a bit annoying but seems hard to avoid.

What little is left behind in
`compiler/rustc_query_system/src/query/plumbing.rs` will be able to
moved into `rustc_query_impl` or `rustc_middle` in the future.
---
 .../src/execution.rs}                         | 284 +++++-------------
 compiler/rustc_query_impl/src/lib.rs          |  16 +-
 compiler/rustc_query_impl/src/plumbing.rs     |  11 +-
 .../rustc_query_system/src/dep_graph/graph.rs |  18 +-
 .../rustc_query_system/src/dep_graph/mod.rs   |   5 +-
 compiler/rustc_query_system/src/lib.rs        |   1 -
 compiler/rustc_query_system/src/query/job.rs  |   8 +-
 compiler/rustc_query_system/src/query/mod.rs  |   6 +-
 8 files changed, 100 insertions(+), 249 deletions(-)
 rename compiler/{rustc_query_system/src/query/plumbing.rs => rustc_query_impl/src/execution.rs} (73%)

diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_impl/src/execution.rs
similarity index 73%
rename from compiler/rustc_query_system/src/query/plumbing.rs
rename to compiler/rustc_query_impl/src/execution.rs
index 795d36c7b808..9fa41f741e08 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -1,133 +1,83 @@
-//! The implementation of the query system itself. This defines the macros that
-//! generate the actual methods on tcx which find and execute the provider,
-//! manage the caches, and so forth.
-
-use std::cell::Cell;
-use std::fmt::Debug;
 use std::hash::Hash;
 use std::mem;
 
-use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::hash_table::{self, Entry, HashTable};
-use rustc_data_structures::sharded::{self, Sharded};
+use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_data_structures::{outline, sync};
+use rustc_data_structures::{outline, sharded, sync};
 use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext};
+use rustc_query_system::query::{
+    ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryContext, QueryDispatcher,
+    QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, QueryMode, QueryStackDeferred,
+    QueryStackFrame, QueryState, incremental_verify_ich, report_cycle,
+};
 use rustc_span::{DUMMY_SP, Span};
-use tracing::instrument;
 
-use super::{QueryDispatcher, QueryStackDeferred, QueryStackFrameExtra};
-use crate::dep_graph::{
-    DepContext, DepGraphData, DepNode, DepNodeIndex, DepNodeKey, HasDepContext,
-};
-use crate::ich::StableHashingContext;
-use crate::query::caches::QueryCache;
-use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, report_cycle};
-use crate::query::{
-    CycleErrorHandling, QueryContext, QueryMap, QueryStackFrame, SerializedDepNodeIndex,
-};
+use crate::dep_graph::{DepContext, DepNode, DepNodeIndex};
 
 #[inline]
 fn equivalent_key(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
     move |x| x.0 == *k
 }
 
-/// For a particular query, keeps track of "active" keys, i.e. keys whose
-/// evaluation has started but has not yet finished successfully.
+/// Obtains the enclosed [`QueryJob`], or panics if this query evaluation
+/// was poisoned by a panic.
+fn expect_job<'tcx>(status: ActiveKeyStatus<'tcx>) -> QueryJob<'tcx> {
+    match status {
+        ActiveKeyStatus::Started(job) => job,
+        ActiveKeyStatus::Poisoned => {
+            panic!("job for query failed to start and was poisoned")
+        }
+    }
+}
+
+pub(crate) fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool {
+    state.active.lock_shards().all(|shard| shard.is_empty())
+}
+
+/// Internal plumbing for collecting the set of active jobs for this query.
 ///
-/// (Successful query evaluation for a key is represented by an entry in the
-/// query's in-memory cache.)
-pub struct QueryState<'tcx, K> {
-    active: Sharded)>>,
-}
+/// Should only be called from `gather_active_jobs`.
+pub(crate) fn gather_active_jobs_inner<'tcx, Qcx: Copy, K: Copy>(
+    state: &QueryState<'tcx, K>,
+    qcx: Qcx,
+    make_frame: fn(Qcx, K) -> QueryStackFrame>,
+    jobs: &mut QueryMap<'tcx>,
+    require_complete: bool,
+) -> Option<()> {
+    let mut active = Vec::new();
 
-/// For a particular query and key, tracks the status of a query evaluation
-/// that has started, but has not yet finished successfully.
-///
-/// (Successful query evaluation for a key is represented by an entry in the
-/// query's in-memory cache.)
-enum ActiveKeyStatus<'tcx> {
-    /// Some thread is already evaluating the query for this key.
-    ///
-    /// The enclosed [`QueryJob`] can be used to wait for it to finish.
-    Started(QueryJob<'tcx>),
-
-    /// The query panicked. Queries trying to wait on this will raise a fatal error which will
-    /// silently panic.
-    Poisoned,
-}
-
-impl<'tcx> ActiveKeyStatus<'tcx> {
-    /// Obtains the enclosed [`QueryJob`], or panics if this query evaluation
-    /// was poisoned by a panic.
-    fn expect_job(self) -> QueryJob<'tcx> {
-        match self {
-            Self::Started(job) => job,
-            Self::Poisoned => {
-                panic!("job for query failed to start and was poisoned")
+    // Helper to gather active jobs from a single shard.
+    let mut gather_shard_jobs = |shard: &HashTable<(K, ActiveKeyStatus<'tcx>)>| {
+        for (k, v) in shard.iter() {
+            if let ActiveKeyStatus::Started(ref job) = *v {
+                active.push((*k, job.clone()));
             }
         }
-    }
-}
+    };
 
-impl<'tcx, K> QueryState<'tcx, K>
-where
-    K: Eq + Hash + Copy + Debug,
-{
-    pub fn all_inactive(&self) -> bool {
-        self.active.lock_shards().all(|shard| shard.is_empty())
-    }
-
-    /// Internal plumbing for collecting the set of active jobs for this query.
-    ///
-    /// Should only be called from `gather_active_jobs`.
-    pub fn gather_active_jobs_inner(
-        &self,
-        qcx: Qcx,
-        make_frame: fn(Qcx, K) -> QueryStackFrame>,
-        jobs: &mut QueryMap<'tcx>,
-        require_complete: bool,
-    ) -> Option<()> {
-        let mut active = Vec::new();
-
-        // Helper to gather active jobs from a single shard.
-        let mut gather_shard_jobs = |shard: &HashTable<(K, ActiveKeyStatus<'tcx>)>| {
-            for (k, v) in shard.iter() {
-                if let ActiveKeyStatus::Started(ref job) = *v {
-                    active.push((*k, job.clone()));
-                }
-            }
-        };
-
-        // Lock shards and gather jobs from each shard.
-        if require_complete {
-            for shard in self.active.lock_shards() {
-                gather_shard_jobs(&shard);
-            }
-        } else {
-            // We use try_lock_shards here since we are called from the
-            // deadlock handler, and this shouldn't be locked.
-            for shard in self.active.try_lock_shards() {
-                let shard = shard?;
-                gather_shard_jobs(&shard);
-            }
+    // Lock shards and gather jobs from each shard.
+    if require_complete {
+        for shard in state.active.lock_shards() {
+            gather_shard_jobs(&shard);
         }
-
-        // Call `make_frame` while we're not holding a `self.active` lock as `make_frame` may call
-        // queries leading to a deadlock.
-        for (key, job) in active {
-            let frame = make_frame(qcx, key);
-            jobs.insert(job.id, QueryJobInfo { frame, job });
+    } else {
+        // We use try_lock_shards here since we are called from the
+        // deadlock handler, and this shouldn't be locked.
+        for shard in state.active.try_lock_shards() {
+            let shard = shard?;
+            gather_shard_jobs(&shard);
         }
-
-        Some(())
     }
-}
 
-impl<'tcx, K> Default for QueryState<'tcx, K> {
-    fn default() -> QueryState<'tcx, K> {
-        QueryState { active: Default::default() }
+    // Call `make_frame` while we're not holding a `state.active` lock as `make_frame` may call
+    // queries leading to a deadlock.
+    for (key, job) in active {
+        let frame = make_frame(qcx, key);
+        jobs.insert(job.id, QueryJobInfo { frame, job });
     }
+
+    Some(())
 }
 
 /// A type representing the responsibility to execute the job in the `job` field.
@@ -217,7 +167,7 @@ where
                 Ok(occupied) => Some(occupied.remove().0.1),
             }
         };
-        let job = job.expect("active query job entry").expect_job();
+        let job = expect_job(job.expect("active query job entry"));
 
         job.signal_complete();
     }
@@ -240,7 +190,7 @@ where
                 Ok(occupied) => {
                     let ((key, value), vacant) = occupied.remove();
                     vacant.insert((key, ActiveKeyStatus::Poisoned));
-                    value.expect_job()
+                    expect_job(value)
                 }
             }
         };
@@ -250,22 +200,6 @@ where
     }
 }
 
-#[derive(Clone, Debug)]
-pub struct CycleError {
-    /// The query and related span that uses the cycle.
-    pub usage: Option<(Span, QueryStackFrame)>,
-    pub cycle: Vec>,
-}
-
-impl<'tcx> CycleError> {
-    fn lift(&self) -> CycleError {
-        CycleError {
-            usage: self.usage.as_ref().map(|(span, frame)| (*span, frame.lift())),
-            cycle: self.cycle.iter().map(|info| info.lift()).collect(),
-        }
-    }
-}
-
 #[cold]
 #[inline(never)]
 fn cycle_error<'tcx, Q>(
@@ -664,89 +598,6 @@ where
     Some((result, dep_node_index))
 }
 
-#[inline]
-#[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")]
-pub(crate) fn incremental_verify_ich(
-    tcx: Tcx,
-    dep_graph_data: &DepGraphData,
-    result: &V,
-    prev_index: SerializedDepNodeIndex,
-    hash_result: Option, &V) -> Fingerprint>,
-    format_value: fn(&V) -> String,
-) where
-    Tcx: DepContext,
-{
-    if !dep_graph_data.is_index_green(prev_index) {
-        incremental_verify_ich_not_green(tcx, prev_index)
-    }
-
-    let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
-        tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
-    });
-
-    let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
-
-    if new_hash != old_hash {
-        incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result));
-    }
-}
-
-#[cold]
-#[inline(never)]
-fn incremental_verify_ich_not_green(tcx: Tcx, prev_index: SerializedDepNodeIndex)
-where
-    Tcx: DepContext,
-{
-    panic!(
-        "fingerprint for green query instance not loaded from cache: {:?}",
-        tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
-    )
-}
-
-// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`,
-// as we want to avoid generating a bunch of different implementations for LLVM to
-// chew on (and filling up the final binary, too).
-#[cold]
-#[inline(never)]
-fn incremental_verify_ich_failed(
-    tcx: Tcx,
-    prev_index: SerializedDepNodeIndex,
-    result: &dyn Fn() -> String,
-) where
-    Tcx: DepContext,
-{
-    // When we emit an error message and panic, we try to debug-print the `DepNode`
-    // and query result. Unfortunately, this can cause us to run additional queries,
-    // which may result in another fingerprint mismatch while we're in the middle
-    // of processing this one. To avoid a double-panic (which kills the process
-    // before we can print out the query static), we print out a terse
-    // but 'safe' message if we detect a reentrant call to this method.
-    thread_local! {
-        static INSIDE_VERIFY_PANIC: Cell = const { Cell::new(false) };
-    };
-
-    let old_in_panic = INSIDE_VERIFY_PANIC.replace(true);
-
-    if old_in_panic {
-        tcx.sess().dcx().emit_err(crate::error::Reentrant);
-    } else {
-        let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
-            format!("`cargo clean -p {crate_name}` or `cargo clean`")
-        } else {
-            "`cargo clean`".to_string()
-        };
-
-        let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
-        tcx.sess().dcx().emit_err(crate::error::IncrementCompilation {
-            run_cmd,
-            dep_node: format!("{dep_node:?}"),
-        });
-        panic!("Found unstable fingerprints for {dep_node:?}: {}", result());
-    }
-
-    INSIDE_VERIFY_PANIC.set(old_in_panic);
-}
-
 /// Ensure that either this query has all green inputs or been executed.
 /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
 /// Returns true if the query should still run.
@@ -801,14 +652,13 @@ where
     (!loadable, Some(dep_node))
 }
 
-#[derive(Debug)]
-pub enum QueryMode {
-    Get,
-    Ensure { check_cache: bool },
-}
-
 #[inline(always)]
-pub fn get_query_non_incr<'tcx, Q>(query: Q, qcx: Q::Qcx, span: Span, key: Q::Key) -> Q::Value
+pub(super) fn get_query_non_incr<'tcx, Q>(
+    query: Q,
+    qcx: Q::Qcx,
+    span: Span,
+    key: Q::Key,
+) -> Q::Value
 where
     Q: QueryDispatcher<'tcx>,
 {
@@ -818,7 +668,7 @@ where
 }
 
 #[inline(always)]
-pub fn get_query_incr<'tcx, Q>(
+pub(super) fn get_query_incr<'tcx, Q>(
     query: Q,
     qcx: Q::Qcx,
     span: Span,
@@ -848,7 +698,7 @@ where
     Some(result)
 }
 
-pub fn force_query<'tcx, Q>(query: Q, qcx: Q::Qcx, key: Q::Key, dep_node: DepNode)
+pub(super) fn force_query<'tcx, Q>(query: Q, qcx: Q::Qcx, key: Q::Key, dep_node: DepNode)
 where
     Q: QueryDispatcher<'tcx>,
 {
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 19d43e9bb93d..228d0070b55b 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -3,6 +3,7 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![feature(adt_const_params)]
+#![feature(core_intrinsics)]
 #![feature(min_specialization)]
 #![feature(rustc_attrs)]
 // tidy-alphabetical-end
@@ -25,21 +26,20 @@ use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{
     CycleError, CycleErrorHandling, HashResult, QueryCache, QueryDispatcher, QueryMap, QueryMode,
-    QueryState, get_query_incr, get_query_non_incr,
+    QueryState,
 };
 use rustc_span::{ErrorGuaranteed, Span};
 
+pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all};
 use crate::plumbing::{encode_all_query_results, try_mark_green};
 use crate::profiling_support::QueryKeyStringCache;
-
-#[macro_use]
-mod plumbing;
-pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all};
-
-mod profiling_support;
-pub use self::profiling_support::alloc_self_profile_query_strings;
+pub use crate::profiling_support::alloc_self_profile_query_strings;
 
 mod error;
+mod execution;
+#[macro_use]
+mod plumbing;
+mod profiling_support;
 
 #[derive(ConstParamTy)] // Allow this struct to be used for const-generic values.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index a71acae3096e..8738370dd1e7 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -29,13 +29,14 @@ use rustc_query_system::dep_graph::{DepNodeKey, FingerprintStyle, HasDepContext}
 use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{
     QueryCache, QueryContext, QueryDispatcher, QueryJobId, QueryMap, QuerySideEffect,
-    QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra, force_query,
+    QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra,
 };
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::def_id::LOCAL_CRATE;
 
 use crate::QueryDispatcherUnerased;
 use crate::error::{QueryOverflow, QueryOverflowNote};
+use crate::execution::{all_inactive, force_query};
 
 /// Implements [`QueryContext`] for use by [`rustc_query_system`], since that
 /// crate does not have direct access to [`TyCtxt`].
@@ -387,7 +388,7 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>(
 {
     let _timer = qcx.tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name());
 
-    assert!(query.query_state(qcx).all_inactive());
+    assert!(all_inactive(query.query_state(qcx)));
     let cache = query.query_cache(qcx);
     cache.iter(&mut |key, value, dep_node| {
         if query.will_cache_on_disk_for_key(qcx.tcx, key) {
@@ -594,7 +595,7 @@ macro_rules! define_queries {
                 ) -> Option>> {
                     #[cfg(debug_assertions)]
                     let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
-                    get_query_incr(
+                    execution::get_query_incr(
                         QueryType::query_dispatcher(tcx),
                         QueryCtxt::new(tcx),
                         span,
@@ -614,7 +615,7 @@ macro_rules! define_queries {
                     key: queries::$name::Key<'tcx>,
                     __mode: QueryMode,
                 ) -> Option>> {
-                    Some(get_query_non_incr(
+                    Some(execution::get_query_non_incr(
                         QueryType::query_dispatcher(tcx),
                         QueryCtxt::new(tcx),
                         span,
@@ -748,7 +749,7 @@ macro_rules! define_queries {
                 };
 
                 // Call `gather_active_jobs_inner` to do the actual work.
-                let res = tcx.query_system.states.$name.gather_active_jobs_inner(
+                let res = crate::execution::gather_active_jobs_inner(&tcx.query_system.states.$name,
                     tcx,
                     make_frame,
                     qmap,
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 06e576baf22a..31aafaea6450 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -82,7 +82,7 @@ pub(super) enum DepNodeColor {
     Unknown,
 }
 
-pub(crate) struct DepGraphData {
+pub struct DepGraphData {
     /// The new encoding of the dependency graph, optimized for red/green
     /// tracking. The `current` field is the dependency graph of only the
     /// current compilation session: We don't merge the previous dep-graph into
@@ -171,7 +171,7 @@ impl DepGraph {
     }
 
     #[inline]
-    pub(crate) fn data(&self) -> Option<&DepGraphData> {
+    pub fn data(&self) -> Option<&DepGraphData> {
         self.data.as_deref()
     }
 
@@ -323,7 +323,7 @@ impl DepGraphData {
     ///
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/queries/incremental-compilation.html
     #[inline(always)]
-    pub(crate) fn with_task, A: Debug, R>(
+    pub fn with_task, A: Debug, R>(
         &self,
         key: DepNode,
         cx: Ctxt,
@@ -373,7 +373,7 @@ impl DepGraphData {
     /// FIXME: This could perhaps return a `WithDepNode` to ensure that the
     /// user of this function actually performs the read; we'll have to see
     /// how to make that work with `anon` in `execute_job_incr`, though.
-    pub(crate) fn with_anon_task_inner, OP, R>(
+    pub fn with_anon_task_inner, OP, R>(
         &self,
         cx: Tcx,
         dep_kind: DepKind,
@@ -653,12 +653,12 @@ impl DepGraphData {
     /// Returns true if the given node has been marked as green during the
     /// current compilation session. Used in various assertions
     #[inline]
-    pub(crate) fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
+    pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
         matches!(self.colors.get(prev_index), DepNodeColor::Green(_))
     }
 
     #[inline]
-    pub(crate) fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
+    pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
         self.previous.fingerprint_by_index(prev_index)
     }
 
@@ -667,7 +667,7 @@ impl DepGraphData {
         self.previous.index_to_node(prev_index)
     }
 
-    pub(crate) fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) {
+    pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) {
         self.debug_loaded_from_disk.lock().insert(dep_node);
     }
 
@@ -863,7 +863,7 @@ impl DepGraphData {
     /// A node will have an index, when it's already been marked green, or when we can mark it
     /// green. This function will mark the current task as a reader of the specified node, when
     /// a node index can be found for that node.
-    pub(crate) fn try_mark_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>(
+    pub fn try_mark_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>(
         &self,
         qcx: Qcx,
         dep_node: &DepNode,
@@ -1074,7 +1074,7 @@ impl DepGraph {
         if let Some(data) = &self.data { data.current.encoder.finish(&data.current) } else { Ok(0) }
     }
 
-    pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex {
+    pub fn next_virtual_depnode_index(&self) -> DepNodeIndex {
         debug_assert!(self.data.is_none());
         let index = self.virtual_dep_node_index.fetch_add(1, Ordering::Relaxed);
         DepNodeIndex::from_u32(index)
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index f2cb01d5e9b4..da110ad2c77c 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -8,8 +8,9 @@ mod serialized;
 use std::panic;
 
 pub use dep_node::{DepKind, DepKindVTable, DepNode, DepNodeKey, WorkProductId};
-pub(crate) use graph::DepGraphData;
-pub use graph::{DepGraph, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap, hash_result};
+pub use graph::{
+    DepGraph, DepGraphData, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap, hash_result,
+};
 pub use query::DepGraphQuery;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::DynSync;
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index b61b9b71b528..7f6426e395f9 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -1,7 +1,6 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![feature(assert_matches)]
-#![feature(core_intrinsics)]
 #![feature(min_specialization)]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index 0fd50128b544..bf7c480b05dd 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -86,7 +86,7 @@ impl<'tcx> QueryJob<'tcx> {
         QueryJob { id, span, parent, latch: None }
     }
 
-    pub(super) fn latch(&mut self) -> QueryLatch<'tcx> {
+    pub fn latch(&mut self) -> QueryLatch<'tcx> {
         if self.latch.is_none() {
             self.latch = Some(QueryLatch::new());
         }
@@ -106,7 +106,7 @@ impl<'tcx> QueryJob<'tcx> {
 }
 
 impl QueryJobId {
-    pub(super) fn find_cycle_in_stack<'tcx>(
+    pub fn find_cycle_in_stack<'tcx>(
         &self,
         query_map: QueryMap<'tcx>,
         current_job: &Option,
@@ -182,7 +182,7 @@ struct QueryLatchInfo<'tcx> {
 }
 
 #[derive(Debug)]
-pub(super) struct QueryLatch<'tcx> {
+pub struct QueryLatch<'tcx> {
     info: Arc>>,
 }
 
@@ -200,7 +200,7 @@ impl<'tcx> QueryLatch<'tcx> {
     }
 
     /// Awaits for the query job to complete.
-    pub(super) fn wait_on(
+    pub fn wait_on(
         &self,
         qcx: impl QueryContext<'tcx>,
         query: Option,
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index dbf7395bd61a..3eea0717f409 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -15,8 +15,8 @@ use rustc_span::def_id::DefId;
 pub use self::caches::{DefIdCache, DefaultCache, QueryCache, SingleCache, VecCache};
 pub use self::dispatcher::{HashResult, QueryDispatcher};
 pub use self::job::{
-    QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryMap, break_query_cycles, print_query_stack,
-    report_cycle,
+    QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, break_query_cycles,
+    print_query_stack, report_cycle,
 };
 pub use self::plumbing::*;
 use crate::dep_graph::{DepKind, DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
@@ -84,7 +84,7 @@ impl<'tcx> QueryStackFrame> {
 #[derive(Clone, Debug)]
 pub struct QueryStackFrameExtra {
     pub description: String,
-    span: Option,
+    pub span: Option,
     pub def_kind: Option,
 }
 

From 563a2f04c7091335c508cddbf56b11f29b2a5ed5 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Sun, 8 Feb 2026 15:25:49 +1100
Subject: [PATCH 1436/1843] Move a lot of `rustc_query_system::plumbing` to
 `rustc_query_impl::execution` (part 2).

[Note: see the previous commit to understand this commit.]
---
 .../rustc_query_system/src/query/plumbing.rs  | 150 ++++++++++++++++++
 1 file changed, 150 insertions(+)
 create mode 100644 compiler/rustc_query_system/src/query/plumbing.rs

diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
new file mode 100644
index 000000000000..d7a24d613583
--- /dev/null
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -0,0 +1,150 @@
+use std::cell::Cell;
+use std::fmt::Debug;
+
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::hash_table::HashTable;
+use rustc_data_structures::sharded::Sharded;
+use rustc_span::Span;
+use tracing::instrument;
+
+use super::{QueryStackDeferred, QueryStackFrameExtra};
+use crate::dep_graph::{DepContext, DepGraphData};
+use crate::ich::StableHashingContext;
+use crate::query::job::{QueryInfo, QueryJob};
+use crate::query::{QueryStackFrame, SerializedDepNodeIndex};
+
+/// For a particular query, keeps track of "active" keys, i.e. keys whose
+/// evaluation has started but has not yet finished successfully.
+///
+/// (Successful query evaluation for a key is represented by an entry in the
+/// query's in-memory cache.)
+pub struct QueryState<'tcx, K> {
+    pub active: Sharded)>>,
+}
+
+/// For a particular query and key, tracks the status of a query evaluation
+/// that has started, but has not yet finished successfully.
+///
+/// (Successful query evaluation for a key is represented by an entry in the
+/// query's in-memory cache.)
+pub enum ActiveKeyStatus<'tcx> {
+    /// Some thread is already evaluating the query for this key.
+    ///
+    /// The enclosed [`QueryJob`] can be used to wait for it to finish.
+    Started(QueryJob<'tcx>),
+
+    /// The query panicked. Queries trying to wait on this will raise a fatal error which will
+    /// silently panic.
+    Poisoned,
+}
+
+impl<'tcx, K> Default for QueryState<'tcx, K> {
+    fn default() -> QueryState<'tcx, K> {
+        QueryState { active: Default::default() }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct CycleError {
+    /// The query and related span that uses the cycle.
+    pub usage: Option<(Span, QueryStackFrame)>,
+    pub cycle: Vec>,
+}
+
+impl<'tcx> CycleError> {
+    pub fn lift(&self) -> CycleError {
+        CycleError {
+            usage: self.usage.as_ref().map(|(span, frame)| (*span, frame.lift())),
+            cycle: self.cycle.iter().map(|info| info.lift()).collect(),
+        }
+    }
+}
+
+#[inline]
+#[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")]
+pub fn incremental_verify_ich(
+    tcx: Tcx,
+    dep_graph_data: &DepGraphData,
+    result: &V,
+    prev_index: SerializedDepNodeIndex,
+    hash_result: Option, &V) -> Fingerprint>,
+    format_value: fn(&V) -> String,
+) where
+    Tcx: DepContext,
+{
+    if !dep_graph_data.is_index_green(prev_index) {
+        incremental_verify_ich_not_green(tcx, prev_index)
+    }
+
+    let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
+        tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
+    });
+
+    let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
+
+    if new_hash != old_hash {
+        incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result));
+    }
+}
+
+#[cold]
+#[inline(never)]
+fn incremental_verify_ich_not_green(tcx: Tcx, prev_index: SerializedDepNodeIndex)
+where
+    Tcx: DepContext,
+{
+    panic!(
+        "fingerprint for green query instance not loaded from cache: {:?}",
+        tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
+    )
+}
+
+// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`,
+// as we want to avoid generating a bunch of different implementations for LLVM to
+// chew on (and filling up the final binary, too).
+#[cold]
+#[inline(never)]
+fn incremental_verify_ich_failed(
+    tcx: Tcx,
+    prev_index: SerializedDepNodeIndex,
+    result: &dyn Fn() -> String,
+) where
+    Tcx: DepContext,
+{
+    // When we emit an error message and panic, we try to debug-print the `DepNode`
+    // and query result. Unfortunately, this can cause us to run additional queries,
+    // which may result in another fingerprint mismatch while we're in the middle
+    // of processing this one. To avoid a double-panic (which kills the process
+    // before we can print out the query static), we print out a terse
+    // but 'safe' message if we detect a reentrant call to this method.
+    thread_local! {
+        static INSIDE_VERIFY_PANIC: Cell = const { Cell::new(false) };
+    };
+
+    let old_in_panic = INSIDE_VERIFY_PANIC.replace(true);
+
+    if old_in_panic {
+        tcx.sess().dcx().emit_err(crate::error::Reentrant);
+    } else {
+        let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
+            format!("`cargo clean -p {crate_name}` or `cargo clean`")
+        } else {
+            "`cargo clean`".to_string()
+        };
+
+        let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
+        tcx.sess().dcx().emit_err(crate::error::IncrementCompilation {
+            run_cmd,
+            dep_node: format!("{dep_node:?}"),
+        });
+        panic!("Found unstable fingerprints for {dep_node:?}: {}", result());
+    }
+
+    INSIDE_VERIFY_PANIC.set(old_in_panic);
+}
+
+#[derive(Debug)]
+pub enum QueryMode {
+    Get,
+    Ensure { check_cache: bool },
+}

From a6dd4d870ac48290294f7342366f2ff4cd031d97 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Sun, 8 Feb 2026 15:56:56 +1100
Subject: [PATCH 1437/1843] Remove now-unnecessary indirection.

The previous commit moved some code from `rustc_query_system`, which
doesn't have access to `TyCtxt` and `QueryCtxt`, to `rustc_query_impl`,
which does. We can now remove quite a bit of indirection.

- Three methods in `trait QueryContext` are no longer needed
  (`next_job_id`, `current_query_job`, `start_query`). As a result,
  `QueryCtxt`'s trait impls of these methods are changed to inherent
  methods.

- `qcx: Q::Qcx` parameters are simplified to `qcx: QueryCtxt<'tcx>`.

- `*qcx.dep_context()` occurrences are simplified to `qcx.tcx`, and
  things like `qcx.dep_context().profiler()` become `qcx.tcx.prof`.

- `DepGraphData<::Deps>` becomes
  `DepGraphData`.

In short, various layers of indirection and abstraction are cut away.
The resulting code is simpler, more concrete, and easier to understand.
It's a good demonstration of the benefits of eliminating
`rustc_query_system`, and there will be more to come.
---
 compiler/rustc_query_impl/src/execution.rs   | 149 +++++++++----------
 compiler/rustc_query_impl/src/plumbing.rs    |  99 ++++++------
 compiler/rustc_query_system/src/query/mod.rs |  10 --
 3 files changed, 120 insertions(+), 138 deletions(-)

diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
index 9fa41f741e08..e58e626a7dfa 100644
--- a/compiler/rustc_query_impl/src/execution.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -5,6 +5,8 @@ use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::{outline, sharded, sync};
 use rustc_errors::{Diag, FatalError, StashKey};
+use rustc_middle::dep_graph::DepsType;
+use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext};
 use rustc_query_system::query::{
     ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryContext, QueryDispatcher,
@@ -14,6 +16,7 @@ use rustc_query_system::query::{
 use rustc_span::{DUMMY_SP, Span};
 
 use crate::dep_graph::{DepContext, DepNode, DepNodeIndex};
+use crate::plumbing::QueryCtxt;
 
 #[inline]
 fn equivalent_key(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
@@ -38,10 +41,10 @@ pub(crate) fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool {
 /// Internal plumbing for collecting the set of active jobs for this query.
 ///
 /// Should only be called from `gather_active_jobs`.
-pub(crate) fn gather_active_jobs_inner<'tcx, Qcx: Copy, K: Copy>(
+pub(crate) fn gather_active_jobs_inner<'tcx, K: Copy>(
     state: &QueryState<'tcx, K>,
-    qcx: Qcx,
-    make_frame: fn(Qcx, K) -> QueryStackFrame>,
+    tcx: TyCtxt<'tcx>,
+    make_frame: fn(TyCtxt<'tcx>, K) -> QueryStackFrame>,
     jobs: &mut QueryMap<'tcx>,
     require_complete: bool,
 ) -> Option<()> {
@@ -73,7 +76,7 @@ pub(crate) fn gather_active_jobs_inner<'tcx, Qcx: Copy, K: Copy>(
     // Call `make_frame` while we're not holding a `state.active` lock as `make_frame` may call
     // queries leading to a deadlock.
     for (key, job) in active {
-        let frame = make_frame(qcx, key);
+        let frame = make_frame(tcx, key);
         jobs.insert(job.id, QueryJobInfo { frame, job });
     }
 
@@ -92,36 +95,36 @@ where
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<'tcx, Q>(query: Q, qcx: Q::Qcx, cycle_error: CycleError) -> Q::Value
+fn mk_cycle<'tcx, Q>(query: Q, qcx: QueryCtxt<'tcx>, cycle_error: CycleError) -> Q::Value
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
-    let error = report_cycle(qcx.dep_context().sess(), &cycle_error);
+    let error = report_cycle(qcx.tcx.sess, &cycle_error);
     handle_cycle_error(query, qcx, &cycle_error, error)
 }
 
 fn handle_cycle_error<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     cycle_error: &CycleError,
     error: Diag<'_>,
 ) -> Q::Value
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     match query.cycle_error_handling() {
         CycleErrorHandling::Error => {
             let guar = error.emit();
-            query.value_from_cycle_error(*qcx.dep_context(), cycle_error, guar)
+            query.value_from_cycle_error(qcx.tcx, cycle_error, guar)
         }
         CycleErrorHandling::Fatal => {
             error.emit();
-            qcx.dep_context().sess().dcx().abort_if_errors();
+            qcx.tcx.dcx().abort_if_errors();
             unreachable!()
         }
         CycleErrorHandling::DelayBug => {
             let guar = error.delay_as_bug();
-            query.value_from_cycle_error(*qcx.dep_context(), cycle_error, guar)
+            query.value_from_cycle_error(qcx.tcx, cycle_error, guar)
         }
         CycleErrorHandling::Stash => {
             let guar = if let Some(root) = cycle_error.cycle.first()
@@ -131,7 +134,7 @@ where
             } else {
                 error.emit()
             };
-            query.value_from_cycle_error(*qcx.dep_context(), cycle_error, guar)
+            query.value_from_cycle_error(qcx.tcx, cycle_error, guar)
         }
     }
 }
@@ -204,12 +207,12 @@ where
 #[inline(never)]
 fn cycle_error<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     try_execute: QueryJobId,
     span: Span,
 ) -> (Q::Value, Option)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     // Ensure there was no errors collecting all active jobs.
     // We need the complete map to ensure we find a cycle to break.
@@ -225,19 +228,19 @@ where
 #[inline(always)]
 fn wait_for_query<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     span: Span,
     key: Q::Key,
     latch: QueryLatch<'tcx>,
     current: Option,
 ) -> (Q::Value, Option)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     // For parallel queries, we'll block and wait until the query running
     // in another thread has completed. Record how long we wait in the
     // self-profiler.
-    let query_blocked_prof_timer = qcx.dep_context().profiler().query_blocked();
+    let query_blocked_prof_timer = qcx.tcx.prof.query_blocked();
 
     // With parallel queries we might just have to wait on some other
     // thread.
@@ -262,7 +265,7 @@ where
                 })
             };
 
-            qcx.dep_context().profiler().query_cache_hit(index.into());
+            qcx.tcx.prof.query_cache_hit(index.into());
             query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
 
             (v, Some(index))
@@ -274,13 +277,13 @@ where
 #[inline(never)]
 fn try_execute_query<'tcx, Q, const INCR: bool>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     span: Span,
     key: Q::Key,
     dep_node: Option,
 ) -> (Q::Value, Option)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     let state = query.query_state(qcx);
     let key_hash = sharded::make_hash(&key);
@@ -292,9 +295,9 @@ where
     // re-executing the query since `try_start` only checks that the query is not currently
     // executing, but another thread may have already completed the query and stores it result
     // in the query cache.
-    if qcx.dep_context().sess().threads() > 1 {
+    if qcx.tcx.sess.threads() > 1 {
         if let Some((value, index)) = query.query_cache(qcx).lookup(&key) {
-            qcx.dep_context().profiler().query_cache_hit(index.into());
+            qcx.tcx.prof.query_cache_hit(index.into());
             return (value, Some(index));
         }
     }
@@ -343,7 +346,7 @@ where
 #[inline(always)]
 fn execute_job<'tcx, Q, const INCR: bool>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     state: &'tcx QueryState<'tcx, Q::Key>,
     key: Q::Key,
     key_hash: u64,
@@ -351,22 +354,15 @@ fn execute_job<'tcx, Q, const INCR: bool>(
     dep_node: Option,
 ) -> (Q::Value, Option)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     // Use `JobOwner` so the query will be poisoned if executing it panics.
     let job_owner = JobOwner { state, key };
 
-    debug_assert_eq!(qcx.dep_context().dep_graph().is_fully_enabled(), INCR);
+    debug_assert_eq!(qcx.tcx.dep_graph.is_fully_enabled(), INCR);
 
     let (result, dep_node_index) = if INCR {
-        execute_job_incr(
-            query,
-            qcx,
-            qcx.dep_context().dep_graph().data().unwrap(),
-            key,
-            dep_node,
-            id,
-        )
+        execute_job_incr(query, qcx, qcx.tcx.dep_graph.data().unwrap(), key, dep_node, id)
     } else {
         execute_job_non_incr(query, qcx, key, id)
     };
@@ -395,7 +391,7 @@ where
                 // We have an inconsistency. This can happen if one of the two
                 // results is tainted by errors.
                 assert!(
-                    qcx.dep_context().sess().dcx().has_errors().is_some(),
+                    qcx.tcx.dcx().has_errors().is_some(),
                     "Computed query value for {:?}({:?}) is inconsistent with fed value,\n\
                         computed={:#?}\nfed={:#?}",
                     query.dep_kind(),
@@ -415,24 +411,24 @@ where
 #[inline(always)]
 fn execute_job_non_incr<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     key: Q::Key,
     job_id: QueryJobId,
 ) -> (Q::Value, DepNodeIndex)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
-    debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled());
+    debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled());
 
     // Fingerprint the key, just to assert that it doesn't
     // have anything we don't consider hashable
     if cfg!(debug_assertions) {
-        let _ = key.to_fingerprint(*qcx.dep_context());
+        let _ = key.to_fingerprint(qcx.tcx);
     }
 
-    let prof_timer = qcx.dep_context().profiler().query_provider();
+    let prof_timer = qcx.tcx.prof.query_provider();
     let result = qcx.start_query(job_id, query.depth_limit(), || query.compute(qcx, key));
-    let dep_node_index = qcx.dep_context().dep_graph().next_virtual_depnode_index();
+    let dep_node_index = qcx.tcx.dep_graph.next_virtual_depnode_index();
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
     // Similarly, fingerprint the result to assert that
@@ -451,19 +447,18 @@ where
 #[inline(always)]
 fn execute_job_incr<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
-    dep_graph_data: &DepGraphData<::Deps>,
+    qcx: QueryCtxt<'tcx>,
+    dep_graph_data: &DepGraphData,
     key: Q::Key,
     mut dep_node_opt: Option,
     job_id: QueryJobId,
 ) -> (Q::Value, DepNodeIndex)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     if !query.anon() && !query.eval_always() {
         // `to_dep_node` is expensive for some `DepKind`s.
-        let dep_node =
-            dep_node_opt.get_or_insert_with(|| query.construct_dep_node(*qcx.dep_context(), &key));
+        let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(qcx.tcx, &key));
 
         // The diagnostics for this query will be promoted to the current session during
         // `try_mark_green()`, so we can ignore them here.
@@ -474,20 +469,16 @@ where
         }
     }
 
-    let prof_timer = qcx.dep_context().profiler().query_provider();
+    let prof_timer = qcx.tcx.prof.query_provider();
 
     let (result, dep_node_index) = qcx.start_query(job_id, query.depth_limit(), || {
         if query.anon() {
-            return dep_graph_data.with_anon_task_inner(
-                *qcx.dep_context(),
-                query.dep_kind(),
-                || query.compute(qcx, key),
-            );
+            return dep_graph_data
+                .with_anon_task_inner(qcx.tcx, query.dep_kind(), || query.compute(qcx, key));
         }
 
         // `to_dep_node` is expensive for some `DepKind`s.
-        let dep_node =
-            dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key));
+        let dep_node = dep_node_opt.unwrap_or_else(|| query.construct_dep_node(qcx.tcx, &key));
 
         dep_graph_data.with_task(
             dep_node,
@@ -506,13 +497,13 @@ where
 #[inline(always)]
 fn try_load_from_disk_and_cache_in_memory<'tcx, Q>(
     query: Q,
-    dep_graph_data: &DepGraphData<::Deps>,
-    qcx: Q::Qcx,
+    dep_graph_data: &DepGraphData,
+    qcx: QueryCtxt<'tcx>,
     key: &Q::Key,
     dep_node: &DepNode,
 ) -> Option<(Q::Value, DepNodeIndex)>
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     // Note this function can be called concurrently from the same query
     // We must ensure that this is handled correctly.
@@ -524,7 +515,7 @@ where
     // First we try to load the result from the on-disk cache.
     // Some things are never cached on disk.
     if let Some(result) = query.try_load_from_disk(qcx, key, prev_dep_node_index, dep_node_index) {
-        if std::intrinsics::unlikely(qcx.dep_context().sess().opts.unstable_opts.query_dep_graph) {
+        if std::intrinsics::unlikely(qcx.tcx.sess.opts.unstable_opts.query_dep_graph) {
             dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
         }
 
@@ -538,10 +529,10 @@ where
         // give us some coverage of potential bugs though.
         let try_verify = prev_fingerprint.split().1.as_u64().is_multiple_of(32);
         if std::intrinsics::unlikely(
-            try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich,
+            try_verify || qcx.tcx.sess.opts.unstable_opts.incremental_verify_ich,
         ) {
             incremental_verify_ich(
-                *qcx.dep_context(),
+                qcx.tcx,
                 dep_graph_data,
                 &result,
                 prev_dep_node_index,
@@ -556,7 +547,7 @@ where
     // We always expect to find a cached result for things that
     // can be forced from `DepNode`.
     debug_assert!(
-        !query.will_cache_on_disk_for_key(*qcx.dep_context(), key)
+        !query.will_cache_on_disk_for_key(qcx.tcx, key)
             || !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
         "missing on-disk cache entry for {dep_node:?}"
     );
@@ -570,10 +561,10 @@ where
 
     // We could not load a result from the on-disk cache, so
     // recompute.
-    let prof_timer = qcx.dep_context().profiler().query_provider();
+    let prof_timer = qcx.tcx.prof.query_provider();
 
     // The dep-graph for this computation is already in-place.
-    let result = qcx.dep_context().dep_graph().with_ignore(|| query.compute(qcx, *key));
+    let result = qcx.tcx.dep_graph.with_ignore(|| query.compute(qcx, *key));
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -587,7 +578,7 @@ where
     // See issue #82920 for an example of a miscompilation that would get turned into
     // an ICE by this check
     incremental_verify_ich(
-        *qcx.dep_context(),
+        qcx.tcx,
         dep_graph_data,
         &result,
         prev_dep_node_index,
@@ -609,12 +600,12 @@ where
 #[inline(never)]
 fn ensure_must_run<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     key: &Q::Key,
     check_cache: bool,
 ) -> (bool, Option)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     if query.eval_always() {
         return (true, None);
@@ -623,9 +614,9 @@ where
     // Ensuring an anonymous query makes no sense
     assert!(!query.anon());
 
-    let dep_node = query.construct_dep_node(*qcx.dep_context(), key);
+    let dep_node = query.construct_dep_node(qcx.tcx, key);
 
-    let dep_graph = qcx.dep_context().dep_graph();
+    let dep_graph = &qcx.tcx.dep_graph;
     let serialized_dep_node_index = match dep_graph.try_mark_green(qcx, &dep_node) {
         None => {
             // A None return from `try_mark_green` means that this is either
@@ -638,7 +629,7 @@ where
         }
         Some((serialized_dep_node_index, dep_node_index)) => {
             dep_graph.read_index(dep_node_index);
-            qcx.dep_context().profiler().query_cache_hit(dep_node_index.into());
+            qcx.tcx.prof.query_cache_hit(dep_node_index.into());
             serialized_dep_node_index
         }
     };
@@ -655,14 +646,14 @@ where
 #[inline(always)]
 pub(super) fn get_query_non_incr<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     span: Span,
     key: Q::Key,
 ) -> Q::Value
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
-    debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled());
+    debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled());
 
     ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, None).0)
 }
@@ -670,15 +661,15 @@ where
 #[inline(always)]
 pub(super) fn get_query_incr<'tcx, Q>(
     query: Q,
-    qcx: Q::Qcx,
+    qcx: QueryCtxt<'tcx>,
     span: Span,
     key: Q::Key,
     mode: QueryMode,
 ) -> Option
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
-    debug_assert!(qcx.dep_context().dep_graph().is_fully_enabled());
+    debug_assert!(qcx.tcx.dep_graph.is_fully_enabled());
 
     let dep_node = if let QueryMode::Ensure { check_cache } = mode {
         let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache);
@@ -693,19 +684,19 @@ where
     let (result, dep_node_index) =
         ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, dep_node));
     if let Some(dep_node_index) = dep_node_index {
-        qcx.dep_context().dep_graph().read_index(dep_node_index)
+        qcx.tcx.dep_graph.read_index(dep_node_index)
     }
     Some(result)
 }
 
-pub(super) fn force_query<'tcx, Q>(query: Q, qcx: Q::Qcx, key: Q::Key, dep_node: DepNode)
+pub(super) fn force_query<'tcx, Q>(query: Q, qcx: QueryCtxt<'tcx>, key: Q::Key, dep_node: DepNode)
 where
-    Q: QueryDispatcher<'tcx>,
+    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
 {
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
     if let Some((_, index)) = query.query_cache(qcx).lookup(&key) {
-        qcx.dep_context().profiler().query_cache_hit(index.into());
+        qcx.tcx.prof.query_cache_hit(index.into());
         return;
     }
 
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 8738370dd1e7..676d01be3385 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -14,7 +14,8 @@ use rustc_hir::limit::Limit;
 use rustc_index::Idx;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::{
-    self, DepContext, DepKindVTable, DepNode, DepNodeIndex, SerializedDepNodeIndex, dep_kinds,
+    self, DepContext, DepKindVTable, DepNode, DepNodeIndex, DepsType, SerializedDepNodeIndex,
+    dep_kinds,
 };
 use rustc_middle::query::Key;
 use rustc_middle::query::on_disk_cache::{
@@ -69,10 +70,57 @@ impl<'tcx> QueryCtxt<'tcx> {
             crate_name: self.tcx.crate_name(LOCAL_CRATE),
         });
     }
+
+    #[inline]
+    pub(crate) fn next_job_id(self) -> QueryJobId {
+        QueryJobId(
+            NonZero::new(
+                self.tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
+            )
+            .unwrap(),
+        )
+    }
+
+    #[inline]
+    pub(crate) fn current_query_job(self) -> Option {
+        tls::with_related_context(self.tcx, |icx| icx.query)
+    }
+
+    /// Executes a job by changing the `ImplicitCtxt` to point to the
+    /// new query job while it executes.
+    #[inline(always)]
+    pub(crate) fn start_query(
+        self,
+        token: QueryJobId,
+        depth_limit: bool,
+        compute: impl FnOnce() -> R,
+    ) -> R {
+        // The `TyCtxt` stored in TLS has the same global interner lifetime
+        // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
+        // when accessing the `ImplicitCtxt`.
+        tls::with_related_context(self.tcx, move |current_icx| {
+            if depth_limit
+                && !self.tcx.recursion_limit().value_within_limit(current_icx.query_depth)
+            {
+                self.depth_limit_error(token);
+            }
+
+            // Update the `ImplicitCtxt` to point to our new query job.
+            let new_icx = ImplicitCtxt {
+                tcx: self.tcx,
+                query: Some(token),
+                query_depth: current_icx.query_depth + depth_limit as usize,
+                task_deps: current_icx.task_deps,
+            };
+
+            // Use the `ImplicitCtxt` while we execute the query.
+            tls::enter_context(&new_icx, compute)
+        })
+    }
 }
 
 impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
-    type Deps = rustc_middle::dep_graph::DepsType;
+    type Deps = DepsType;
     type DepContext = TyCtxt<'tcx>;
 
     #[inline]
@@ -87,21 +135,6 @@ impl<'tcx> QueryContext<'tcx> for QueryCtxt<'tcx> {
         &self.tcx.jobserver_proxy
     }
 
-    #[inline]
-    fn next_job_id(self) -> QueryJobId {
-        QueryJobId(
-            NonZero::new(
-                self.tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
-            )
-            .unwrap(),
-        )
-    }
-
-    #[inline]
-    fn current_query_job(self) -> Option {
-        tls::with_related_context(self.tcx, |icx| icx.query)
-    }
-
     /// Returns a map of currently active query jobs, collected from all queries.
     ///
     /// If `require_complete` is `true`, this function locks all shards of the
@@ -147,38 +180,6 @@ impl<'tcx> QueryContext<'tcx> for QueryCtxt<'tcx> {
             c.store_side_effect(dep_node_index, side_effect)
         }
     }
-
-    /// Executes a job by changing the `ImplicitCtxt` to point to the
-    /// new query job while it executes.
-    #[inline(always)]
-    fn start_query(
-        self,
-        token: QueryJobId,
-        depth_limit: bool,
-        compute: impl FnOnce() -> R,
-    ) -> R {
-        // The `TyCtxt` stored in TLS has the same global interner lifetime
-        // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
-        // when accessing the `ImplicitCtxt`.
-        tls::with_related_context(self.tcx, move |current_icx| {
-            if depth_limit
-                && !self.tcx.recursion_limit().value_within_limit(current_icx.query_depth)
-            {
-                self.depth_limit_error(token);
-            }
-
-            // Update the `ImplicitCtxt` to point to our new query job.
-            let new_icx = ImplicitCtxt {
-                tcx: self.tcx,
-                query: Some(token),
-                query_depth: current_icx.query_depth + depth_limit as usize,
-                task_deps: current_icx.task_deps,
-            };
-
-            // Use the `ImplicitCtxt` while we execute the query.
-            tls::enter_context(&new_icx, compute)
-        })
-    }
 }
 
 pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 3eea0717f409..29a572b726f0 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -161,11 +161,6 @@ pub trait QueryContext<'tcx>: HasDepContext {
     /// a token while waiting on a query.
     fn jobserver_proxy(&self) -> &Proxy;
 
-    fn next_job_id(self) -> QueryJobId;
-
-    /// Get the query information from the TLS context.
-    fn current_query_job(self) -> Option;
-
     fn collect_active_jobs_from_all_queries(
         self,
         require_complete: bool,
@@ -179,9 +174,4 @@ pub trait QueryContext<'tcx>: HasDepContext {
 
     /// Register a side effect for the given node, for use in next session.
     fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect);
-
-    /// Executes a job by changing the `ImplicitCtxt` to point to the
-    /// new query job while it executes.
-    fn start_query(self, token: QueryJobId, depth_limit: bool, compute: impl FnOnce() -> R)
-    -> R;
 }

From 118372ed1325f07ebc4ec9c3f28e3f00d4519a05 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sun, 8 Feb 2026 18:15:53 +1100
Subject: [PATCH 1438/1843] Replace some `feature(core_intrinsics)` with stable
 hints

---
 compiler/rustc_arena/src/lib.rs                   | 5 ++---
 compiler/rustc_data_structures/src/lib.rs         | 2 +-
 compiler/rustc_data_structures/src/profiling.rs   | 9 +++++----
 compiler/rustc_data_structures/src/sync/freeze.rs | 5 +++--
 compiler/rustc_data_structures/src/sync/lock.rs   | 9 +++++----
 5 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs
index 6217bf46a942..97dd21db07e7 100644
--- a/compiler/rustc_arena/src/lib.rs
+++ b/compiler/rustc_arena/src/lib.rs
@@ -13,7 +13,6 @@
 #![cfg_attr(test, feature(test))]
 #![deny(unsafe_op_in_unsafe_fn)]
 #![doc(test(no_crate_inject, attr(deny(warnings), allow(internal_features))))]
-#![feature(core_intrinsics)]
 #![feature(decl_macro)]
 #![feature(dropck_eyepatch)]
 #![feature(never_type)]
@@ -26,7 +25,7 @@ use std::cell::{Cell, RefCell};
 use std::marker::PhantomData;
 use std::mem::{self, MaybeUninit};
 use std::ptr::{self, NonNull};
-use std::{cmp, intrinsics, slice};
+use std::{cmp, hint, slice};
 
 use smallvec::SmallVec;
 
@@ -452,7 +451,7 @@ impl DroplessArena {
             let bytes = align_up(layout.size(), DROPLESS_ALIGNMENT);
 
             // Tell LLVM that `end` is aligned to DROPLESS_ALIGNMENT.
-            unsafe { intrinsics::assume(end == align_down(end, DROPLESS_ALIGNMENT)) };
+            unsafe { hint::assert_unchecked(end == align_down(end, DROPLESS_ALIGNMENT)) };
 
             if let Some(sub) = end.checked_sub(bytes) {
                 let new_end = align_down(sub, layout.align());
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 4467a2811181..bef19d77d1e7 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -10,6 +10,7 @@
 #![allow(internal_features)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
+#![cfg_attr(bootstrap, feature(cold_path))]
 #![deny(unsafe_op_in_unsafe_fn)]
 #![feature(allocator_api)]
 #![feature(ascii_char)]
@@ -19,7 +20,6 @@
 #![feature(cfg_select)]
 #![feature(const_default)]
 #![feature(const_trait_impl)]
-#![feature(core_intrinsics)]
 #![feature(dropck_eyepatch)]
 #![feature(extend_one)]
 #![feature(file_buffered)]
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index b04caa69adfb..cab57edb8946 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -85,12 +85,11 @@ use std::borrow::Borrow;
 use std::collections::hash_map::Entry;
 use std::error::Error;
 use std::fmt::Display;
-use std::intrinsics::unlikely;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::Ordering;
 use std::time::{Duration, Instant};
-use std::{fs, process};
+use std::{fs, hint, process};
 
 pub use measureme::EventId;
 use measureme::{EventIdBuilder, Profiler, SerializableString, StringId};
@@ -427,7 +426,8 @@ impl SelfProfilerRef {
                     .unwrap()
                     .increment_query_cache_hit_counters(QueryInvocationId(query_invocation_id.0));
             }
-            if unlikely(profiler_ref.event_filter_mask.contains(EventFilter::QUERY_CACHE_HITS)) {
+            if profiler_ref.event_filter_mask.contains(EventFilter::QUERY_CACHE_HITS) {
+                hint::cold_path();
                 profiler_ref.instant_query_event(
                     |profiler| profiler.query_cache_hit_event_kind,
                     query_invocation_id,
@@ -437,7 +437,8 @@ impl SelfProfilerRef {
 
         // We check both kinds of query cache hit events at once, to reduce overhead in the
         // common case (with self-profile disabled).
-        if unlikely(self.event_filter_mask.intersects(EventFilter::QUERY_CACHE_HIT_COMBINED)) {
+        if self.event_filter_mask.intersects(EventFilter::QUERY_CACHE_HIT_COMBINED) {
+            hint::cold_path();
             cold_call(self, query_invocation_id);
         }
     }
diff --git a/compiler/rustc_data_structures/src/sync/freeze.rs b/compiler/rustc_data_structures/src/sync/freeze.rs
index 6338afb92c34..fb2dcb25556a 100644
--- a/compiler/rustc_data_structures/src/sync/freeze.rs
+++ b/compiler/rustc_data_structures/src/sync/freeze.rs
@@ -1,5 +1,5 @@
 use std::cell::UnsafeCell;
-use std::intrinsics::likely;
+use std::hint;
 use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut};
 use std::ptr::NonNull;
@@ -60,10 +60,11 @@ impl FreezeLock {
     /// Get the inner value if frozen.
     #[inline]
     pub fn get(&self) -> Option<&T> {
-        if likely(self.frozen.load(Ordering::Acquire)) {
+        if self.frozen.load(Ordering::Acquire) {
             // SAFETY: This is frozen so the data cannot be modified.
             unsafe { Some(&*self.data.get()) }
         } else {
+            hint::cold_path();
             None
         }
     }
diff --git a/compiler/rustc_data_structures/src/sync/lock.rs b/compiler/rustc_data_structures/src/sync/lock.rs
index a8161c515115..f183af0c0dab 100644
--- a/compiler/rustc_data_structures/src/sync/lock.rs
+++ b/compiler/rustc_data_structures/src/sync/lock.rs
@@ -1,7 +1,7 @@
 //! This module implements a lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
 //! It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync` traits.
 
-use std::fmt;
+use std::{fmt, hint};
 
 #[derive(Clone, Copy, PartialEq)]
 pub enum Mode {
@@ -10,7 +10,6 @@ pub enum Mode {
 }
 
 use std::cell::{Cell, UnsafeCell};
-use std::intrinsics::unlikely;
 use std::marker::PhantomData;
 use std::mem::ManuallyDrop;
 use std::ops::{Deref, DerefMut};
@@ -92,7 +91,8 @@ pub struct Lock {
 impl Lock {
     #[inline(always)]
     pub fn new(inner: T) -> Self {
-        let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) {
+        let (mode, mode_union) = if mode::might_be_dyn_thread_safe() {
+            hint::cold_path();
             // Create the lock with synchronization enabled using the `RawMutex` type.
             (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) })
         } else {
@@ -150,7 +150,8 @@ impl Lock {
         unsafe {
             match mode {
                 Mode::NoSync => {
-                    if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) {
+                    if self.mode_union.no_sync.replace(LOCKED) == LOCKED {
+                        hint::cold_path();
                         lock_held()
                     }
                 }

From 7c95f57bc95c36f44a50fffb07cb2b933184ee2b Mon Sep 17 00:00:00 2001
From: mikhailofff 
Date: Sun, 8 Feb 2026 11:25:51 +0400
Subject: [PATCH 1439/1843] fix outdated doc comments

---
 book/src/development/type_checking.md | 2 +-
 tests/ui/unconditional_recursion.rs   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/book/src/development/type_checking.md b/book/src/development/type_checking.md
index 578836ecc568..d1c88619d598 100644
--- a/book/src/development/type_checking.md
+++ b/book/src/development/type_checking.md
@@ -154,7 +154,7 @@ in this chapter:
 [expr_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypeckResults.html#method.expr_ty
 [node_type]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypeckResults.html#method.node_type
 [is_char]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html#method.is_char
-[is_char_source]: https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_middle/ty/sty.rs.html#1831-1834
+[is_char_source]: https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_middle/ty/sty.rs.html#1429-1432
 [kind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html#method.kind
 [LateContext]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LateContext.html
 [LateLintPass]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/trait.LateLintPass.html
diff --git a/tests/ui/unconditional_recursion.rs b/tests/ui/unconditional_recursion.rs
index d9f4c07dc902..09a0de21ccd0 100644
--- a/tests/ui/unconditional_recursion.rs
+++ b/tests/ui/unconditional_recursion.rs
@@ -334,7 +334,7 @@ mod issue12154 {
     }
 
     // Not necessarily related to the issue but another FP from the http crate that was fixed with it:
-    // https://docs.rs/http/latest/src/http/header/name.rs.html#1424
+    // https://docs.rs/http/latest/src/http/header/name.rs.html#1408
     // We used to simply peel refs from the LHS and RHS, so we couldn't differentiate
     // between `PartialEq for &T` and `PartialEq<&T> for T` impls.
     #[derive(PartialEq)]

From 6b1ed212b70be56296d3aadba518028e6049eb56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Sun, 8 Feb 2026 09:16:31 +0100
Subject: [PATCH 1440/1843] Remove types adhoc group

---
 triagebot.toml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index 4a4054391ce7..43db49873c4d 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1536,13 +1536,6 @@ mir-opt = [
     "@wesleywiser",
     "@saethlin",
 ]
-types = [
-    "@jackh726",
-    "@lcnr",
-    "@oli-obk",
-    "@spastorino",
-    "@BoxyUwU",
-]
 borrowck = [
     "@davidtwco",
     "@matthewjasper"

From c19c780d4557f4d1e2ee285c08184e9c37070c0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Sun, 8 Feb 2026 09:22:24 +0100
Subject: [PATCH 1441/1843] Remove project-const-traits adhoc group

---
 triagebot.toml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index 43db49873c4d..d6106024a1fd 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1555,11 +1555,6 @@ style-team = [
     "@joshtriplett",
     "@traviscross",
 ]
-project-const-traits = [
-    "@fee1-dead",
-    "@fmease",
-    "@oli-obk",
-]
 project-stable-mir = [
     "@celinval",
     "@oli-obk",

From 5e33ea7b926a8ed4f6228588564e036fba792897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Sun, 8 Feb 2026 09:23:04 +0100
Subject: [PATCH 1442/1843] Remove project-stable-mir adhoc group

---
 triagebot.toml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index d6106024a1fd..f2e69b625a81 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1555,12 +1555,6 @@ style-team = [
     "@joshtriplett",
     "@traviscross",
 ]
-project-stable-mir = [
-    "@celinval",
-    "@oli-obk",
-    "@scottmcm",
-    "@makai410",
-]
 project-exploit-mitigations = [
     "@cuviper",
     "@rcvalle",

From f46032089a94f7185823b0b519529d50311ff489 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Sun, 8 Feb 2026 09:24:38 +0100
Subject: [PATCH 1443/1843] Remove project-exploit-mitigations adhoc group

---
 triagebot.toml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index f2e69b625a81..98e88f4e2e9b 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1555,10 +1555,6 @@ style-team = [
     "@joshtriplett",
     "@traviscross",
 ]
-project-exploit-mitigations = [
-    "@cuviper",
-    "@rcvalle",
-]
 compiletest = [
     "@jieyouxu",
 ]

From 5862618fab1bd3e6c904914db953bdf56720c28e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= 
Date: Sun, 8 Feb 2026 09:52:34 +0100
Subject: [PATCH 1444/1843] Remove the compiler adhoc group

---
 triagebot.toml | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index 4a4054391ce7..43f56a71cf4e 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -1449,29 +1449,6 @@ compiler_leads = [
     "@davidtwco",
     "@wesleywiser",
 ]
-compiler = [
-    "@BoxyUwU",
-    "@chenyukang",
-    "@davidtwco",
-    "@eholk",
-    "@fee1-dead",
-    "@fmease",
-    "@jackh726",
-    "@jieyouxu",
-    "@jdonszelmann",
-    "@JonathanBrouwer",
-    "@madsmtm",
-    "@mati865",
-    "@Nadrieril",
-    "@nnethercote",
-    "@oli-obk",
-    "@petrochenkov",
-    "@SparrowLii",
-    "@TaKO8Ki",
-    "@tiif",
-    "@WaffleLapkin",
-    "@wesleywiser",
-]
 libs = [
     "@Mark-Simulacrum",
     "@workingjubilee",

From 953aa57c7503177f494dfb168306503867aa26fc Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 10:23:31 +0100
Subject: [PATCH 1445/1843] Fix suggestion on `AutoTraitItems` diagnostic

---
 compiler/rustc_ast_passes/src/errors.rs      |  2 +-
 tests/ui/auto-traits/assoc-ty.current.stderr |  9 +++++----
 tests/ui/auto-traits/assoc-ty.next.stderr    | 11 ++++++-----
 tests/ui/auto-traits/assoc-ty.rs             |  5 ++++-
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index baf6f6beaeed..9f66641b550c 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -503,7 +503,7 @@ pub(crate) struct AutoTraitItems {
     #[primary_span]
     pub spans: Vec,
     #[suggestion(
-        "remove the super traits or lifetime bounds",
+        "remove the associated items",
         code = "",
         applicability = "machine-applicable",
         style = "tool-only"
diff --git a/tests/ui/auto-traits/assoc-ty.current.stderr b/tests/ui/auto-traits/assoc-ty.current.stderr
index d793ae665267..5d3df17d26b1 100644
--- a/tests/ui/auto-traits/assoc-ty.current.stderr
+++ b/tests/ui/auto-traits/assoc-ty.current.stderr
@@ -1,9 +1,9 @@
 error[E0380]: auto traits cannot have associated items
-  --> $DIR/assoc-ty.rs:10:10
+  --> $DIR/assoc-ty.rs:11:10
    |
 LL | auto trait Trait {
    |            ----- auto traits cannot have associated items
-LL |
+...
 LL |     type Output;
    |          ^^^^^^
 
@@ -12,8 +12,9 @@ error[E0658]: auto traits are experimental and possibly buggy
    |
 LL | / auto trait Trait {
 LL | |
-LL | |     type Output;
 LL | |
+LL | |     type Output;
+...  |
 LL | | }
    | |_^
    |
@@ -22,7 +23,7 @@ LL | | }
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
 error[E0308]: mismatched types
-  --> $DIR/assoc-ty.rs:15:36
+  --> $DIR/assoc-ty.rs:17:36
    |
 LL |     let _: <() as Trait>::Output = ();
    |            ---------------------   ^^ expected associated type, found `()`
diff --git a/tests/ui/auto-traits/assoc-ty.next.stderr b/tests/ui/auto-traits/assoc-ty.next.stderr
index a41f7d992785..19621064aa42 100644
--- a/tests/ui/auto-traits/assoc-ty.next.stderr
+++ b/tests/ui/auto-traits/assoc-ty.next.stderr
@@ -1,9 +1,9 @@
 error[E0380]: auto traits cannot have associated items
-  --> $DIR/assoc-ty.rs:10:10
+  --> $DIR/assoc-ty.rs:11:10
    |
 LL | auto trait Trait {
    |            ----- auto traits cannot have associated items
-LL |
+...
 LL |     type Output;
    |          ^^^^^^
 
@@ -12,8 +12,9 @@ error[E0658]: auto traits are experimental and possibly buggy
    |
 LL | / auto trait Trait {
 LL | |
-LL | |     type Output;
 LL | |
+LL | |     type Output;
+...  |
 LL | | }
    | |_^
    |
@@ -22,13 +23,13 @@ LL | | }
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
 error[E0271]: type mismatch resolving `<() as Trait>::Output normalizes-to _`
-  --> $DIR/assoc-ty.rs:15:12
+  --> $DIR/assoc-ty.rs:17:12
    |
 LL |     let _: <() as Trait>::Output = ();
    |            ^^^^^^^^^^^^^^^^^^^^^ types differ
 
 error[E0271]: type mismatch resolving `<() as Trait>::Output normalizes-to _`
-  --> $DIR/assoc-ty.rs:15:12
+  --> $DIR/assoc-ty.rs:17:12
    |
 LL |     let _: <() as Trait>::Output = ();
    |            ^^^^^^^^^^^^^^^^^^^^^ types differ
diff --git a/tests/ui/auto-traits/assoc-ty.rs b/tests/ui/auto-traits/assoc-ty.rs
index efbfead9cd03..98a86d9a7540 100644
--- a/tests/ui/auto-traits/assoc-ty.rs
+++ b/tests/ui/auto-traits/assoc-ty.rs
@@ -7,13 +7,16 @@
 
 auto trait Trait {
     //~^ ERROR auto traits are experimental and possibly buggy
+    //~| HELP add `#![feature(auto_traits)]` to the crate attributes to enable
     type Output;
     //~^ ERROR auto traits cannot have associated items
+    //~| HELP remove the associated items
 }
 
 fn main() {
     let _: <() as Trait>::Output = ();
     //[current]~^ ERROR mismatched types
-    //[next]~^^ ERROR type mismatch resolving `<() as Trait>::Output normalizes-to _`
+    //[current]~| HELP consider constraining the associated type `<() as Trait>::Output` to `()` or calling a method that returns `<() as Trait>::Output`
+    //[next]~^^^ ERROR type mismatch resolving `<() as Trait>::Output normalizes-to _`
     //[next]~| ERROR type mismatch resolving `<() as Trait>::Output normalizes-to _`
 }

From ea8733133cacf4496a2766b3d002492cc29ea7bf Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 18:27:38 +0100
Subject: [PATCH 1446/1843] Remove `DiagMessage::FluentIdentifier`

---
 compiler/rustc_error_messages/src/lib.rs | 41 ++------------
 compiler/rustc_errors/src/translation.rs | 72 ++----------------------
 2 files changed, 8 insertions(+), 105 deletions(-)

diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 641defd1eabf..916a370659aa 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -230,9 +230,6 @@ pub fn fallback_fluent_bundle(
     })))
 }
 
-/// Identifier for the Fluent message/attribute corresponding to a diagnostic message.
-type FluentId = Cow<'static, str>;
-
 /// Abstraction over a message in a subdiagnostic (i.e. label, note, help, etc) to support both
 /// translatable and non-translatable diagnostic messages.
 ///
@@ -244,18 +241,9 @@ type FluentId = Cow<'static, str>;
 pub enum SubdiagMessage {
     /// Non-translatable diagnostic message.
     Str(Cow<'static, str>),
-    /// Identifier of a Fluent message. Instances of this variant are generated by the
-    /// `Subdiagnostic` derive.
-    FluentIdentifier(FluentId),
     /// An inline Fluent message. Instances of this variant are generated by the
     /// `Subdiagnostic` derive.
     Inline(Cow<'static, str>),
-    /// Attribute of a Fluent message. Needs to be combined with a Fluent identifier to produce an
-    /// actual translated message. Instances of this variant are generated by the `fluent_messages`
-    /// macro.
-    ///
-    /// 
-    FluentAttr(FluentId),
 }
 
 impl From for SubdiagMessage {
@@ -288,12 +276,6 @@ pub enum DiagMessage {
     /// are translated when they are added to the parent diagnostic. This is one of the ways
     /// this variant of `DiagMessage` is produced.
     Str(Cow<'static, str>),
-    /// Identifier for a Fluent message (with optional attribute) corresponding to the diagnostic
-    /// message. Yet to be translated.
-    ///
-    /// 
-    /// 
-    FluentIdentifier(FluentId, Option),
     /// An inline Fluent message, containing the to be translated diagnostic message.
     Inline(Cow<'static, str>),
 }
@@ -305,27 +287,16 @@ impl DiagMessage {
     /// - If the `SubdiagMessage` is non-translatable then return the message as a `DiagMessage`.
     /// - If `self` is non-translatable then return `self`'s message.
     pub fn with_subdiagnostic_message(&self, sub: SubdiagMessage) -> Self {
-        let attr = match sub {
-            SubdiagMessage::Str(s) => return DiagMessage::Str(s),
-            SubdiagMessage::FluentIdentifier(id) => {
-                return DiagMessage::FluentIdentifier(id, None);
-            }
-            SubdiagMessage::Inline(s) => return DiagMessage::Inline(s),
-            SubdiagMessage::FluentAttr(attr) => attr,
-        };
-
-        match self {
-            DiagMessage::FluentIdentifier(id, _) => {
-                DiagMessage::FluentIdentifier(id.clone(), Some(attr))
-            }
-            _ => panic!("Tried to add a subdiagnostic to a message without a fluent identifier"),
+        match sub {
+            SubdiagMessage::Str(s) => DiagMessage::Str(s),
+            SubdiagMessage::Inline(s) => DiagMessage::Inline(s),
         }
     }
 
     pub fn as_str(&self) -> Option<&str> {
         match self {
             DiagMessage::Str(s) => Some(s),
-            DiagMessage::FluentIdentifier(_, _) | DiagMessage::Inline(_) => None,
+            DiagMessage::Inline(_) => None,
         }
     }
 }
@@ -355,10 +326,6 @@ impl From for SubdiagMessage {
     fn from(val: DiagMessage) -> Self {
         match val {
             DiagMessage::Str(s) => SubdiagMessage::Str(s),
-            DiagMessage::FluentIdentifier(id, None) => SubdiagMessage::FluentIdentifier(id),
-            // There isn't really a sensible behaviour for this because it loses information but
-            // this is the most sensible of the behaviours.
-            DiagMessage::FluentIdentifier(_, Some(attr)) => SubdiagMessage::FluentAttr(attr),
             DiagMessage::Inline(s) => SubdiagMessage::Inline(s),
         }
     }
diff --git a/compiler/rustc_errors/src/translation.rs b/compiler/rustc_errors/src/translation.rs
index ed3f9a405342..0b9845772590 100644
--- a/compiler/rustc_errors/src/translation.rs
+++ b/compiler/rustc_errors/src/translation.rs
@@ -1,5 +1,4 @@
 use std::borrow::Cow;
-use std::env;
 use std::error::Report;
 use std::sync::Arc;
 
@@ -7,7 +6,7 @@ pub use rustc_error_messages::{FluentArgs, LazyFallbackBundle};
 use rustc_error_messages::{langid, register_functions};
 use tracing::{debug, trace};
 
-use crate::error::{TranslateError, TranslateErrorKind};
+use crate::error::TranslateError;
 use crate::fluent_bundle::FluentResource;
 use crate::{DiagArg, DiagMessage, FluentBundle, Style, fluent_bundle};
 
@@ -76,11 +75,8 @@ impl Translator {
         args: &'a FluentArgs<'_>,
     ) -> Result, TranslateError<'a>> {
         trace!(?message, ?args);
-        let (identifier, attr) = match message {
-            DiagMessage::Str(msg) => {
-                return Ok(Cow::Borrowed(msg));
-            }
-            DiagMessage::FluentIdentifier(identifier, attr) => (identifier, attr),
+        match message {
+            DiagMessage::Str(msg) => Ok(Cow::Borrowed(msg)),
             // This translates an inline fluent diagnostic message
             // It does this by creating a new `FluentBundle` with only one message,
             // and then translating using this bundle.
@@ -98,71 +94,11 @@ impl Translator {
                 let mut errs = vec![];
                 let translated = bundle.format_pattern(value, Some(args), &mut errs).to_string();
                 debug!(?translated, ?errs);
-                return if errs.is_empty() {
+                if errs.is_empty() {
                     Ok(Cow::Owned(translated))
                 } else {
                     Err(TranslateError::fluent(&Cow::Borrowed(GENERATED_MSG_ID), args, errs))
-                };
-            }
-        };
-        let translate_with_bundle =
-            |bundle: &'a FluentBundle| -> Result, TranslateError<'_>> {
-                let message = bundle
-                    .get_message(identifier)
-                    .ok_or(TranslateError::message(identifier, args))?;
-                let value = match attr {
-                    Some(attr) => message
-                        .get_attribute(attr)
-                        .ok_or(TranslateError::attribute(identifier, args, attr))?
-                        .value(),
-                    None => message.value().ok_or(TranslateError::value(identifier, args))?,
-                };
-                debug!(?message, ?value);
-
-                let mut errs = vec![];
-                let translated = bundle.format_pattern(value, Some(args), &mut errs);
-                debug!(?translated, ?errs);
-                if errs.is_empty() {
-                    Ok(translated)
-                } else {
-                    Err(TranslateError::fluent(identifier, args, errs))
                 }
-            };
-
-        try {
-            match self.fluent_bundle.as_ref().map(|b| translate_with_bundle(b)) {
-                // The primary bundle was present and translation succeeded
-                Some(Ok(t)) => t,
-
-                // If `translate_with_bundle` returns `Err` with the primary bundle, this is likely
-                // just that the primary bundle doesn't contain the message being translated, so
-                // proceed to the fallback bundle.
-                Some(Err(
-                    primary @ TranslateError::One {
-                        kind: TranslateErrorKind::MessageMissing, ..
-                    },
-                )) => translate_with_bundle(&self.fallback_fluent_bundle)
-                    .map_err(|fallback| primary.and(fallback))?,
-
-                // Always yeet out for errors on debug (unless
-                // `RUSTC_TRANSLATION_NO_DEBUG_ASSERT` is set in the environment - this allows
-                // local runs of the test suites, of builds with debug assertions, to test the
-                // behaviour in a normal build).
-                Some(Err(primary))
-                    if cfg!(debug_assertions)
-                        && env::var("RUSTC_TRANSLATION_NO_DEBUG_ASSERT").is_err() =>
-                {
-                    do yeet primary
-                }
-
-                // ..otherwise, for end users, an error about this wouldn't be useful or actionable, so
-                // just hide it and try with the fallback bundle.
-                Some(Err(primary)) => translate_with_bundle(&self.fallback_fluent_bundle)
-                    .map_err(|fallback| primary.and(fallback))?,
-
-                // The primary bundle is missing, proceed to the fallback bundle
-                None => translate_with_bundle(&self.fallback_fluent_bundle)
-                    .map_err(|fallback| TranslateError::primary(identifier, args).and(fallback))?,
             }
         }
     }

From 62dd37131f6f43d0dab3e26cf125cd841a08e003 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 19:04:18 +0100
Subject: [PATCH 1447/1843] Remove slugs from the `#[derive(Diagnostic)]` macro

---
 .../src/diagnostics/diagnostic.rs             | 27 +----
 .../src/diagnostics/diagnostic_builder.rs     | 98 +++++--------------
 .../rustc_macros/src/diagnostics/message.rs   | 75 ++------------
 .../src/diagnostics/subdiagnostic.rs          | 22 ++---
 .../rustc_macros/src/diagnostics/utils.rs     | 43 +++-----
 5 files changed, 60 insertions(+), 205 deletions(-)

diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic.rs b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
index e8356655dd9f..dc8231e5f0b0 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
@@ -1,7 +1,5 @@
 #![deny(unused_must_use)]
 
-use std::cell::RefCell;
-
 use proc_macro2::TokenStream;
 use quote::quote;
 use synstructure::Structure;
@@ -22,7 +20,6 @@ impl<'a> DiagnosticDerive<'a> {
     pub(crate) fn into_tokens(self) -> TokenStream {
         let DiagnosticDerive { mut structure } = self;
         let kind = DiagnosticDeriveKind::Diagnostic;
-        let messages = RefCell::new(Vec::new());
         let implementation = kind.each_variant(&mut structure, |mut builder, variant| {
             let preamble = builder.preamble(variant);
             let body = builder.body(variant);
@@ -30,7 +27,6 @@ impl<'a> DiagnosticDerive<'a> {
             let Some(message) = builder.primary_message() else {
                 return DiagnosticDeriveError::ErrorHandled.to_compile_error();
             };
-            messages.borrow_mut().push(message.clone());
             let message = message.diag_message(Some(variant));
 
             let init = quote! {
@@ -52,9 +48,7 @@ impl<'a> DiagnosticDerive<'a> {
         });
 
         // A lifetime of `'a` causes conflicts, but `_sess` is fine.
-        // FIXME(edition_2024): Fix the `keyword_idents_2024` lint to not trigger here?
-        #[allow(keyword_idents_2024)]
-        let mut imp = structure.gen_impl(quote! {
+        structure.gen_impl(quote! {
             gen impl<'_sess, G> rustc_errors::Diagnostic<'_sess, G> for @Self
                 where G: rustc_errors::EmissionGuarantee
             {
@@ -67,11 +61,7 @@ impl<'a> DiagnosticDerive<'a> {
                     #implementation
                 }
             }
-        });
-        for test in messages.borrow().iter().map(|s| s.generate_test(&structure)) {
-            imp.extend(test);
-        }
-        imp
+        })
     }
 }
 
@@ -88,7 +78,6 @@ impl<'a> LintDiagnosticDerive<'a> {
     pub(crate) fn into_tokens(self) -> TokenStream {
         let LintDiagnosticDerive { mut structure } = self;
         let kind = DiagnosticDeriveKind::LintDiagnostic;
-        let messages = RefCell::new(Vec::new());
         let implementation = kind.each_variant(&mut structure, |mut builder, variant| {
             let preamble = builder.preamble(variant);
             let body = builder.body(variant);
@@ -96,7 +85,6 @@ impl<'a> LintDiagnosticDerive<'a> {
             let Some(message) = builder.primary_message() else {
                 return DiagnosticDeriveError::ErrorHandled.to_compile_error();
             };
-            messages.borrow_mut().push(message.clone());
             let message = message.diag_message(Some(variant));
             let primary_message = quote! {
                 diag.primary_message(#message);
@@ -112,9 +100,7 @@ impl<'a> LintDiagnosticDerive<'a> {
             }
         });
 
-        // FIXME(edition_2024): Fix the `keyword_idents_2024` lint to not trigger here?
-        #[allow(keyword_idents_2024)]
-        let mut imp = structure.gen_impl(quote! {
+        structure.gen_impl(quote! {
             gen impl<'__a> rustc_errors::LintDiagnostic<'__a, ()> for @Self {
                 #[track_caller]
                 fn decorate_lint<'__b>(
@@ -124,11 +110,6 @@ impl<'a> LintDiagnosticDerive<'a> {
                     #implementation;
                 }
             }
-        });
-        for test in messages.borrow().iter().map(|s| s.generate_test(&structure)) {
-            imp.extend(test);
-        }
-
-        imp
+        })
     }
 }
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
index 6107b181eea2..b386408a1918 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
@@ -4,7 +4,7 @@ use proc_macro2::{Ident, Span, TokenStream};
 use quote::{format_ident, quote, quote_spanned};
 use syn::parse::ParseStream;
 use syn::spanned::Spanned;
-use syn::{Attribute, LitStr, Meta, Path, Token, Type, parse_quote};
+use syn::{Attribute, LitStr, Meta, Path, Token, Type};
 use synstructure::{BindingInfo, Structure, VariantInfo};
 
 use super::utils::SubdiagnosticVariant;
@@ -109,24 +109,14 @@ impl DiagnosticDeriveVariantBuilder {
     pub(crate) fn primary_message(&self) -> Option<&Message> {
         match self.message.as_ref() {
             None => {
-                span_err(self.span, "diagnostic slug not specified")
+                span_err(self.span, "diagnostic message not specified")
                     .help(
-                        "specify the slug as the first argument to the `#[diag(...)]` \
-                            attribute, such as `#[diag(hir_analysis_example_error)]`",
+                        "specify the message as the first argument to the `#[diag(...)]` \
+                            attribute, such as `#[diag(\"Example error\")]`",
                     )
                     .emit();
                 None
             }
-            Some(Message::Slug(slug))
-                if let Some(Mismatch { slug_name, crate_name, slug_prefix }) =
-                    Mismatch::check(slug) =>
-            {
-                span_err(slug.span().unwrap(), "diagnostic slug and crate name do not match")
-                    .note(format!("slug is `{slug_name}` but the crate name is `{crate_name}`"))
-                    .help(format!("expected a slug starting with `{slug_prefix}_...`"))
-                    .emit();
-                None
-            }
             Some(msg) => Some(msg),
         }
     }
@@ -177,25 +167,15 @@ impl DiagnosticDeriveVariantBuilder {
                 .help("consider creating a `Subdiagnostic` instead"));
         }
 
-        // For subdiagnostics without a message specified, insert a placeholder slug
-        let slug = subdiag.slug.unwrap_or_else(|| {
-            Message::Slug(match subdiag.kind {
-                SubdiagnosticKind::Label => parse_quote! { _subdiag::label },
-                SubdiagnosticKind::Note => parse_quote! { _subdiag::note },
-                SubdiagnosticKind::NoteOnce => parse_quote! { _subdiag::note_once },
-                SubdiagnosticKind::Help => parse_quote! { _subdiag::help },
-                SubdiagnosticKind::HelpOnce => parse_quote! { _subdiag::help_once },
-                SubdiagnosticKind::Warn => parse_quote! { _subdiag::warn },
-                SubdiagnosticKind::Suggestion { .. } => parse_quote! { _subdiag::suggestion },
-                SubdiagnosticKind::MultipartSuggestion { .. } => unreachable!(),
-            })
-        });
+        let Some(message) = subdiag.message else {
+            throw_invalid_attr!(attr, |diag| diag.help("subdiagnostic message is missing"))
+        };
 
-        Ok(Some((subdiag.kind, slug, false)))
+        Ok(Some((subdiag.kind, message, false)))
     }
 
     /// Establishes state in the `DiagnosticDeriveBuilder` resulting from the struct
-    /// attributes like `#[diag(..)]`, such as the slug and error code. Generates
+    /// attributes like `#[diag(..)]`, such as the message and error code. Generates
     /// diagnostic builder calls for setting error code and creating note/help messages.
     fn generate_structure_code_for_attr(
         &mut self,
@@ -213,9 +193,6 @@ impl DiagnosticDeriveVariantBuilder {
         if name == "diag" {
             let mut tokens = TokenStream::new();
             attr.parse_args_with(|input: ParseStream<'_>| {
-                let mut input = &*input;
-                let slug_recovery_point = input.fork();
-
                 if input.peek(LitStr) {
                     // Parse an inline message
                     let message = input.parse::()?;
@@ -226,15 +203,8 @@ impl DiagnosticDeriveVariantBuilder {
                         )
                         .emit();
                     }
-                    self.message = Some(Message::Inline(message.span(), message.value()));
-                } else {
-                    // Parse a slug
-                    let slug = input.parse::()?;
-                    if input.is_empty() || input.peek(Token![,]) {
-                        self.message = Some(Message::Slug(slug));
-                    } else {
-                        input = &slug_recovery_point;
-                    }
+                    self.message =
+                        Some(Message { message_span: message.span(), value: message.value() });
                 }
 
                 // Parse arguments
@@ -248,7 +218,7 @@ impl DiagnosticDeriveVariantBuilder {
                     if input.peek(Token![,]) {
                         span_err(
                             arg_name.span().unwrap(),
-                            "diagnostic slug must be the first argument",
+                            "diagnostic message must be the first argument",
                         )
                         .emit();
                         continue;
@@ -265,7 +235,7 @@ impl DiagnosticDeriveVariantBuilder {
                         }
                         _ => {
                             span_err(arg_name.span().unwrap(), "unknown argument")
-                                .note("only the `code` parameter is valid after the slug")
+                                .note("only the `code` parameter is valid after the message")
                                 .emit();
                         }
                     }
@@ -276,7 +246,7 @@ impl DiagnosticDeriveVariantBuilder {
             return Ok(tokens);
         }
 
-        let Some((subdiag, slug, _no_span)) = self.parse_subdiag_attribute(attr)? else {
+        let Some((subdiag, message, _no_span)) = self.parse_subdiag_attribute(attr)? else {
             // Some attributes aren't errors - like documentation comments - but also aren't
             // subdiagnostics.
             return Ok(quote! {});
@@ -287,7 +257,7 @@ impl DiagnosticDeriveVariantBuilder {
             | SubdiagnosticKind::NoteOnce
             | SubdiagnosticKind::Help
             | SubdiagnosticKind::HelpOnce
-            | SubdiagnosticKind::Warn => Ok(self.add_subdiagnostic(&fn_ident, slug, variant)),
+            | SubdiagnosticKind::Warn => Ok(self.add_subdiagnostic(&fn_ident, message, variant)),
             SubdiagnosticKind::Label | SubdiagnosticKind::Suggestion { .. } => {
                 throw_invalid_attr!(attr, |diag| diag
                     .help("`#[label]` and `#[suggestion]` can only be applied to fields"));
@@ -406,7 +376,7 @@ impl DiagnosticDeriveVariantBuilder {
             _ => (),
         }
 
-        let Some((subdiag, slug, _no_span)) = self.parse_subdiag_attribute(attr)? else {
+        let Some((subdiag, message, _no_span)) = self.parse_subdiag_attribute(attr)? else {
             // Some attributes aren't errors - like documentation comments - but also aren't
             // subdiagnostics.
             return Ok(quote! {});
@@ -415,7 +385,7 @@ impl DiagnosticDeriveVariantBuilder {
         match subdiag {
             SubdiagnosticKind::Label => {
                 report_error_if_not_applied_to_span(attr, &info)?;
-                Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, slug, variant))
+                Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, message, variant))
             }
             SubdiagnosticKind::Note
             | SubdiagnosticKind::NoteOnce
@@ -426,11 +396,11 @@ impl DiagnosticDeriveVariantBuilder {
                 if type_matches_path(inner, &["rustc_span", "Span"])
                     || type_matches_path(inner, &["rustc_span", "MultiSpan"])
                 {
-                    Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, slug, variant))
+                    Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, message, variant))
                 } else if type_is_unit(inner)
                     || (matches!(info.ty, FieldInnerTy::Plain(_)) && type_is_bool(inner))
                 {
-                    Ok(self.add_subdiagnostic(&fn_ident, slug, variant))
+                    Ok(self.add_subdiagnostic(&fn_ident, message, variant))
                 } else {
                     report_type_error(attr, "`Span`, `MultiSpan`, `bool` or `()`")?
                 }
@@ -456,7 +426,7 @@ impl DiagnosticDeriveVariantBuilder {
                     applicability.set_once(quote! { #static_applicability }, span);
                 }
 
-                let message = slug.diag_message(Some(variant));
+                let message = message.diag_message(Some(variant));
                 let applicability = applicability
                     .value()
                     .unwrap_or_else(|| quote! { rustc_errors::Applicability::Unspecified });
@@ -477,7 +447,7 @@ impl DiagnosticDeriveVariantBuilder {
         }
     }
 
-    /// Adds a spanned subdiagnostic by generating a `diag.span_$kind` call with the current slug
+    /// Adds a spanned subdiagnostic by generating a `diag.span_$kind` call with the current message
     /// and `fluent_attr_identifier`.
     fn add_spanned_subdiagnostic(
         &self,
@@ -496,7 +466,7 @@ impl DiagnosticDeriveVariantBuilder {
         }
     }
 
-    /// Adds a subdiagnostic by generating a `diag.span_$kind` call with the current slug
+    /// Adds a subdiagnostic by generating a `diag.span_$kind` call with the current message
     /// and `fluent_attr_identifier`.
     fn add_subdiagnostic(
         &self,
@@ -567,27 +537,3 @@ impl DiagnosticDeriveVariantBuilder {
         }
     }
 }
-
-struct Mismatch {
-    slug_name: String,
-    crate_name: String,
-    slug_prefix: String,
-}
-
-impl Mismatch {
-    /// Checks whether the slug starts with the crate name it's in.
-    fn check(slug: &syn::Path) -> Option {
-        // If this is missing we're probably in a test, so bail.
-        let crate_name = std::env::var("CARGO_CRATE_NAME").ok()?;
-
-        // If we're not in a "rustc_" crate, bail.
-        let Some(("rustc", slug_prefix)) = crate_name.split_once('_') else { return None };
-
-        let slug_name = slug.segments.first()?.ident.to_string();
-        if slug_name.starts_with(slug_prefix) {
-            return None;
-        }
-
-        Some(Mismatch { slug_name, slug_prefix: slug_prefix.to_string(), crate_name })
-    }
-}
diff --git a/compiler/rustc_macros/src/diagnostics/message.rs b/compiler/rustc_macros/src/diagnostics/message.rs
index 2db8df2f69ae..18d4d60dde3e 100644
--- a/compiler/rustc_macros/src/diagnostics/message.rs
+++ b/compiler/rustc_macros/src/diagnostics/message.rs
@@ -2,16 +2,15 @@ use fluent_bundle::FluentResource;
 use fluent_syntax::ast::{Expression, InlineExpression, Pattern, PatternElement};
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
-use syn::Path;
 use syn::ext::IdentExt;
-use synstructure::{Structure, VariantInfo};
+use synstructure::VariantInfo;
 
 use crate::diagnostics::error::span_err;
 
 #[derive(Clone)]
-pub(crate) enum Message {
-    Slug(Path),
-    Inline(Span, String),
+pub(crate) struct Message {
+    pub message_span: Span,
+    pub value: String,
 }
 
 impl Message {
@@ -19,69 +18,9 @@ impl Message {
     /// The passed `variant` is used to check whether all variables in the message are used.
     /// For subdiagnostics, we cannot check this.
     pub(crate) fn diag_message(&self, variant: Option<&VariantInfo<'_>>) -> TokenStream {
-        match self {
-            Message::Slug(slug) => {
-                quote! { crate::fluent_generated::#slug }
-            }
-            Message::Inline(message_span, message) => {
-                verify_fluent_message(*message_span, &message, variant);
-                quote! { rustc_errors::DiagMessage::Inline(std::borrow::Cow::Borrowed(#message)) }
-            }
-        }
-    }
-
-    /// Generates a `#[test]` that verifies that all referenced variables
-    /// exist on this structure.
-    pub(crate) fn generate_test(&self, structure: &Structure<'_>) -> TokenStream {
-        match self {
-            Message::Slug(slug) => {
-                // FIXME: We can't identify variables in a subdiagnostic
-                for field in structure.variants().iter().flat_map(|v| v.ast().fields.iter()) {
-                    for attr_name in field.attrs.iter().filter_map(|at| at.path().get_ident()) {
-                        if attr_name == "subdiagnostic" {
-                            return quote!();
-                        }
-                    }
-                }
-                use std::sync::atomic::{AtomicUsize, Ordering};
-                // We need to make sure that the same diagnostic slug can be used multiple times without
-                // causing an error, so just have a global counter here.
-                static COUNTER: AtomicUsize = AtomicUsize::new(0);
-                let slug = slug.get_ident().unwrap();
-                let ident = quote::format_ident!(
-                    "verify_{slug}_{}",
-                    COUNTER.fetch_add(1, Ordering::Relaxed)
-                );
-                let ref_slug = quote::format_ident!("{slug}_refs");
-                let struct_name = &structure.ast().ident;
-                let variables: Vec<_> = structure
-                    .variants()
-                    .iter()
-                    .flat_map(|v| {
-                        v.ast()
-                            .fields
-                            .iter()
-                            .filter_map(|f| f.ident.as_ref().map(|i| i.to_string()))
-                    })
-                    .collect();
-                // tidy errors on `#[test]` outside of test files, so we use `#[test ]` to work around this
-                quote! {
-                    #[cfg(test)]
-                    #[test ]
-                    fn #ident() {
-                        let variables = [#(#variables),*];
-                        for vref in crate::fluent_generated::#ref_slug {
-                            assert!(variables.contains(vref), "{}: variable `{vref}` not found ({})", stringify!(#struct_name), stringify!(#slug));
-                        }
-                    }
-                }
-            }
-            Message::Inline(..) => {
-                // We don't generate a test for inline diagnostics, we can verify these at compile-time!
-                // This verification is done in the `diag_message` function above
-                quote! {}
-            }
-        }
+        let message = &self.value;
+        verify_fluent_message(self.message_span, &message, variant);
+        quote! { rustc_errors::DiagMessage::Inline(std::borrow::Cow::Borrowed(#message)) }
     }
 }
 
diff --git a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
index ac1fa984664c..7eb170ec236a 100644
--- a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
@@ -186,10 +186,10 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
     fn identify_kind(
         &mut self,
     ) -> Result, DiagnosticDeriveError> {
-        let mut kind_slugs = vec![];
+        let mut kind_messages = vec![];
 
         for attr in self.variant.ast().attrs {
-            let Some(SubdiagnosticVariant { kind, slug }) =
+            let Some(SubdiagnosticVariant { kind, message }) =
                 SubdiagnosticVariant::from_attr(attr, &self.fields)?
             else {
                 // Some attributes aren't errors - like documentation comments - but also aren't
@@ -197,22 +197,22 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                 continue;
             };
 
-            let Some(slug) = slug else {
+            let Some(message) = message else {
                 let name = attr.path().segments.last().unwrap().ident.to_string();
                 let name = name.as_str();
 
                 throw_span_err!(
                     attr.span().unwrap(),
                     format!(
-                        "diagnostic slug must be first argument of a `#[{name}(...)]` attribute"
+                        "diagnostic message must be first argument of a `#[{name}(...)]` attribute"
                     )
                 );
             };
 
-            kind_slugs.push((kind, slug));
+            kind_messages.push((kind, message));
         }
 
-        Ok(kind_slugs)
+        Ok(kind_messages)
     }
 
     /// Generates the code for a field with no attributes.
@@ -498,9 +498,9 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
     }
 
     pub(crate) fn into_tokens(&mut self) -> Result {
-        let kind_slugs = self.identify_kind()?;
+        let kind_messages = self.identify_kind()?;
 
-        let kind_stats: KindsStatistics = kind_slugs.iter().map(|(kind, _slug)| kind).collect();
+        let kind_stats: KindsStatistics = kind_messages.iter().map(|(kind, _msg)| kind).collect();
 
         let init = if kind_stats.has_multipart_suggestion {
             quote! { let mut suggestions = Vec::new(); }
@@ -516,7 +516,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
             .map(|binding| self.generate_field_attr_code(binding, kind_stats))
             .collect();
 
-        if kind_slugs.is_empty() && !self.has_subdiagnostic {
+        if kind_messages.is_empty() && !self.has_subdiagnostic {
             if self.is_enum {
                 // It's okay for a variant to not be a subdiagnostic at all..
                 return Ok(quote! {});
@@ -533,9 +533,9 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
 
         let diag = &self.parent.diag;
         let mut calls = TokenStream::new();
-        for (kind, slug) in kind_slugs {
+        for (kind, messages) in kind_messages {
             let message = format_ident!("__message");
-            let message_stream = slug.diag_message(None);
+            let message_stream = messages.diag_message(None);
             calls.extend(quote! { let #message = #diag.eagerly_translate(#message_stream); });
 
             let name = format_ident!("{}{}", if span_field.is_some() { "span_" } else { "" }, kind);
diff --git a/compiler/rustc_macros/src/diagnostics/utils.rs b/compiler/rustc_macros/src/diagnostics/utils.rs
index a5265a847a9c..c308f6126325 100644
--- a/compiler/rustc_macros/src/diagnostics/utils.rs
+++ b/compiler/rustc_macros/src/diagnostics/utils.rs
@@ -267,7 +267,7 @@ pub(super) type FieldMap = HashMap;
 ///
 /// ```ignore (not-usage-example)
 /// /// Suggest `==` when users wrote `===`.
-/// #[suggestion(slug = "parser-not-javascript-eq", code = "{lhs} == {rhs}")]
+/// #[suggestion("example message", code = "{lhs} == {rhs}")]
 /// struct NotJavaScriptEq {
 ///     #[primary_span]
 ///     span: Span,
@@ -588,13 +588,13 @@ pub(super) enum SubdiagnosticKind {
 
 pub(super) struct SubdiagnosticVariant {
     pub(super) kind: SubdiagnosticKind,
-    pub(super) slug: Option,
+    pub(super) message: Option,
 }
 
 impl SubdiagnosticVariant {
     /// Constructs a `SubdiagnosticVariant` from a field or type attribute such as `#[note]`,
-    /// `#[error(parser::add_paren)]` or `#[suggestion(code = "...")]`. Returns the
-    /// `SubdiagnosticKind` and the diagnostic slug, if specified.
+    /// `#[error("add parenthesis")]` or `#[suggestion(code = "...")]`. Returns the
+    /// `SubdiagnosticKind` and the diagnostic message, if specified.
     pub(super) fn from_attr(
         attr: &Attribute,
         fields: &FieldMap,
@@ -660,11 +660,11 @@ impl SubdiagnosticVariant {
         let list = match &attr.meta {
             Meta::List(list) => {
                 // An attribute with properties, such as `#[suggestion(code = "...")]` or
-                // `#[error(some::slug)]`
+                // `#[error("message")]`
                 list
             }
             Meta::Path(_) => {
-                // An attribute without a slug or other properties, such as `#[note]` - return
+                // An attribute without a message or other properties, such as `#[note]` - return
                 // without further processing.
                 //
                 // Only allow this if there are no mandatory properties, such as `code = "..."` in
@@ -677,7 +677,7 @@ impl SubdiagnosticVariant {
                     | SubdiagnosticKind::HelpOnce
                     | SubdiagnosticKind::Warn
                     | SubdiagnosticKind::MultipartSuggestion { .. } => {
-                        return Ok(Some(SubdiagnosticVariant { kind, slug: None }));
+                        return Ok(Some(SubdiagnosticVariant { kind, message: None }));
                     }
                     SubdiagnosticKind::Suggestion { .. } => {
                         throw_span_err!(span, "suggestion without `code = \"...\"`")
@@ -692,45 +692,34 @@ impl SubdiagnosticVariant {
         let mut code = None;
         let mut suggestion_kind = None;
 
-        let mut slug = None;
+        let mut message = None;
 
         list.parse_args_with(|input: ParseStream<'_>| {
             let mut is_first = true;
             while !input.is_empty() {
                 // Try to parse an inline diagnostic message
                 if input.peek(LitStr) {
-                    let message = input.parse::()?;
-                    if !message.suffix().is_empty() {
+                    let inline_message = input.parse::()?;
+                    if !inline_message.suffix().is_empty() {
                         span_err(
-                            message.span().unwrap(),
+                            inline_message.span().unwrap(),
                             "Inline message is not allowed to have a suffix",
                         ).emit();
                     }
                     if !input.is_empty() { input.parse::()?; }
                     if is_first {
-                        slug = Some(Message::Inline(message.span(), message.value()));
+                        message = Some(Message { message_span: inline_message.span(), value: inline_message.value() });
                         is_first = false;
                     } else {
-                        span_err(message.span().unwrap(), "a diagnostic message must be the first argument to the attribute").emit();
-                    }
-                    continue
-                }
-
-                // Try to parse a slug instead
-                let arg_name: Path = input.parse::()?;
-                let arg_name_span = arg_name.span().unwrap();
-                if input.is_empty() || input.parse::().is_ok() {
-                    if is_first {
-                        slug = Some(Message::Slug(arg_name));
-                        is_first = false;
-                    } else {
-                        span_err(arg_name_span, "a diagnostic slug must be the first argument to the attribute").emit();
+                        span_err(inline_message.span().unwrap(), "a diagnostic message must be the first argument to the attribute").emit();
                     }
                     continue
                 }
                 is_first = false;
 
                 // Try to parse an argument
+                let arg_name: Path = input.parse::()?;
+                let arg_name_span = arg_name.span().unwrap();
                 match (arg_name.require_ident()?.to_string().as_str(), &mut kind) {
                     ("code", SubdiagnosticKind::Suggestion { code_field, .. }) => {
                         let code_init = build_suggestion_code(
@@ -836,7 +825,7 @@ impl SubdiagnosticVariant {
             | SubdiagnosticKind::Warn => {}
         }
 
-        Ok(Some(SubdiagnosticVariant { kind, slug }))
+        Ok(Some(SubdiagnosticVariant { kind, message }))
     }
 }
 

From 2289e6cfb7e379eba777a10df0a6f474f9450b02 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 11:04:30 +0100
Subject: [PATCH 1448/1843] Adjust expected uitests results for the updated
 `derive(Diagnostic)` macro

---
 .../diagnostic-derive-inline.rs               |  38 +++--
 .../diagnostic-derive-inline.stderr           | 158 +++++++++---------
 .../subdiagnostic-derive-inline.rs            |  16 +-
 .../subdiagnostic-derive-inline.stderr        |  22 +--
 4 files changed, 124 insertions(+), 110 deletions(-)

diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
index 1d2bfc66aaff..ca803d450b92 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
@@ -45,9 +45,9 @@ struct Hello {}
 //~^ ERROR unsupported type attribute for diagnostic derive enum
 enum DiagnosticOnEnum {
     Foo,
-    //~^ ERROR diagnostic slug not specified
+    //~^ ERROR diagnostic message not specified
     Bar,
-    //~^ ERROR diagnostic slug not specified
+    //~^ ERROR diagnostic message not specified
 }
 
 #[derive(Diagnostic)]
@@ -59,28 +59,28 @@ struct WrongStructAttrStyle {}
 #[derive(Diagnostic)]
 #[nonsense("this is an example message", code = E0123)]
 //~^ ERROR `#[nonsense(...)]` is not a valid attribute
-//~^^ ERROR diagnostic slug not specified
+//~^^ ERROR diagnostic message not specified
 //~^^^ ERROR cannot find attribute `nonsense` in this scope
 struct InvalidStructAttr {}
 
 #[derive(Diagnostic)]
 #[diag(code = E0123)]
-//~^ ERROR diagnostic slug not specified
+//~^ ERROR diagnostic message not specified
 struct InvalidLitNestedAttr {}
 
 #[derive(Diagnostic)]
 #[diag(nonsense("foo"), code = E0123, slug = "foo")]
-//~^ ERROR derive(Diagnostic): diagnostic slug not specified
+//~^ ERROR derive(Diagnostic): diagnostic message not specified
 struct InvalidNestedStructAttr1 {}
 
 #[derive(Diagnostic)]
 #[diag(nonsense = "...", code = E0123, slug = "foo")]
-//~^ ERROR diagnostic slug not specified
+//~^ ERROR diagnostic message not specified
 struct InvalidNestedStructAttr2 {}
 
 #[derive(Diagnostic)]
 #[diag(nonsense = 4, code = E0123, slug = "foo")]
-//~^ ERROR diagnostic slug not specified
+//~^ ERROR diagnostic message not specified
 struct InvalidNestedStructAttr3 {}
 
 #[derive(Diagnostic)]
@@ -109,15 +109,15 @@ struct CodeSpecifiedTwice {}
 
 #[derive(Diagnostic)]
 #[diag("this is an example message", no_crate::example, code = E0123)]
-//~^ ERROR diagnostic slug must be the first argument
+//~^ ERROR diagnostic message must be the first argument
 struct SlugSpecifiedTwice {}
 
 #[derive(Diagnostic)]
-struct KindNotProvided {} //~ ERROR diagnostic slug not specified
+struct KindNotProvided {} //~ ERROR diagnostic message not specified
 
 #[derive(Diagnostic)]
 #[diag(code = E0123)]
-//~^ ERROR diagnostic slug not specified
+//~^ ERROR diagnostic message not specified
 struct SlugNotProvided {}
 
 #[derive(Diagnostic)]
@@ -467,7 +467,15 @@ struct BoolField {
 #[diag("this is an example message", code = E0123)]
 struct LabelWithTrailingPath {
     #[label("with a label", foo)]
-    //~^ ERROR a diagnostic slug must be the first argument to the attribute
+    //~^ ERROR derive(Diagnostic): no nested attribute expected here
+    span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag("this is an example message", code = E0123)]
+struct LabelWithTrailingMessage {
+    #[label("with a label", "and another one?")]
+    //~^ ERROR derive(Diagnostic): a diagnostic message must be the first argument to the attribute
     span: Span,
 }
 
@@ -516,28 +524,28 @@ struct ErrorWithWarn {
 #[derive(Diagnostic)]
 #[error("this is an example message", code = E0123)]
 //~^ ERROR `#[error(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
+//~| ERROR diagnostic message not specified
 //~| ERROR cannot find attribute `error` in this scope
 struct ErrorAttribute {}
 
 #[derive(Diagnostic)]
 #[warn_("this is an example message", code = E0123)]
 //~^ ERROR `#[warn_(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
+//~| ERROR diagnostic message not specified
 //~| ERROR cannot find attribute `warn_` in this scope
 struct WarnAttribute {}
 
 #[derive(Diagnostic)]
 #[lint("this is an example message", code = E0123)]
 //~^ ERROR `#[lint(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
+//~| ERROR diagnostic message not specified
 //~| ERROR cannot find attribute `lint` in this scope
 struct LintAttributeOnSessionDiag {}
 
 #[derive(LintDiagnostic)]
 #[lint("this is an example message", code = E0123)]
 //~^ ERROR `#[lint(...)]` is not a valid attribute
-//~| ERROR diagnostic slug not specified
+//~| ERROR diagnostic message not specified
 //~| ERROR cannot find attribute `lint` in this scope
 struct LintAttributeOnLintDiag {}
 
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr
index 71ed77804a6c..b32235c02b07 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.stderr
@@ -4,21 +4,21 @@ error: derive(Diagnostic): unsupported type attribute for diagnostic derive enum
 LL | #[diag("this is an example message", code = E0123)]
    | ^
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:47:5
    |
 LL |     Foo,
    |     ^^^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:49:5
    |
 LL |     Bar,
    |     ^^^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: expected parentheses: #[diag(...)]
   --> $DIR/diagnostic-derive-inline.rs:55:8
@@ -32,45 +32,45 @@ error: derive(Diagnostic): `#[nonsense(...)]` is not a valid attribute
 LL | #[nonsense("this is an example message", code = E0123)]
    | ^
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:60:1
    |
 LL | #[nonsense("this is an example message", code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:67:1
    |
 LL | #[diag(code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:72:1
    |
 LL | #[diag(nonsense("foo"), code = E0123, slug = "foo")]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:77:1
    |
 LL | #[diag(nonsense = "...", code = E0123, slug = "foo")]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:82:1
    |
 LL | #[diag(nonsense = 4, code = E0123, slug = "foo")]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: derive(Diagnostic): unknown argument
   --> $DIR/diagnostic-derive-inline.rs:87:52
@@ -78,7 +78,7 @@ error: derive(Diagnostic): unknown argument
 LL | #[diag("this is an example message", code = E0123, slug = "foo")]
    |                                                    ^^^^
    |
-   = note: only the `code` parameter is valid after the slug
+   = note: only the `code` parameter is valid after the message
 
 error: derive(Diagnostic): `#[suggestion = ...]` is not a valid attribute
   --> $DIR/diagnostic-derive-inline.rs:94:5
@@ -110,27 +110,27 @@ note: previously specified here
 LL | #[diag("this is an example message", code = E0123, code = E0456)]
    |                                      ^^^^
 
-error: derive(Diagnostic): diagnostic slug must be the first argument
+error: derive(Diagnostic): diagnostic message must be the first argument
   --> $DIR/diagnostic-derive-inline.rs:111:38
    |
 LL | #[diag("this is an example message", no_crate::example, code = E0123)]
    |                                      ^^^^^^^^
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:116:1
    |
 LL | struct KindNotProvided {}
    | ^^^^^^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
-error: derive(Diagnostic): diagnostic slug not specified
+error: derive(Diagnostic): diagnostic message not specified
   --> $DIR/diagnostic-derive-inline.rs:119:1
    |
 LL | #[diag(code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: derive(Diagnostic): the `#[primary_span]` attribute can only be applied to fields of type `Span` or `MultiSpan`
   --> $DIR/diagnostic-derive-inline.rs:130:5
@@ -276,26 +276,32 @@ error: derive(Diagnostic): the `#[help(...)]` attribute can only be applied to f
 LL |     #[help("with a help")]
    |     ^
 
-error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
+error: derive(Diagnostic): no nested attribute expected here
   --> $DIR/diagnostic-derive-inline.rs:469:29
    |
 LL |     #[label("with a label", foo)]
    |                             ^^^
 
-error: derive(Diagnostic): no nested attribute expected here
+error: derive(Diagnostic): a diagnostic message must be the first argument to the attribute
   --> $DIR/diagnostic-derive-inline.rs:477:29
    |
+LL |     #[label("with a label", "and another one?")]
+   |                             ^^^^^^^^^^^^^^^^^^
+
+error: derive(Diagnostic): no nested attribute expected here
+  --> $DIR/diagnostic-derive-inline.rs:485:29
+   |
 LL |     #[label("with a label", foo = "...")]
    |                             ^^^
 
 error: derive(Diagnostic): no nested attribute expected here
-  --> $DIR/diagnostic-derive-inline.rs:485:29
+  --> $DIR/diagnostic-derive-inline.rs:493:29
    |
 LL |     #[label("with a label", foo("..."))]
    |                             ^^^
 
 error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:497:5
+  --> $DIR/diagnostic-derive-inline.rs:505:5
    |
 LL |     #[primary_span]
    |     ^
@@ -303,75 +309,75 @@ LL |     #[primary_span]
    = help: the `primary_span` field attribute is not valid for lint diagnostics
 
 error: derive(Diagnostic): `#[error(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:517:1
+  --> $DIR/diagnostic-derive-inline.rs:525:1
    |
 LL | #[error("this is an example message", code = E0123)]
    | ^
 
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:517:1
+error: derive(Diagnostic): diagnostic message not specified
+  --> $DIR/diagnostic-derive-inline.rs:525:1
    |
 LL | #[error("this is an example message", code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: derive(Diagnostic): `#[warn_(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:524:1
+  --> $DIR/diagnostic-derive-inline.rs:532:1
    |
 LL | #[warn_("this is an example message", code = E0123)]
    | ^
 
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:524:1
+error: derive(Diagnostic): diagnostic message not specified
+  --> $DIR/diagnostic-derive-inline.rs:532:1
    |
 LL | #[warn_("this is an example message", code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: derive(Diagnostic): `#[lint(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:531:1
+  --> $DIR/diagnostic-derive-inline.rs:539:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
 
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:531:1
+error: derive(Diagnostic): diagnostic message not specified
+  --> $DIR/diagnostic-derive-inline.rs:539:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: derive(Diagnostic): `#[lint(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:538:1
+  --> $DIR/diagnostic-derive-inline.rs:546:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
 
-error: derive(Diagnostic): diagnostic slug not specified
-  --> $DIR/diagnostic-derive-inline.rs:538:1
+error: derive(Diagnostic): diagnostic message not specified
+  --> $DIR/diagnostic-derive-inline.rs:546:1
    |
 LL | #[lint("this is an example message", code = E0123)]
    | ^
    |
-   = help: specify the slug as the first argument to the `#[diag(...)]` attribute, such as `#[diag(hir_analysis_example_error)]`
+   = help: specify the message as the first argument to the `#[diag(...)]` attribute, such as `#[diag("Example error")]`
 
 error: derive(Diagnostic): attribute specified multiple times
-  --> $DIR/diagnostic-derive-inline.rs:547:53
+  --> $DIR/diagnostic-derive-inline.rs:555:53
    |
 LL |     #[suggestion("with a suggestion", code = "...", code = ",,,")]
    |                                                     ^^^^
    |
 note: previously specified here
-  --> $DIR/diagnostic-derive-inline.rs:547:39
+  --> $DIR/diagnostic-derive-inline.rs:555:39
    |
 LL |     #[suggestion("with a suggestion", code = "...", code = ",,,")]
    |                                       ^^^^
 
 error: derive(Diagnostic): wrong types for suggestion
-  --> $DIR/diagnostic-derive-inline.rs:556:24
+  --> $DIR/diagnostic-derive-inline.rs:564:24
    |
 LL |     suggestion: (Span, usize),
    |                        ^^^^^
@@ -379,7 +385,7 @@ LL |     suggestion: (Span, usize),
    = help: `#[suggestion(...)]` on a tuple field must be applied to fields of type `(Span, Applicability)`
 
 error: derive(Diagnostic): wrong types for suggestion
-  --> $DIR/diagnostic-derive-inline.rs:564:17
+  --> $DIR/diagnostic-derive-inline.rs:572:17
    |
 LL |     suggestion: (Span,),
    |                 ^^^^^^^
@@ -387,13 +393,13 @@ LL |     suggestion: (Span,),
    = help: `#[suggestion(...)]` on a tuple field must be applied to fields of type `(Span, Applicability)`
 
 error: derive(Diagnostic): suggestion without `code = "..."`
-  --> $DIR/diagnostic-derive-inline.rs:571:5
+  --> $DIR/diagnostic-derive-inline.rs:579:5
    |
 LL |     #[suggestion("with a suggestion")]
    |     ^
 
 error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:578:1
+  --> $DIR/diagnostic-derive-inline.rs:586:1
    |
 LL | #[multipart_suggestion("with a suggestion")]
    | ^
@@ -401,7 +407,7 @@ LL | #[multipart_suggestion("with a suggestion")]
    = help: consider creating a `Subdiagnostic` instead
 
 error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:581:1
+  --> $DIR/diagnostic-derive-inline.rs:589:1
    |
 LL | #[multipart_suggestion()]
    | ^
@@ -409,7 +415,7 @@ LL | #[multipart_suggestion()]
    = help: consider creating a `Subdiagnostic` instead
 
 error: derive(Diagnostic): `#[multipart_suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:585:5
+  --> $DIR/diagnostic-derive-inline.rs:593:5
    |
 LL |     #[multipart_suggestion("with a suggestion")]
    |     ^
@@ -417,7 +423,7 @@ LL |     #[multipart_suggestion("with a suggestion")]
    = help: consider creating a `Subdiagnostic` instead
 
 error: derive(Diagnostic): `#[suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:593:1
+  --> $DIR/diagnostic-derive-inline.rs:601:1
    |
 LL | #[suggestion("with a suggestion", code = "...")]
    | ^
@@ -425,41 +431,35 @@ LL | #[suggestion("with a suggestion", code = "...")]
    = help: `#[label]` and `#[suggestion]` can only be applied to fields
 
 error: derive(Diagnostic): `#[label]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:602:1
+  --> $DIR/diagnostic-derive-inline.rs:610:1
    |
 LL | #[label]
    | ^
    |
-   = help: `#[label]` and `#[suggestion]` can only be applied to fields
+   = help: subdiagnostic message is missing
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:636:5
+  --> $DIR/diagnostic-derive-inline.rs:644:5
    |
 LL |     #[subdiagnostic(bad)]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic = ...]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:644:5
+  --> $DIR/diagnostic-derive-inline.rs:652:5
    |
 LL |     #[subdiagnostic = "bad"]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:652:5
+  --> $DIR/diagnostic-derive-inline.rs:660:5
    |
 LL |     #[subdiagnostic(bad, bad)]
    |     ^
 
-error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:660:5
-   |
-LL |     #[subdiagnostic("bad")]
-   |     ^
-
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
   --> $DIR/diagnostic-derive-inline.rs:668:5
    |
-LL |     #[subdiagnostic(eager)]
+LL |     #[subdiagnostic("bad")]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
@@ -469,37 +469,43 @@ LL |     #[subdiagnostic(eager)]
    |     ^
 
 error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:697:5
+  --> $DIR/diagnostic-derive-inline.rs:684:5
+   |
+LL |     #[subdiagnostic(eager)]
+   |     ^
+
+error: derive(Diagnostic): `#[subdiagnostic(...)]` is not a valid attribute
+  --> $DIR/diagnostic-derive-inline.rs:705:5
    |
 LL |     #[subdiagnostic(eager)]
    |     ^
 
 error: derive(Diagnostic): expected at least one string literal for `code(...)`
-  --> $DIR/diagnostic-derive-inline.rs:728:44
+  --> $DIR/diagnostic-derive-inline.rs:736:44
    |
 LL |     #[suggestion("with a suggestion", code())]
    |                                            ^
 
 error: derive(Diagnostic): `code(...)` must contain only string literals
-  --> $DIR/diagnostic-derive-inline.rs:736:44
+  --> $DIR/diagnostic-derive-inline.rs:744:44
    |
 LL |     #[suggestion("with a suggestion", code(foo))]
    |                                            ^^^
 
 error: unexpected token, expected `)`
-  --> $DIR/diagnostic-derive-inline.rs:736:44
+  --> $DIR/diagnostic-derive-inline.rs:744:44
    |
 LL |     #[suggestion("with a suggestion", code(foo))]
    |                                            ^^^
 
 error: expected string literal
-  --> $DIR/diagnostic-derive-inline.rs:745:46
+  --> $DIR/diagnostic-derive-inline.rs:753:46
    |
 LL |     #[suggestion("with a suggestion", code = 3)]
    |                                              ^
 
 error: derive(Diagnostic): `#[suggestion(...)]` is not a valid attribute
-  --> $DIR/diagnostic-derive-inline.rs:760:5
+  --> $DIR/diagnostic-derive-inline.rs:768:5
    |
 LL |     #[suggestion("with a suggestion", code = "")]
    |     ^
@@ -509,7 +515,7 @@ LL |     #[suggestion("with a suggestion", code = "")]
    = help: to show a variable set of suggestions, use a `Vec` of `Subdiagnostic`s annotated with `#[suggestion(...)]`
 
 error: derive(Diagnostic): Variable `nosub` not found in diagnostic 
-  --> $DIR/diagnostic-derive-inline.rs:772:8
+  --> $DIR/diagnostic-derive-inline.rs:780:8
    |
 LL | #[diag("does not exist: {$nosub}")]
    |        ^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -529,7 +535,7 @@ LL |     #[nonsense]
    |       ^^^^^^^^
 
 error: cannot find attribute `error` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:517:3
+  --> $DIR/diagnostic-derive-inline.rs:525:3
    |
 LL | #[error("this is an example message", code = E0123)]
    |   ^^^^^
@@ -541,7 +547,7 @@ LL | struct ErrorAttribute {}
    |
 
 error: cannot find attribute `warn_` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:524:3
+  --> $DIR/diagnostic-derive-inline.rs:532:3
    |
 LL | #[warn_("this is an example message", code = E0123)]
    |   ^^^^^
@@ -553,7 +559,7 @@ LL + #[warn("this is an example message", code = E0123)]
    |
 
 error: cannot find attribute `lint` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:531:3
+  --> $DIR/diagnostic-derive-inline.rs:539:3
    |
 LL | #[lint("this is an example message", code = E0123)]
    |   ^^^^
@@ -565,7 +571,7 @@ LL + #[link("this is an example message", code = E0123)]
    |
 
 error: cannot find attribute `lint` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:538:3
+  --> $DIR/diagnostic-derive-inline.rs:546:3
    |
 LL | #[lint("this is an example message", code = E0123)]
    |   ^^^^
@@ -577,7 +583,7 @@ LL + #[link("this is an example message", code = E0123)]
    |
 
 error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:578:3
+  --> $DIR/diagnostic-derive-inline.rs:586:3
    |
 LL | #[multipart_suggestion("with a suggestion")]
    |   ^^^^^^^^^^^^^^^^^^^^
@@ -589,7 +595,7 @@ LL | struct MultipartSuggestion {
    |
 
 error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:581:3
+  --> $DIR/diagnostic-derive-inline.rs:589:3
    |
 LL | #[multipart_suggestion()]
    |   ^^^^^^^^^^^^^^^^^^^^
@@ -601,7 +607,7 @@ LL | struct MultipartSuggestion {
    |
 
 error: cannot find attribute `multipart_suggestion` in this scope
-  --> $DIR/diagnostic-derive-inline.rs:585:7
+  --> $DIR/diagnostic-derive-inline.rs:593:7
    |
 LL |     #[multipart_suggestion("with a suggestion")]
    |       ^^^^^^^^^^^^^^^^^^^^
@@ -630,6 +636,6 @@ note: required by a bound in `Diag::<'a, G>::arg`
    = note: in this macro invocation
    = note: this error originates in the macro `with_fn` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: aborting due to 80 previous errors
+error: aborting due to 81 previous errors
 
 For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
index 079c2a74c4a1..99b2cba89da4 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
@@ -54,7 +54,7 @@ struct C {
 
 #[derive(Subdiagnostic)]
 #[label]
-//~^ ERROR diagnostic slug must be first argument
+//~^ ERROR diagnostic message must be first argument
 struct D {
     #[primary_span]
     span: Span,
@@ -83,7 +83,7 @@ struct F {
 #[derive(Subdiagnostic)]
 #[label(bug = "...")]
 //~^ ERROR no nested attribute expected here
-//~| ERROR diagnostic slug must be first argument
+//~| ERROR diagnostic message must be first argument
 struct G {
     #[primary_span]
     span: Span,
@@ -93,7 +93,7 @@ struct G {
 #[derive(Subdiagnostic)]
 #[label(slug = 4)]
 //~^ ERROR no nested attribute expected here
-//~| ERROR diagnostic slug must be first argument
+//~| ERROR diagnostic message must be first argument
 struct J {
     #[primary_span]
     span: Span,
@@ -103,7 +103,7 @@ struct J {
 #[derive(Subdiagnostic)]
 #[label(slug("..."))]
 //~^ ERROR no nested attribute expected here
-//~| ERROR diagnostic slug must be first argument
+//~| ERROR diagnostic message must be first argument
 struct K {
     #[primary_span]
     span: Span,
@@ -112,7 +112,7 @@ struct K {
 
 #[derive(Subdiagnostic)]
 #[label()]
-//~^ ERROR diagnostic slug must be first argument of a `#[label(...)]` attribute
+//~^ ERROR diagnostic message must be first argument of a `#[label(...)]` attribute
 struct M {
     #[primary_span]
     span: Span,
@@ -201,7 +201,7 @@ enum T {
 #[derive(Subdiagnostic)]
 enum U {
     #[label(code = "...")]
-    //~^ ERROR diagnostic slug must be first argument of a `#[label(...)]` attribute
+    //~^ ERROR diagnostic message must be first argument of a `#[label(...)]` attribute
     //~| ERROR no nested attribute expected here
     A {
         #[primary_span]
@@ -303,7 +303,7 @@ struct AD {
 
 #[derive(Subdiagnostic)]
 #[label("example message", no_crate::example)]
-//~^ ERROR a diagnostic slug must be the first argument to the attribute
+//~^ ERROR expected this path to be an identifier
 struct AE {
     #[primary_span]
     span: Span,
@@ -771,7 +771,7 @@ struct SuggestionStyleInvalid2 {
 
 #[derive(Subdiagnostic)]
 #[suggestion("example message", code = "", style)]
-//~^ ERROR a diagnostic slug must be the first argument to the attribute
+//~^ ERROR expected `=`
 struct SuggestionStyleInvalid3 {
     #[primary_span]
     sub: Span,
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
index 8ff5ed00d0ba..90e0c285ee74 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
@@ -4,7 +4,7 @@ error: derive(Diagnostic): label without `#[primary_span]` field
 LL | #[label("example message")]
    | ^
 
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
+error: derive(Diagnostic): diagnostic message must be first argument of a `#[label(...)]` attribute
   --> $DIR/subdiagnostic-derive-inline.rs:56:1
    |
 LL | #[label]
@@ -28,7 +28,7 @@ error: derive(Diagnostic): no nested attribute expected here
 LL | #[label(bug = "...")]
    |         ^^^
 
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
+error: derive(Diagnostic): diagnostic message must be first argument of a `#[label(...)]` attribute
   --> $DIR/subdiagnostic-derive-inline.rs:84:1
    |
 LL | #[label(bug = "...")]
@@ -40,7 +40,7 @@ error: derive(Diagnostic): no nested attribute expected here
 LL | #[label(slug = 4)]
    |         ^^^^
 
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
+error: derive(Diagnostic): diagnostic message must be first argument of a `#[label(...)]` attribute
   --> $DIR/subdiagnostic-derive-inline.rs:94:1
    |
 LL | #[label(slug = 4)]
@@ -52,13 +52,13 @@ error: derive(Diagnostic): no nested attribute expected here
 LL | #[label(slug("..."))]
    |         ^^^^
 
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
+error: derive(Diagnostic): diagnostic message must be first argument of a `#[label(...)]` attribute
   --> $DIR/subdiagnostic-derive-inline.rs:104:1
    |
 LL | #[label(slug("..."))]
    | ^
 
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
+error: derive(Diagnostic): diagnostic message must be first argument of a `#[label(...)]` attribute
   --> $DIR/subdiagnostic-derive-inline.rs:114:1
    |
 LL | #[label()]
@@ -112,7 +112,7 @@ error: derive(Diagnostic): no nested attribute expected here
 LL |     #[label(code = "...")]
    |             ^^^^
 
-error: derive(Diagnostic): diagnostic slug must be first argument of a `#[label(...)]` attribute
+error: derive(Diagnostic): diagnostic message must be first argument of a `#[label(...)]` attribute
   --> $DIR/subdiagnostic-derive-inline.rs:203:5
    |
 LL |     #[label(code = "...")]
@@ -168,11 +168,11 @@ LL | |     b: u64,
 LL | | }
    | |_^
 
-error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
+error: expected this path to be an identifier
   --> $DIR/subdiagnostic-derive-inline.rs:305:28
    |
 LL | #[label("example message", no_crate::example)]
-   |                            ^^^^^^^^
+   |                            ^^^^^^^^^^^^^^^^^
 
 error: derive(Diagnostic): attribute specified multiple times
   --> $DIR/subdiagnostic-derive-inline.rs:318:5
@@ -464,11 +464,11 @@ error: expected string literal
 LL | #[suggestion("example message", code = "", style = 42)]
    |                                                    ^^
 
-error: derive(Diagnostic): a diagnostic slug must be the first argument to the attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:773:44
+error: expected `=`
+  --> $DIR/subdiagnostic-derive-inline.rs:773:49
    |
 LL | #[suggestion("example message", code = "", style)]
-   |                                            ^^^^^
+   |                                                 ^
 
 error: expected `=`
   --> $DIR/subdiagnostic-derive-inline.rs:781:49

From 0db0acd6993cbdf84384b00773d7509df6bc20fb Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 20:08:45 +0100
Subject: [PATCH 1449/1843] Remove the fallback bundle

---
 compiler/rustc_driver_impl/src/lib.rs         |  7 ++-----
 compiler/rustc_errors/src/json/tests.rs       |  2 +-
 compiler/rustc_errors/src/translation.rs      | 17 ++--------------
 compiler/rustc_interface/src/interface.rs     |  4 ----
 compiler/rustc_interface/src/tests.rs         |  1 -
 compiler/rustc_parse/src/parser/tests.rs      | 20 +++++++++----------
 compiler/rustc_session/src/parse.rs           |  6 +++---
 compiler/rustc_session/src/session.rs         | 13 +++---------
 src/librustdoc/clean/render_macro_matchers.rs |  2 +-
 src/librustdoc/core.rs                        |  1 -
 src/librustdoc/doctest.rs                     |  1 -
 .../ui-fulldeps/mod_dir_path_canonicalized.rs |  2 +-
 tests/ui-fulldeps/pprust-expr-roundtrip.rs    |  2 +-
 .../pprust-parenthesis-insertion.rs           |  2 +-
 tests/ui-fulldeps/run-compiler-twice.rs       |  1 -
 15 files changed, 25 insertions(+), 56 deletions(-)

diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index d72913764e9d..83b27520457b 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -108,11 +108,9 @@ use crate::session_diagnostics::{
 };
 
 pub fn default_translator() -> Translator {
-    Translator::with_fallback_bundle(DEFAULT_LOCALE_RESOURCES.to_vec(), false)
+    Translator::new()
 }
 
-pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[];
-
 /// Exit status code used for successful compilation and help output.
 pub const EXIT_SUCCESS: i32 = 0;
 
@@ -219,7 +217,6 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
         output_dir: odir,
         ice_file,
         file_loader: None,
-        locale_resources: DEFAULT_LOCALE_RESOURCES.to_vec(),
         lint_caps: Default::default(),
         psess_created: None,
         hash_untracked_state: None,
@@ -1528,7 +1525,7 @@ fn report_ice(
     extra_info: fn(&DiagCtxt),
     using_internal_features: &AtomicBool,
 ) {
-    let translator = default_translator();
+    let translator = Translator::new();
     let emitter =
         Box::new(rustc_errors::annotate_snippet_emitter_writer::AnnotateSnippetEmitter::new(
             stderr_destination(rustc_errors::ColorConfig::Auto),
diff --git a/compiler/rustc_errors/src/json/tests.rs b/compiler/rustc_errors/src/json/tests.rs
index 6f99930c433b..ffcf9947a7a8 100644
--- a/compiler/rustc_errors/src/json/tests.rs
+++ b/compiler/rustc_errors/src/json/tests.rs
@@ -45,7 +45,7 @@ fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
     rustc_span::create_default_session_globals_then(|| {
         let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
         sm.new_source_file(filename(&sm, "test.rs"), code.to_owned());
-        let translator = Translator::with_fallback_bundle(vec![], false);
+        let translator = Translator::new();
 
         let output = Arc::new(Mutex::new(Vec::new()));
         let je = JsonEmitter::new(
diff --git a/compiler/rustc_errors/src/translation.rs b/compiler/rustc_errors/src/translation.rs
index 0b9845772590..b5de3f87e9f3 100644
--- a/compiler/rustc_errors/src/translation.rs
+++ b/compiler/rustc_errors/src/translation.rs
@@ -34,24 +34,11 @@ pub struct Translator {
     /// Localized diagnostics for the locale requested by the user. If no language was requested by
     /// the user then this will be `None` and `fallback_fluent_bundle` should be used.
     pub fluent_bundle: Option>,
-    /// Return `FluentBundle` with localized diagnostics for the default locale of the compiler.
-    /// Used when the user has not requested a specific language or when a localized diagnostic is
-    /// unavailable for the requested locale.
-    pub fallback_fluent_bundle: LazyFallbackBundle,
 }
 
 impl Translator {
-    pub fn with_fallback_bundle(
-        resources: Vec<&'static str>,
-        with_directionality_markers: bool,
-    ) -> Translator {
-        Translator {
-            fluent_bundle: None,
-            fallback_fluent_bundle: crate::fallback_fluent_bundle(
-                resources,
-                with_directionality_markers,
-            ),
-        }
+    pub fn new() -> Translator {
+        Translator { fluent_bundle: None }
     }
 
     /// Convert `DiagMessage`s to a string, performing translation if necessary.
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 722e9a420540..72080c5f5369 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -332,9 +332,6 @@ pub struct Config {
     /// bjorn3 for "hooking rust-analyzer's VFS into rustc at some point for
     /// running rustc without having to save". (See #102759.)
     pub file_loader: Option>,
-    /// The list of fluent resources, used for lints declared with
-    /// [`Diagnostic`](rustc_errors::Diagnostic) and [`LintDiagnostic`](rustc_errors::LintDiagnostic).
-    pub locale_resources: Vec<&'static str>,
 
     pub lint_caps: FxHashMap,
 
@@ -458,7 +455,6 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se
                     temps_dir,
                 },
                 bundle,
-                config.locale_resources,
                 config.lint_caps,
                 target,
                 util::rustc_version_str().unwrap_or("unknown"),
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 4241af244941..5ebf898f45a3 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -72,7 +72,6 @@ where
             sessopts,
             io,
             None,
-            vec![],
             Default::default(),
             target,
             "",
diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs
index 1b17f422345f..ac9c3e63e853 100644
--- a/compiler/rustc_parse/src/parser/tests.rs
+++ b/compiler/rustc_parse/src/parser/tests.rs
@@ -42,7 +42,7 @@ fn string_to_parser(psess: &ParseSess, source_str: String) -> Parser<'_> {
 fn create_test_handler(theme: OutputTheme) -> (DiagCtxt, Arc, Arc>>) {
     let output = Arc::new(Mutex::new(Vec::new()));
     let source_map = Arc::new(SourceMap::new(FilePathMapping::empty()));
-    let translator = Translator::with_fallback_bundle(vec![], false);
+    let translator = Translator::new();
     let shared: Box = Box::new(Shared { data: output.clone() });
     let auto_stream = AutoStream::never(shared);
     let dcx = DiagCtxt::new(Box::new(
@@ -89,7 +89,7 @@ where
 
 /// Maps a string to tts, using a made-up filename.
 pub(crate) fn string_to_stream(source_str: String) -> TokenStream {
-    let psess = ParseSess::new(vec![]);
+    let psess = ParseSess::new();
     unwrap_or_emit_fatal(source_str_to_stream(
         &psess,
         filename(psess.source_map(), "bogofile"),
@@ -2239,12 +2239,12 @@ fn sp(a: u32, b: u32) -> Span {
 
 /// Parses a string, return an expression.
 fn string_to_expr(source_str: String) -> Box {
-    with_error_checking_parse(source_str, &ParseSess::new(vec![]), |p| p.parse_expr())
+    with_error_checking_parse(source_str, &ParseSess::new(), |p| p.parse_expr())
 }
 
 /// Parses a string, returns an item.
 fn string_to_item(source_str: String) -> Option> {
-    with_error_checking_parse(source_str, &ParseSess::new(vec![]), |p| {
+    with_error_checking_parse(source_str, &ParseSess::new(), |p| {
         p.parse_item(ForceCollect::No, AllowConstBlockItems::Yes)
     })
 }
@@ -2480,7 +2480,7 @@ let mut fflags: c_int = wb();
 #[test]
 fn crlf_doc_comments() {
     create_default_session_globals_then(|| {
-        let psess = ParseSess::new(vec![]);
+        let psess = ParseSess::new();
 
         let name_1 = FileName::Custom("crlf_source_1".to_string());
         let source = "/// doc comment\r\nfn foo() {}".to_string();
@@ -2515,7 +2515,7 @@ fn ttdelim_span() {
     }
 
     create_default_session_globals_then(|| {
-        let psess = ParseSess::new(vec![]);
+        let psess = ParseSess::new();
         let expr = parse_expr_from_source_str(
             filename(psess.source_map(), "foo"),
             "foo!( fn main() { body } )".to_string(),
@@ -2551,7 +2551,7 @@ fn look_ahead() {
         let sym_S = Symbol::intern("S");
         let raw_no = IdentIsRaw::No;
 
-        let psess = ParseSess::new(vec![]);
+        let psess = ParseSess::new();
         let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
 
         // Current position is the `fn`.
@@ -2626,7 +2626,7 @@ fn look_ahead_non_outermost_stream() {
         let sym_S = Symbol::intern("S");
         let raw_no = IdentIsRaw::No;
 
-        let psess = ParseSess::new(vec![]);
+        let psess = ParseSess::new();
         let mut p = string_to_parser(&psess, "mod m { fn f(x: u32) { x } struct S; }".to_string());
 
         // Move forward to the `fn`, which is not within the outermost token
@@ -2658,7 +2658,7 @@ fn look_ahead_non_outermost_stream() {
 #[test]
 fn debug_lookahead() {
     create_default_session_globals_then(|| {
-        let psess = ParseSess::new(vec![]);
+        let psess = ParseSess::new();
         let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
 
         // Current position is the `fn`.
@@ -2879,7 +2879,7 @@ fn debug_lookahead() {
 #[test]
 fn out_of_line_mod() {
     create_default_session_globals_then(|| {
-        let psess = ParseSess::new(vec![]);
+        let psess = ParseSess::new();
         let item = parse_item_from_source_str(
             filename(psess.source_map(), "foo"),
             "mod foo { struct S; mod this_does_not_exist; }".to_owned(),
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 7bebbc579c43..62c23424f371 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -280,8 +280,8 @@ pub struct ParseSess {
 
 impl ParseSess {
     /// Used for testing.
-    pub fn new(locale_resources: Vec<&'static str>) -> Self {
-        let translator = Translator::with_fallback_bundle(locale_resources, false);
+    pub fn new() -> Self {
+        let translator = Translator::new();
         let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
         let emitter = Box::new(
             AnnotateSnippetEmitter::new(stderr_destination(ColorConfig::Auto), translator)
@@ -314,7 +314,7 @@ impl ParseSess {
     }
 
     pub fn emitter_with_note(note: String) -> Self {
-        let translator = Translator::with_fallback_bundle(vec![], false);
+        let translator = Translator::new();
         let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
         let emitter = Box::new(AnnotateSnippetEmitter::new(
             stderr_destination(ColorConfig::Auto),
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 92c47e3bfb36..ed37e9e960cd 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -19,7 +19,7 @@ use rustc_errors::timings::TimingSectionHandler;
 use rustc_errors::translation::Translator;
 use rustc_errors::{
     Diag, DiagCtxt, DiagCtxtHandle, DiagMessage, Diagnostic, ErrorGuaranteed, FatalAbort,
-    TerminalUrl, fallback_fluent_bundle,
+    TerminalUrl,
 };
 use rustc_hir::limit::Limit;
 use rustc_macros::HashStable_Generic;
@@ -967,7 +967,6 @@ pub fn build_session(
     sopts: config::Options,
     io: CompilerIO,
     fluent_bundle: Option>,
-    fluent_resources: Vec<&'static str>,
     driver_lint_caps: FxHashMap,
     target: Target,
     cfg_version: &'static str,
@@ -985,13 +984,7 @@ pub fn build_session(
     let cap_lints_allow = sopts.lint_cap.is_some_and(|cap| cap == lint::Allow);
     let can_emit_warnings = !(warnings_allow || cap_lints_allow);
 
-    let translator = Translator {
-        fluent_bundle,
-        fallback_fluent_bundle: fallback_fluent_bundle(
-            fluent_resources,
-            sopts.unstable_opts.translate_directionality_markers,
-        ),
-    };
+    let translator = Translator { fluent_bundle };
     let source_map = rustc_span::source_map::get_source_map().unwrap();
     let emitter = default_emitter(&sopts, Arc::clone(&source_map), translator);
 
@@ -1430,7 +1423,7 @@ impl EarlyDiagCtxt {
 fn mk_emitter(output: ErrorOutputType) -> Box {
     // FIXME(#100717): early errors aren't translated at the moment, so this is fine, but it will
     // need to reference every crate that might emit an early error for translation to work.
-    let translator = Translator::with_fallback_bundle(vec![], false);
+    let translator = Translator::new();
     let emitter: Box = match output {
         config::ErrorOutputType::HumanReadable { kind, color_config } => match kind {
             HumanReadableErrorType { short, unicode } => Box::new(
diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs
index b5a8d64ff4f5..a69e3808bd7f 100644
--- a/src/librustdoc/clean/render_macro_matchers.rs
+++ b/src/librustdoc/clean/render_macro_matchers.rs
@@ -63,7 +63,7 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option, input: Input, options: RustdocOptions
         output_file: None,
         output_dir: None,
         file_loader: None,
-        locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
         lint_caps,
         psess_created: None,
         hash_untracked_state: None,
diff --git a/tests/ui-fulldeps/mod_dir_path_canonicalized.rs b/tests/ui-fulldeps/mod_dir_path_canonicalized.rs
index 894fec248f1f..806d395a90ad 100644
--- a/tests/ui-fulldeps/mod_dir_path_canonicalized.rs
+++ b/tests/ui-fulldeps/mod_dir_path_canonicalized.rs
@@ -29,7 +29,7 @@ pub fn main() {
 }
 
 fn parse() {
-    let psess = ParseSess::new(vec![]);
+    let psess = ParseSess::new();
 
     let path = Path::new(file!());
     let path = path.canonicalize().unwrap();
diff --git a/tests/ui-fulldeps/pprust-expr-roundtrip.rs b/tests/ui-fulldeps/pprust-expr-roundtrip.rs
index 7fe71004522a..8d7425186680 100644
--- a/tests/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/tests/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -219,7 +219,7 @@ fn main() {
 }
 
 fn run() {
-    let psess = ParseSess::new(vec![]);
+    let psess = ParseSess::new();
 
     iter_exprs(2, &mut |mut e| {
         // If the pretty printer is correct, then `parse(print(e))` should be identical to `e`,
diff --git a/tests/ui-fulldeps/pprust-parenthesis-insertion.rs b/tests/ui-fulldeps/pprust-parenthesis-insertion.rs
index 90cf9b89a293..85ee87155a7a 100644
--- a/tests/ui-fulldeps/pprust-parenthesis-insertion.rs
+++ b/tests/ui-fulldeps/pprust-parenthesis-insertion.rs
@@ -196,7 +196,7 @@ fn main() -> ExitCode {
     };
 
     rustc_span::create_default_session_globals_then(|| {
-        let psess = &ParseSess::new(vec![]);
+        let psess = &ParseSess::new();
 
         for &source_code in EXPRS {
             let Some(expr) = parse_expr(psess, source_code) else {
diff --git a/tests/ui-fulldeps/run-compiler-twice.rs b/tests/ui-fulldeps/run-compiler-twice.rs
index 8b5e99cbee8d..bd92cc6218f1 100644
--- a/tests/ui-fulldeps/run-compiler-twice.rs
+++ b/tests/ui-fulldeps/run-compiler-twice.rs
@@ -64,7 +64,6 @@ fn compile(code: String, output: PathBuf, sysroot: Sysroot, linker: Option<&Path
         output_dir: None,
         ice_file: None,
         file_loader: None,
-        locale_resources: Vec::new(),
         lint_caps: Default::default(),
         psess_created: None,
         hash_untracked_state: None,

From 81d42146040c4a6b3d252e3dc3ac32e563694796 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Fri, 6 Feb 2026 20:09:13 +0100
Subject: [PATCH 1450/1843] Remove fluent tests in `rustc_error`

---
 compiler/rustc_errors/src/lib.rs   |   2 -
 compiler/rustc_errors/src/tests.rs | 182 -----------------------------
 2 files changed, 184 deletions(-)
 delete mode 100644 compiler/rustc_errors/src/tests.rs

diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 534fe32a897f..2e0afb8fc4c2 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -83,8 +83,6 @@ pub mod error;
 pub mod json;
 mod lock;
 pub mod markdown;
-#[cfg(test)]
-mod tests;
 pub mod timings;
 pub mod translation;
 
diff --git a/compiler/rustc_errors/src/tests.rs b/compiler/rustc_errors/src/tests.rs
deleted file mode 100644
index 34ebac0fde11..000000000000
--- a/compiler/rustc_errors/src/tests.rs
+++ /dev/null
@@ -1,182 +0,0 @@
-use std::sync::{Arc, LazyLock};
-
-use rustc_data_structures::sync::IntoDynSyncSend;
-use rustc_error_messages::fluent_bundle::resolver::errors::{ReferenceKind, ResolverError};
-use rustc_error_messages::{DiagMessage, langid};
-
-use crate::FluentBundle;
-use crate::error::{TranslateError, TranslateErrorKind};
-use crate::fluent_bundle::*;
-use crate::translation::Translator;
-
-fn make_translator(ftl: &'static str) -> Translator {
-    let resource = FluentResource::try_new(ftl.into()).expect("Failed to parse an FTL string.");
-
-    let langid_en = langid!("en-US");
-
-    let mut bundle: FluentBundle =
-        IntoDynSyncSend(crate::fluent_bundle::bundle::FluentBundle::new_concurrent(vec![
-            langid_en,
-        ]));
-
-    bundle.add_resource(resource).expect("Failed to add FTL resources to the bundle.");
-
-    Translator {
-        fluent_bundle: None,
-        fallback_fluent_bundle: Arc::new(LazyLock::new(Box::new(|| bundle))),
-    }
-}
-
-#[test]
-fn wellformed_fluent() {
-    let translator = make_translator("mir_build_borrow_of_moved_value = borrow of moved value
-    .label = value moved into `{$name}` here
-    .occurs_because_label = move occurs because `{$name}` has type `{$ty}` which does not implement the `Copy` trait
-    .value_borrowed_label = value borrowed here after move
-    .suggestion = borrow this binding in the pattern to avoid moving the value");
-
-    let mut args = FluentArgs::new();
-    args.set("name", "Foo");
-    args.set("ty", "std::string::String");
-    {
-        let message = DiagMessage::FluentIdentifier(
-            "mir_build_borrow_of_moved_value".into(),
-            Some("suggestion".into()),
-        );
-
-        assert_eq!(
-            translator.translate_message(&message, &args).unwrap(),
-            "borrow this binding in the pattern to avoid moving the value"
-        );
-    }
-
-    {
-        let message = DiagMessage::FluentIdentifier(
-            "mir_build_borrow_of_moved_value".into(),
-            Some("value_borrowed_label".into()),
-        );
-
-        assert_eq!(
-            translator.translate_message(&message, &args).unwrap(),
-            "value borrowed here after move"
-        );
-    }
-
-    {
-        let message = DiagMessage::FluentIdentifier(
-            "mir_build_borrow_of_moved_value".into(),
-            Some("occurs_because_label".into()),
-        );
-
-        assert_eq!(
-            translator.translate_message(&message, &args).unwrap(),
-            "move occurs because `\u{2068}Foo\u{2069}` has type `\u{2068}std::string::String\u{2069}` which does not implement the `Copy` trait"
-        );
-
-        {
-            let message = DiagMessage::FluentIdentifier(
-                "mir_build_borrow_of_moved_value".into(),
-                Some("label".into()),
-            );
-
-            assert_eq!(
-                translator.translate_message(&message, &args).unwrap(),
-                "value moved into `\u{2068}Foo\u{2069}` here"
-            );
-        }
-    }
-}
-
-#[test]
-fn misformed_fluent() {
-    let translator = make_translator("mir_build_borrow_of_moved_value = borrow of moved value
-    .label = value moved into `{name}` here
-    .occurs_because_label = move occurs because `{$oops}` has type `{$ty}` which does not implement the `Copy` trait
-    .suggestion = borrow this binding in the pattern to avoid moving the value");
-
-    let mut args = FluentArgs::new();
-    args.set("name", "Foo");
-    args.set("ty", "std::string::String");
-    {
-        let message = DiagMessage::FluentIdentifier(
-            "mir_build_borrow_of_moved_value".into(),
-            Some("value_borrowed_label".into()),
-        );
-
-        let err = translator.translate_message(&message, &args).unwrap_err();
-        assert!(
-            matches!(
-                &err,
-                TranslateError::Two {
-                    primary: box TranslateError::One {
-                        kind: TranslateErrorKind::PrimaryBundleMissing,
-                        ..
-                    },
-                    fallback: box TranslateError::One {
-                        kind: TranslateErrorKind::AttributeMissing { attr: "value_borrowed_label" },
-                        ..
-                    }
-                }
-            ),
-            "{err:#?}"
-        );
-        assert_eq!(
-            format!("{err}"),
-            "failed while formatting fluent string `mir_build_borrow_of_moved_value`: \nthe attribute `value_borrowed_label` was missing\nhelp: add `.value_borrowed_label = `\n"
-        );
-    }
-
-    {
-        let message = DiagMessage::FluentIdentifier(
-            "mir_build_borrow_of_moved_value".into(),
-            Some("label".into()),
-        );
-
-        let err = translator.translate_message(&message, &args).unwrap_err();
-        if let TranslateError::Two {
-            primary: box TranslateError::One { kind: TranslateErrorKind::PrimaryBundleMissing, .. },
-            fallback: box TranslateError::One { kind: TranslateErrorKind::Fluent { errs }, .. },
-        } = &err
-            && let [
-                FluentError::ResolverError(ResolverError::Reference(
-                    ReferenceKind::Message { id, .. } | ReferenceKind::Variable { id, .. },
-                )),
-            ] = &**errs
-            && id == "name"
-        {
-        } else {
-            panic!("{err:#?}")
-        };
-        assert_eq!(
-            format!("{err}"),
-            "failed while formatting fluent string `mir_build_borrow_of_moved_value`: \nargument `name` exists but was not referenced correctly\nhelp: try using `{$name}` instead\n"
-        );
-    }
-
-    {
-        let message = DiagMessage::FluentIdentifier(
-            "mir_build_borrow_of_moved_value".into(),
-            Some("occurs_because_label".into()),
-        );
-
-        let err = translator.translate_message(&message, &args).unwrap_err();
-        if let TranslateError::Two {
-            primary: box TranslateError::One { kind: TranslateErrorKind::PrimaryBundleMissing, .. },
-            fallback: box TranslateError::One { kind: TranslateErrorKind::Fluent { errs }, .. },
-        } = &err
-            && let [
-                FluentError::ResolverError(ResolverError::Reference(
-                    ReferenceKind::Message { id, .. } | ReferenceKind::Variable { id, .. },
-                )),
-            ] = &**errs
-            && id == "oops"
-        {
-        } else {
-            panic!("{err:#?}")
-        };
-        assert_eq!(
-            format!("{err}"),
-            "failed while formatting fluent string `mir_build_borrow_of_moved_value`: \nthe fluent string has an argument `oops` that was not found.\nhelp: the arguments `name` and `ty` are available\n"
-        );
-    }
-}

From 0f3108327b7679e13da6e07957508a1ab3ce2043 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Wed, 4 Feb 2026 13:53:29 +1100
Subject: [PATCH 1451/1843] Streamline `rustc_span::HashStableContext`.

Currently this trait has five methods. But it only really needs three.

For example, currently stable hashing of spans is implemented in
`rustc_span`, except a couple of sub-operations are delegated to
`rustc_query_system`: `def_span` and `span_data_to_lines_and_cols`.
These two delegated sub-operations can be reduced to a single delegated
operation that does the full hash computation.

Likewise, `assert_default_hashing_controls` depends on two delegated
sub-operations, `hashing_controls` and
`unstable_opts_incremental_ignore_spans`, and can be simplified.

I find the resulting code simpler and clearer -- when necessary, we do a
whole operation in `rustc_query_system` instead of doing it partly in
`rustc_span` and partly in `rustc_query_system`.
---
 compiler/rustc_query_system/src/ich/hcx.rs | 126 ++++++++++++++++++---
 compiler/rustc_span/src/hygiene.rs         |  29 +----
 compiler/rustc_span/src/lib.rs             | 102 ++---------------
 3 files changed, 121 insertions(+), 136 deletions(-)

diff --git a/compiler/rustc_query_system/src/ich/hcx.rs b/compiler/rustc_query_system/src/ich/hcx.rs
index 397ec7994f18..2e118dc3359f 100644
--- a/compiler/rustc_query_system/src/ich/hcx.rs
+++ b/compiler/rustc_query_system/src/ich/hcx.rs
@@ -1,10 +1,12 @@
-use rustc_data_structures::stable_hasher::HashingControls;
+use std::hash::Hash;
+
+use rustc_data_structures::stable_hasher::{HashStable, HashingControls, StableHasher};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::definitions::DefPathHash;
 use rustc_session::Session;
 use rustc_session::cstore::Untracked;
 use rustc_span::source_map::SourceMap;
-use rustc_span::{BytePos, CachingSourceMapView, DUMMY_SP, SourceFile, Span, SpanData};
+use rustc_span::{CachingSourceMapView, DUMMY_SP, Pos, Span};
 
 // Very often, we are hashing something that does not need the `CachingSourceMapView`, so we
 // initialize it lazily.
@@ -60,6 +62,11 @@ impl<'a> StableHashingContext<'a> {
         }
     }
 
+    #[inline]
+    fn def_span(&self, def_id: LocalDefId) -> Span {
+        self.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP)
+    }
+
     #[inline]
     pub fn hashing_controls(&self) -> HashingControls {
         self.hashing_controls
@@ -67,9 +74,88 @@ impl<'a> StableHashingContext<'a> {
 }
 
 impl<'a> rustc_span::HashStableContext for StableHashingContext<'a> {
+    /// Hashes a span in a stable way. We can't directly hash the span's `BytePos` fields (that
+    /// would be similar to hashing pointers, since those are just offsets into the `SourceMap`).
+    /// Instead, we hash the (file name, line, column) triple, which stays the same even if the
+    /// containing `SourceFile` has moved within the `SourceMap`.
+    ///
+    /// Also note that we are hashing byte offsets for the column, not unicode codepoint offsets.
+    /// For the purpose of the hash that's sufficient. Also, hashing filenames is expensive so we
+    /// avoid doing it twice when the span starts and ends in the same file, which is almost always
+    /// the case.
+    ///
+    /// IMPORTANT: changes to this method should be reflected in implementations of `SpanEncoder`.
     #[inline]
-    fn unstable_opts_incremental_ignore_spans(&self) -> bool {
-        self.incremental_ignore_spans
+    fn span_hash_stable(&mut self, span: Span, hasher: &mut StableHasher) {
+        const TAG_VALID_SPAN: u8 = 0;
+        const TAG_INVALID_SPAN: u8 = 1;
+        const TAG_RELATIVE_SPAN: u8 = 2;
+
+        if !self.hashing_controls().hash_spans {
+            return;
+        }
+
+        let span = span.data_untracked();
+        span.ctxt.hash_stable(self, hasher);
+        span.parent.hash_stable(self, hasher);
+
+        if span.is_dummy() {
+            Hash::hash(&TAG_INVALID_SPAN, hasher);
+            return;
+        }
+
+        let parent = span.parent.map(|parent| self.def_span(parent).data_untracked());
+        if let Some(parent) = parent
+            && parent.contains(span)
+        {
+            // This span is enclosed in a definition: only hash the relative position. This catches
+            // a subset of the cases from the `file.contains(parent.lo)`. But we can do this check
+            // cheaply without the expensive `span_data_to_lines_and_cols` query.
+            Hash::hash(&TAG_RELATIVE_SPAN, hasher);
+            (span.lo - parent.lo).to_u32().hash_stable(self, hasher);
+            (span.hi - parent.lo).to_u32().hash_stable(self, hasher);
+            return;
+        }
+
+        // If this is not an empty or invalid span, we want to hash the last position that belongs
+        // to it, as opposed to hashing the first position past it.
+        let Some((file, line_lo, col_lo, line_hi, col_hi)) =
+            self.source_map().span_data_to_lines_and_cols(&span)
+        else {
+            Hash::hash(&TAG_INVALID_SPAN, hasher);
+            return;
+        };
+
+        if let Some(parent) = parent
+            && file.contains(parent.lo)
+        {
+            // This span is relative to another span in the same file,
+            // only hash the relative position.
+            Hash::hash(&TAG_RELATIVE_SPAN, hasher);
+            Hash::hash(&(span.lo.0.wrapping_sub(parent.lo.0)), hasher);
+            Hash::hash(&(span.hi.0.wrapping_sub(parent.lo.0)), hasher);
+            return;
+        }
+
+        Hash::hash(&TAG_VALID_SPAN, hasher);
+        Hash::hash(&file.stable_id, hasher);
+
+        // Hash both the length and the end location (line/column) of a span. If we hash only the
+        // length, for example, then two otherwise equal spans with different end locations will
+        // have the same hash. This can cause a problem during incremental compilation wherein a
+        // previous result for a query that depends on the end location of a span will be
+        // incorrectly reused when the end location of the span it depends on has changed (see
+        // issue #74890). A similar analysis applies if some query depends specifically on the
+        // length of the span, but we only hash the end location. So hash both.
+
+        let col_lo_trunc = (col_lo.0 as u64) & 0xFF;
+        let line_lo_trunc = ((line_lo as u64) & 0xFF_FF_FF) << 8;
+        let col_hi_trunc = (col_hi.0 as u64) & 0xFF << 32;
+        let line_hi_trunc = ((line_hi as u64) & 0xFF_FF_FF) << 40;
+        let col_line = col_lo_trunc | line_lo_trunc | col_hi_trunc | line_hi_trunc;
+        let len = (span.hi - span.lo).0;
+        Hash::hash(&col_line, hasher);
+        Hash::hash(&len, hasher);
     }
 
     #[inline]
@@ -81,22 +167,26 @@ impl<'a> rustc_span::HashStableContext for StableHashingContext<'a> {
         }
     }
 
+    /// Assert that the provided `HashStableContext` is configured with the default
+    /// `HashingControls`. We should always have bailed out before getting to here with a
+    /// non-default mode. With this check in place, we can avoid the need to maintain separate
+    /// versions of `ExpnData` hashes for each permutation of `HashingControls` settings.
     #[inline]
-    fn def_span(&self, def_id: LocalDefId) -> Span {
-        self.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP)
-    }
+    fn assert_default_hashing_controls(&self, msg: &str) {
+        let hashing_controls = self.hashing_controls;
+        let HashingControls { hash_spans } = hashing_controls;
 
-    #[inline]
-    fn span_data_to_lines_and_cols(
-        &mut self,
-        span: &SpanData,
-    ) -> Option<(&SourceFile, usize, BytePos, usize, BytePos)> {
-        self.source_map().span_data_to_lines_and_cols(span)
-    }
-
-    #[inline]
-    fn hashing_controls(&self) -> HashingControls {
-        self.hashing_controls
+        // Note that we require that `hash_spans` be the inverse of the global `-Z
+        // incremental-ignore-spans` option. Normally, this option is disabled, in which case
+        // `hash_spans` must be true.
+        //
+        // Span hashing can also be disabled without `-Z incremental-ignore-spans`. This is the
+        // case for instance when building a hash for name mangling. Such configuration must not be
+        // used for metadata.
+        assert_eq!(
+            hash_spans, !self.incremental_ignore_spans,
+            "Attempted hashing of {msg} with non-default HashingControls: {hashing_controls:?}"
+        );
     }
 }
 
diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs
index 07a85ad75daa..25834f03af5e 100644
--- a/compiler/rustc_span/src/hygiene.rs
+++ b/compiler/rustc_span/src/hygiene.rs
@@ -30,7 +30,7 @@ use std::{fmt, iter, mem};
 
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::stable_hasher::{HashStable, HashingControls, StableHasher};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_hashes::Hash64;
@@ -126,29 +126,6 @@ rustc_index::newtype_index! {
 impl !Ord for LocalExpnId {}
 impl !PartialOrd for LocalExpnId {}
 
-/// Assert that the provided `HashStableContext` is configured with the 'default'
-/// `HashingControls`. We should always have bailed out before getting to here
-/// with a non-default mode. With this check in place, we can avoid the need
-/// to maintain separate versions of `ExpnData` hashes for each permutation
-/// of `HashingControls` settings.
-fn assert_default_hashing_controls(ctx: &impl HashStableContext, msg: &str) {
-    let hashing_controls = ctx.hashing_controls();
-    let HashingControls { hash_spans } = hashing_controls;
-
-    // Note that we require that `hash_spans` be the inverse of the global
-    // `-Z incremental-ignore-spans` option. Normally, this option is disabled,
-    // in which case `hash_spans` must be true.
-    //
-    // Span hashing can also be disabled without `-Z incremental-ignore-spans`.
-    // This is the case for instance when building a hash for name mangling.
-    // Such configuration must not be used for metadata.
-    assert_eq!(
-        hash_spans,
-        !ctx.unstable_opts_incremental_ignore_spans(),
-        "Attempted hashing of {msg} with non-default HashingControls: {hashing_controls:?}"
-    );
-}
-
 /// A unique hash value associated to an expansion.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encodable, Decodable, HashStable_Generic)]
 pub struct ExpnHash(Fingerprint);
@@ -1508,7 +1485,7 @@ pub fn raw_encode_syntax_context(
 fn update_disambiguator(expn_data: &mut ExpnData, mut ctx: impl HashStableContext) -> ExpnHash {
     // This disambiguator should not have been set yet.
     assert_eq!(expn_data.disambiguator, 0, "Already set disambiguator for ExpnData: {expn_data:?}");
-    assert_default_hashing_controls(&ctx, "ExpnData (disambiguator)");
+    ctx.assert_default_hashing_controls("ExpnData (disambiguator)");
     let mut expn_hash = expn_data.hash_expn(&mut ctx);
 
     let disambiguator = HygieneData::with(|data| {
@@ -1558,7 +1535,7 @@ impl HashStable for SyntaxContext {
 
 impl HashStable for ExpnId {
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
-        assert_default_hashing_controls(ctx, "ExpnId");
+        ctx.assert_default_hashing_controls("ExpnId");
         let hash = if *self == ExpnId::root() {
             // Avoid fetching TLS storage for a trivial often-used value.
             Fingerprint::ZERO
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 0ad80b9d7879..bececc548494 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -54,7 +54,6 @@ use hygiene::Transparency;
 pub use hygiene::{
     DesugaringKind, ExpnData, ExpnHash, ExpnId, ExpnKind, LocalExpnId, MacroKind, SyntaxContext,
 };
-use rustc_data_structures::stable_hasher::HashingControls;
 pub mod def_id;
 use def_id::{CrateNum, DefId, DefIndex, DefPathHash, LOCAL_CRATE, LocalDefId, StableCrateId};
 pub mod edit_distance;
@@ -2798,105 +2797,24 @@ impl InnerSpan {
 /// This is a hack to allow using the [`HashStable_Generic`] derive macro
 /// instead of implementing everything in rustc_middle.
 pub trait HashStableContext {
+    /// The main event: stable hashing of a span.
+    fn span_hash_stable(&mut self, span: Span, hasher: &mut StableHasher);
+
+    /// Compute a `DefPathHash`.
     fn def_path_hash(&self, def_id: DefId) -> DefPathHash;
-    /// Accesses `sess.opts.unstable_opts.incremental_ignore_spans` since
-    /// we don't have easy access to a `Session`
-    fn unstable_opts_incremental_ignore_spans(&self) -> bool;
-    fn def_span(&self, def_id: LocalDefId) -> Span;
-    fn span_data_to_lines_and_cols(
-        &mut self,
-        span: &SpanData,
-    ) -> Option<(&SourceFile, usize, BytePos, usize, BytePos)>;
-    fn hashing_controls(&self) -> HashingControls;
+
+    /// Assert that the provided `HashStableContext` is configured with the default
+    /// `HashingControls`. We should always have bailed out before getting to here with a
+    fn assert_default_hashing_controls(&self, msg: &str);
 }
 
 impl HashStable for Span
 where
     CTX: HashStableContext,
 {
-    /// Hashes a span in a stable way. We can't directly hash the span's `BytePos`
-    /// fields (that would be similar to hashing pointers, since those are just
-    /// offsets into the `SourceMap`). Instead, we hash the (file name, line, column)
-    /// triple, which stays the same even if the containing `SourceFile` has moved
-    /// within the `SourceMap`.
-    ///
-    /// Also note that we are hashing byte offsets for the column, not unicode
-    /// codepoint offsets. For the purpose of the hash that's sufficient.
-    /// Also, hashing filenames is expensive so we avoid doing it twice when the
-    /// span starts and ends in the same file, which is almost always the case.
-    ///
-    /// IMPORTANT: changes to this method should be reflected in implementations of `SpanEncoder`.
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
-        const TAG_VALID_SPAN: u8 = 0;
-        const TAG_INVALID_SPAN: u8 = 1;
-        const TAG_RELATIVE_SPAN: u8 = 2;
-
-        if !ctx.hashing_controls().hash_spans {
-            return;
-        }
-
-        let span = self.data_untracked();
-        span.ctxt.hash_stable(ctx, hasher);
-        span.parent.hash_stable(ctx, hasher);
-
-        if span.is_dummy() {
-            Hash::hash(&TAG_INVALID_SPAN, hasher);
-            return;
-        }
-
-        let parent = span.parent.map(|parent| ctx.def_span(parent).data_untracked());
-        if let Some(parent) = parent
-            && parent.contains(span)
-        {
-            // This span is enclosed in a definition: only hash the relative position.
-            // This catches a subset of the cases from the `file.contains(parent.lo)`,
-            // But we can do this check cheaply without the expensive `span_data_to_lines_and_cols` query.
-            Hash::hash(&TAG_RELATIVE_SPAN, hasher);
-            (span.lo - parent.lo).to_u32().hash_stable(ctx, hasher);
-            (span.hi - parent.lo).to_u32().hash_stable(ctx, hasher);
-            return;
-        }
-
-        // If this is not an empty or invalid span, we want to hash the last
-        // position that belongs to it, as opposed to hashing the first
-        // position past it.
-        let Some((file, line_lo, col_lo, line_hi, col_hi)) = ctx.span_data_to_lines_and_cols(&span)
-        else {
-            Hash::hash(&TAG_INVALID_SPAN, hasher);
-            return;
-        };
-
-        if let Some(parent) = parent
-            && file.contains(parent.lo)
-        {
-            // This span is relative to another span in the same file,
-            // only hash the relative position.
-            Hash::hash(&TAG_RELATIVE_SPAN, hasher);
-            Hash::hash(&(span.lo.0.wrapping_sub(parent.lo.0)), hasher);
-            Hash::hash(&(span.hi.0.wrapping_sub(parent.lo.0)), hasher);
-            return;
-        }
-
-        Hash::hash(&TAG_VALID_SPAN, hasher);
-        Hash::hash(&file.stable_id, hasher);
-
-        // Hash both the length and the end location (line/column) of a span. If we
-        // hash only the length, for example, then two otherwise equal spans with
-        // different end locations will have the same hash. This can cause a problem
-        // during incremental compilation wherein a previous result for a query that
-        // depends on the end location of a span will be incorrectly reused when the
-        // end location of the span it depends on has changed (see issue #74890). A
-        // similar analysis applies if some query depends specifically on the length
-        // of the span, but we only hash the end location. So hash both.
-
-        let col_lo_trunc = (col_lo.0 as u64) & 0xFF;
-        let line_lo_trunc = ((line_lo as u64) & 0xFF_FF_FF) << 8;
-        let col_hi_trunc = (col_hi.0 as u64) & 0xFF << 32;
-        let line_hi_trunc = ((line_hi as u64) & 0xFF_FF_FF) << 40;
-        let col_line = col_lo_trunc | line_lo_trunc | col_hi_trunc | line_hi_trunc;
-        let len = (span.hi - span.lo).0;
-        Hash::hash(&col_line, hasher);
-        Hash::hash(&len, hasher);
+        // `span_hash_stable` does all the work.
+        ctx.span_hash_stable(*self, hasher)
     }
 }
 

From 842bed6f939f74c8c005bc223acb50e22464d108 Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Wed, 4 Feb 2026 20:32:49 +0000
Subject: [PATCH 1452/1843] Parse #[rustc_delayed_bug_from_inside_query]

---
 compiler/rustc_attr_parsing/src/attributes/test_attrs.rs | 9 +++++++++
 compiler/rustc_attr_parsing/src/context.rs               | 1 +
 compiler/rustc_hir/src/attrs/data_structures.rs          | 3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs       | 1 +
 compiler/rustc_interface/src/passes.rs                   | 4 ++--
 compiler/rustc_passes/src/check_attr.rs                  | 2 +-
 6 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
index 908941c2b335..504466d3795a 100644
--- a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
@@ -190,3 +190,12 @@ impl SingleAttributeParser for RustcAbiParser {
         Some(AttributeKind::RustcAbi { attr_span: cx.attr_span, kind })
     }
 }
+
+pub(crate) struct RustcDelayedBugFromInsideQueryParser;
+
+impl NoArgsAttributeParser for RustcDelayedBugFromInsideQueryParser {
+    const PATH: &[Symbol] = &[sym::rustc_delayed_bug_from_inside_query];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcDelayedBugFromInsideQuery;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index ad39b0ec26d9..704a2f3c8d10 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -262,6 +262,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 79452913c259..ca1913faeea1 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1084,6 +1084,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_def_path]`
     RustcDefPath(Span),
 
+    /// Represents `#[rustc_delayed_bug_from_inside_query]`
+    RustcDelayedBugFromInsideQuery,
+
     /// Represents `#[rustc_deny_explicit_impl]`.
     RustcDenyExplicitImpl(Span),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index ba351d80544c..33661807e1a9 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -106,6 +106,7 @@ impl AttributeKind {
             RustcConstStabilityIndirect => No,
             RustcDeallocator => No,
             RustcDefPath(..) => No,
+            RustcDelayedBugFromInsideQuery => No,
             RustcDenyExplicitImpl(..) => No,
             RustcDummy => No,
             RustcDumpDefParents => No,
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index f5fb8031ab0f..fe406f79f8dc 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -18,11 +18,11 @@ use rustc_errors::timings::TimingSection;
 use rustc_expand::base::{ExtCtxt, LintStoreExpand};
 use rustc_feature::Features;
 use rustc_fs_util::try_canonicalize;
-use rustc_hir::Attribute;
 use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def_id::{LOCAL_CRATE, StableCrateId, StableCrateIdMap};
 use rustc_hir::definitions::Definitions;
 use rustc_hir::limit::Limit;
+use rustc_hir::{Attribute, find_attr};
 use rustc_incremental::setup_dep_graph;
 use rustc_lint::{BufferedEarlyLint, EarlyCheckNode, LintStore, unerased_lint_store};
 use rustc_metadata::EncodedMetadata;
@@ -1227,7 +1227,7 @@ pub(crate) fn start_codegen<'tcx>(
 
     // Hook for tests.
     if let Some((def_id, _)) = tcx.entry_fn(())
-        && tcx.has_attr(def_id, sym::rustc_delayed_bug_from_inside_query)
+        && find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcDelayedBugFromInsideQuery)
     {
         tcx.ensure_ok().trigger_delayed_bug(def_id);
     }
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 6c4b76a08ad2..a67ee43a9b2e 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -301,6 +301,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcConstStabilityIndirect
                     | AttributeKind::RustcDeallocator
                     | AttributeKind::RustcDefPath(..)
+                    | AttributeKind::RustcDelayedBugFromInsideQuery
                     | AttributeKind::RustcDenyExplicitImpl(..)
                     | AttributeKind::RustcDummy
                     | AttributeKind::RustcDumpDefParents
@@ -406,7 +407,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_mir
                             | sym::rustc_outlives
                             | sym::rustc_evaluate_where_clauses
-                            | sym::rustc_delayed_bug_from_inside_query
                             | sym::rustc_partition_reused
                             | sym::rustc_partition_codegened
                             | sym::rustc_expected_cgu_reuse

From 76684962741d4647638d61b5466a4f0e4ca76258 Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Wed, 4 Feb 2026 20:51:17 +0000
Subject: [PATCH 1453/1843] Parse #[rustc_evaluate_where_clauses]

---
 .../src/attributes/test_attrs.rs                  | 15 +++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs        |  1 +
 compiler/rustc_hir/src/attrs/data_structures.rs   |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_hir_typeck/src/callee.rs           |  8 ++++++--
 compiler/rustc_passes/src/check_attr.rs           |  2 +-
 6 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
index 504466d3795a..9effba39571c 100644
--- a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
@@ -199,3 +199,18 @@ impl NoArgsAttributeParser for RustcDelayedBugFromInsideQueryParser
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcDelayedBugFromInsideQuery;
 }
+
+pub(crate) struct RustcEvaluateWhereClausesParser;
+
+impl NoArgsAttributeParser for RustcEvaluateWhereClausesParser {
+    const PATH: &[Symbol] = &[sym::rustc_evaluate_where_clauses];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+        Allow(Target::Method(MethodKind::Trait { body: false })),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcEvaluateWhereClauses;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 704a2f3c8d10..d714a6399b82 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -269,6 +269,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index ca1913faeea1..39f76002c419 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1114,6 +1114,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_effective_visibility]`.
     RustcEffectiveVisibility,
 
+    /// Represents `#[rustc_evaluate_where_clauses]`
+    RustcEvaluateWhereClauses,
+
     /// Represents `#[rustc_has_incoherent_inherent_impls]`
     RustcHasIncoherentInherentImpls,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 33661807e1a9..0b223c82c9b7 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -116,6 +116,7 @@ impl AttributeKind {
             RustcDumpVtable(..) => No,
             RustcDynIncompatibleTrait(..) => No,
             RustcEffectiveVisibility => Yes,
+            RustcEvaluateWhereClauses => Yes,
             RustcHasIncoherentInherentImpls => Yes,
             RustcHiddenTypeOfOpaques => No,
             RustcIfThisChanged(..) => No,
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index d04bd6f2295a..e20d5cdf2889 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -3,9 +3,10 @@ use std::iter;
 use rustc_abi::{CanonAbi, ExternAbi};
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, StashKey, inline_fluent};
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::{self, CtorKind, Namespace, Res};
 use rustc_hir::def_id::DefId;
-use rustc_hir::{self as hir, HirId, LangItem};
+use rustc_hir::{self as hir, HirId, LangItem, find_attr};
 use rustc_hir_analysis::autoderef::Autoderef;
 use rustc_infer::infer::BoundRegionConversionTime;
 use rustc_infer::traits::{Obligation, ObligationCause, ObligationCauseCode};
@@ -526,7 +527,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // `#[rustc_evaluate_where_clauses]` trigger special output
                 // to let us test the trait evaluation system.
                 if self.has_rustc_attrs
-                    && self.tcx.has_attr(def_id, sym::rustc_evaluate_where_clauses)
+                    && find_attr!(
+                        self.tcx.get_all_attrs(def_id),
+                        AttributeKind::RustcEvaluateWhereClauses
+                    )
                 {
                     let predicates = self.tcx.predicates_of(def_id);
                     let predicates = predicates.instantiate(self.tcx, args);
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index a67ee43a9b2e..47dce81b8f5a 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -311,6 +311,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDumpVtable(..)
                     | AttributeKind::RustcDynIncompatibleTrait(..)
                     | AttributeKind::RustcEffectiveVisibility
+                    | AttributeKind::RustcEvaluateWhereClauses
                     | AttributeKind::RustcHasIncoherentInherentImpls
                     | AttributeKind::RustcHiddenTypeOfOpaques
                     | AttributeKind::RustcIfThisChanged(..)
@@ -406,7 +407,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_strict_coherence
                             | sym::rustc_mir
                             | sym::rustc_outlives
-                            | sym::rustc_evaluate_where_clauses
                             | sym::rustc_partition_reused
                             | sym::rustc_partition_codegened
                             | sym::rustc_expected_cgu_reuse

From 6361d5f279058c287d8684b232df512fe2562512 Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Wed, 4 Feb 2026 21:32:14 +0000
Subject: [PATCH 1454/1843] Parse #[rustc_outlives]

---
 .../src/attributes/test_attrs.rs                   | 14 ++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs         |  1 +
 compiler/rustc_hir/src/attrs/data_structures.rs    |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 compiler/rustc_hir_analysis/src/outlives/dump.rs   |  4 +++-
 compiler/rustc_passes/src/check_attr.rs            |  2 +-
 6 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
index 9effba39571c..18d700a20d31 100644
--- a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs
@@ -214,3 +214,17 @@ impl NoArgsAttributeParser for RustcEvaluateWhereClausesParser {
     ]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcEvaluateWhereClauses;
 }
+
+pub(crate) struct RustcOutlivesParser;
+
+impl NoArgsAttributeParser for RustcOutlivesParser {
+    const PATH: &[Symbol] = &[sym::rustc_outlives];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Struct),
+        Allow(Target::Enum),
+        Allow(Target::Union),
+        Allow(Target::TyAlias),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcOutlives;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index d714a6399b82..f2d5d1c8c37e 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -283,6 +283,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 39f76002c419..4208da6ce722 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1192,6 +1192,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_offload_kernel]`
     RustcOffloadKernel,
 
+    /// Represents `#[rustc_outlives]`
+    RustcOutlives,
+
     /// Represents `#[rustc_paren_sugar]`.
     RustcParenSugar(Span),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 0b223c82c9b7..c87f31209ce5 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -142,6 +142,7 @@ impl AttributeKind {
             RustcObjcSelector { .. } => No,
             RustcObjectLifetimeDefault => No,
             RustcOffloadKernel => Yes,
+            RustcOutlives => No,
             RustcParenSugar(..) => No,
             RustcPassByValue(..) => Yes,
             RustcPassIndirectlyInNonRusticAbis(..) => No,
diff --git a/compiler/rustc_hir_analysis/src/outlives/dump.rs b/compiler/rustc_hir_analysis/src/outlives/dump.rs
index 4233896c3720..cf770a2561db 100644
--- a/compiler/rustc_hir_analysis/src/outlives/dump.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/dump.rs
@@ -1,10 +1,12 @@
+use rustc_hir::attrs::AttributeKind;
+use rustc_hir::find_attr;
 use rustc_middle::bug;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::sym;
 
 pub(crate) fn inferred_outlives(tcx: TyCtxt<'_>) {
     for id in tcx.hir_free_items() {
-        if !tcx.has_attr(id.owner_id, sym::rustc_outlives) {
+        if !find_attr!(tcx.get_all_attrs(id.owner_id), AttributeKind::RustcOutlives) {
             continue;
         }
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 47dce81b8f5a..5f8e887675d0 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -334,6 +334,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcObjcClass { .. }
                     | AttributeKind::RustcObjcSelector { .. }
                     | AttributeKind::RustcOffloadKernel
+                    | AttributeKind::RustcOutlives
                     | AttributeKind::RustcParenSugar(..)
                     | AttributeKind::RustcPassByValue (..)
                     | AttributeKind::RustcPassIndirectlyInNonRusticAbis(..)
@@ -406,7 +407,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_capture_analysis
                             | sym::rustc_strict_coherence
                             | sym::rustc_mir
-                            | sym::rustc_outlives
                             | sym::rustc_partition_reused
                             | sym::rustc_partition_codegened
                             | sym::rustc_expected_cgu_reuse

From fb55b5dcf33c3f134a41851771c3b13710a8a048 Mon Sep 17 00:00:00 2001
From: Jonatan Lindh 
Date: Sun, 8 Feb 2026 13:20:13 +0100
Subject: [PATCH 1455/1843] diagnostics: fix ICE in closure signature mismatch

This fixes the ICE by renaming conflicting arguments in the diagnostic.
---
 compiler/rustc_trait_selection/src/errors.rs  |  8 ++---
 .../closure-arg-borrow-ice-issue-152331.rs    | 10 ++++++
 ...closure-arg-borrow-ice-issue-152331.stderr | 32 +++++++++++++++++++
 3 files changed, 46 insertions(+), 4 deletions(-)
 create mode 100644 tests/ui/closures/closure-arg-borrow-ice-issue-152331.rs
 create mode 100644 tests/ui/closures/closure-arg-borrow-ice-issue-152331.stderr

diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs
index e0690b8a6245..c904eb83896d 100644
--- a/compiler/rustc_trait_selection/src/errors.rs
+++ b/compiler/rustc_trait_selection/src/errors.rs
@@ -153,10 +153,10 @@ impl Subdiagnostic for AdjustSignatureBorrow {
     fn add_to_diag(self, diag: &mut Diag<'_, G>) {
         match self {
             AdjustSignatureBorrow::Borrow { to_borrow } => {
-                diag.arg("len", to_borrow.len());
+                diag.arg("borrow_len", to_borrow.len());
                 diag.multipart_suggestion_verbose(
                     inline_fluent!(
-                        "consider adjusting the signature so it borrows its {$len ->
+                        "consider adjusting the signature so it borrows its {$borrow_len ->
 [one] argument
 *[other] arguments
 }"
@@ -166,10 +166,10 @@ impl Subdiagnostic for AdjustSignatureBorrow {
                 );
             }
             AdjustSignatureBorrow::RemoveBorrow { remove_borrow } => {
-                diag.arg("len", remove_borrow.len());
+                diag.arg("remove_borrow_len", remove_borrow.len());
                 diag.multipart_suggestion_verbose(
                     inline_fluent!(
-                        "consider adjusting the signature so it does not borrow its {$len ->
+                        "consider adjusting the signature so it does not borrow its {$remove_borrow_len ->
 [one] argument
 *[other] arguments
 }"
diff --git a/tests/ui/closures/closure-arg-borrow-ice-issue-152331.rs b/tests/ui/closures/closure-arg-borrow-ice-issue-152331.rs
new file mode 100644
index 000000000000..261f8fedc38f
--- /dev/null
+++ b/tests/ui/closures/closure-arg-borrow-ice-issue-152331.rs
@@ -0,0 +1,10 @@
+fn main() {
+    h2(|_: (), _: (), _: (), x: &_| {});
+    //~^ ERROR type mismatch in closure arguments
+}
+
+fn h2(_: F)
+where
+    F: for<'t0> Fn(&(), Box, &'t0 (), fn(&(), &())),
+{
+}
diff --git a/tests/ui/closures/closure-arg-borrow-ice-issue-152331.stderr b/tests/ui/closures/closure-arg-borrow-ice-issue-152331.stderr
new file mode 100644
index 000000000000..7cfc9f0576ec
--- /dev/null
+++ b/tests/ui/closures/closure-arg-borrow-ice-issue-152331.stderr
@@ -0,0 +1,32 @@
+error[E0631]: type mismatch in closure arguments
+  --> $DIR/closure-arg-borrow-ice-issue-152331.rs:2:5
+   |
+LL |     h2(|_: (), _: (), _: (), x: &_| {});
+   |     ^^^----------------------------^^^^
+   |     |  |
+   |     |  found signature defined here
+   |     expected due to this
+   |
+   = note: expected closure signature `for<'a, 't0> fn(&'a (), Box<(dyn for<'a> Fn(&'a ()) + 'static)>, &'t0 (), for<'a, 'b> fn(&'a (), &'b ())) -> _`
+              found closure signature `fn((), (), (), &_) -> _`
+note: required by a bound in `h2`
+  --> $DIR/closure-arg-borrow-ice-issue-152331.rs:8:8
+   |
+LL | fn h2(_: F)
+   |    -- required by a bound in this function
+LL | where
+LL |     F: for<'t0> Fn(&(), Box, &'t0 (), fn(&(), &())),
+   |        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `h2`
+help: consider adjusting the signature so it borrows its arguments
+   |
+LL |     h2(|_: &(), _: (), _: &(), x: &_| {});
+   |            +              +
+help: consider adjusting the signature so it does not borrow its argument
+   |
+LL -     h2(|_: (), _: (), _: (), x: &_| {});
+LL +     h2(|_: (), _: (), _: (), x: _| {});
+   |
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0631`.

From 053f76e9f07c1e307cd343a59cff6ed3276fc4ad Mon Sep 17 00:00:00 2001
From: Folkert de Vries 
Date: Sun, 8 Feb 2026 13:46:29 +0100
Subject: [PATCH 1456/1843] `cfg_select!`: allow optional comma after { /* ...
 */ }

---
 compiler/rustc_parse/src/parser/cfg_select.rs | 13 +++++---
 tests/ui/macros/cfg_select.rs                 |  7 ++--
 tests/ui/macros/cfg_select.stderr             | 32 +++++++++----------
 3 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/cfg_select.rs b/compiler/rustc_parse/src/parser/cfg_select.rs
index 9692d2c70893..b12209f0b92e 100644
--- a/compiler/rustc_parse/src/parser/cfg_select.rs
+++ b/compiler/rustc_parse/src/parser/cfg_select.rs
@@ -7,14 +7,19 @@ use crate::exp;
 use crate::parser::{AttrWrapper, ForceCollect, Parser, Restrictions, Trailing, UsePreAttrPos};
 
 impl<'a> Parser<'a> {
-    /// Parses a `TokenTree` consisting either of `{ /* ... */ }` (and strip the braces) or an
-    /// expression followed by a comma (and strip the comma).
+    /// Parses a `TokenTree` consisting either of `{ /* ... */ }` optionally followed by a comma
+    /// (and strip the braces and the optional comma) or an expression followed by a comma
+    /// (and strip the comma).
     pub fn parse_delimited_token_tree(&mut self) -> PResult<'a, TokenStream> {
         if self.token == token::OpenBrace {
             // Strip the outer '{' and '}'.
             match self.parse_token_tree() {
-                TokenTree::Token(..) => unreachable!("because of the expect above"),
-                TokenTree::Delimited(.., tts) => return Ok(tts),
+                TokenTree::Token(..) => unreachable!("because the current token is a '{{'"),
+                TokenTree::Delimited(.., tts) => {
+                    // Optionally end with a comma.
+                    let _ = self.eat(exp!(Comma));
+                    return Ok(tts);
+                }
             }
         }
         let expr = self.collect_tokens(None, AttrWrapper::empty(), ForceCollect::Yes, |p, _| {
diff --git a/tests/ui/macros/cfg_select.rs b/tests/ui/macros/cfg_select.rs
index 0f009756e211..693e27daad7a 100644
--- a/tests/ui/macros/cfg_select.rs
+++ b/tests/ui/macros/cfg_select.rs
@@ -34,6 +34,7 @@ fn arm_rhs_expr_3() -> i32 {
         any(true) => 1,
         any(false) => 2,
         any(true) => { 42 }
+        any(true) => { 42 },
         any(false) => -1 as i32,
         any(true) => 2 + 2,
         any(false) => "",
@@ -62,7 +63,7 @@ fn expand_to_statements() -> i32 {
 }
 
 type ExpandToType = cfg_select! {
-    unix => u32,
+    unix => { u32 },
     _ => i32,
 };
 
@@ -113,7 +114,7 @@ impl T for S {
     cfg_select! {
         false => {
             fn a() {}
-        }
+        },
         _ => {
             fn b() {}
         }
@@ -145,7 +146,7 @@ cfg_select! {
 
 cfg_select! {
     unix => {}
-    not(unix) => {}
+    not(unix) => {},
     _ => {}
     //~^ WARN unreachable configuration predicate
 }
diff --git a/tests/ui/macros/cfg_select.stderr b/tests/ui/macros/cfg_select.stderr
index 5878708668d3..2da0c2169497 100644
--- a/tests/ui/macros/cfg_select.stderr
+++ b/tests/ui/macros/cfg_select.stderr
@@ -1,5 +1,5 @@
 error: none of the predicates in this `cfg_select` evaluated to true
-  --> $DIR/cfg_select.rs:161:1
+  --> $DIR/cfg_select.rs:162:1
    |
 LL | / cfg_select! {
 LL | |
@@ -8,55 +8,55 @@ LL | | }
    | |_^
 
 error: none of the predicates in this `cfg_select` evaluated to true
-  --> $DIR/cfg_select.rs:166:1
+  --> $DIR/cfg_select.rs:167:1
    |
 LL | cfg_select! {}
    | ^^^^^^^^^^^^^^
 
 error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `=>`
-  --> $DIR/cfg_select.rs:170:5
+  --> $DIR/cfg_select.rs:171:5
    |
 LL |     => {}
    |     ^^
 
 error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found expression
-  --> $DIR/cfg_select.rs:175:5
+  --> $DIR/cfg_select.rs:176:5
    |
 LL |     () => {}
    |     ^^ expressions are not allowed here
 
 error[E0539]: malformed `cfg_select` macro input
-  --> $DIR/cfg_select.rs:180:5
+  --> $DIR/cfg_select.rs:181:5
    |
 LL |     "str" => {}
    |     ^^^^^ expected a valid identifier here
 
 error[E0539]: malformed `cfg_select` macro input
-  --> $DIR/cfg_select.rs:185:5
+  --> $DIR/cfg_select.rs:186:5
    |
 LL |     a::b => {}
    |     ^^^^ expected a valid identifier here
 
 error[E0537]: invalid predicate `a`
-  --> $DIR/cfg_select.rs:190:5
+  --> $DIR/cfg_select.rs:191:5
    |
 LL |     a() => {}
    |     ^^^
 
 error: expected one of `(`, `::`, `=>`, or `=`, found `+`
-  --> $DIR/cfg_select.rs:195:7
+  --> $DIR/cfg_select.rs:196:7
    |
 LL |     a + 1 => {}
    |       ^ expected one of `(`, `::`, `=>`, or `=`
 
 error: expected one of `(`, `::`, `=>`, or `=`, found `!`
-  --> $DIR/cfg_select.rs:201:8
+  --> $DIR/cfg_select.rs:202:8
    |
 LL |     cfg!() => {}
    |        ^ expected one of `(`, `::`, `=>`, or `=`
 
 warning: unreachable configuration predicate
-  --> $DIR/cfg_select.rs:136:5
+  --> $DIR/cfg_select.rs:137:5
    |
 LL |     _ => {}
    |     - always matches
@@ -70,7 +70,7 @@ LL | #![warn(unreachable_cfg_select_predicates)] // Unused warnings are disabled
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 warning: unreachable configuration predicate
-  --> $DIR/cfg_select.rs:142:5
+  --> $DIR/cfg_select.rs:143:5
    |
 LL |     true => {}
    |     ---- always matches
@@ -78,25 +78,25 @@ LL |     _ => {}
    |     ^ this configuration predicate is never reached
 
 warning: unreachable configuration predicate
-  --> $DIR/cfg_select.rs:149:5
+  --> $DIR/cfg_select.rs:150:5
    |
 LL |     _ => {}
    |     ^ this configuration predicate is never reached
 
 warning: unreachable configuration predicate
-  --> $DIR/cfg_select.rs:155:5
+  --> $DIR/cfg_select.rs:156:5
    |
 LL |     test => {}
    |     ^^^^ this configuration predicate is never reached
 
 warning: unreachable configuration predicate
-  --> $DIR/cfg_select.rs:157:5
+  --> $DIR/cfg_select.rs:158:5
    |
 LL |     _ => {}
    |     ^ this configuration predicate is never reached
 
 warning: unexpected `cfg` condition name: `a`
-  --> $DIR/cfg_select.rs:195:5
+  --> $DIR/cfg_select.rs:196:5
    |
 LL |     a + 1 => {}
    |     ^ help: found config with similar value: `target_feature = "a"`
@@ -107,7 +107,7 @@ LL |     a + 1 => {}
    = note: `#[warn(unexpected_cfgs)]` on by default
 
 warning: unexpected `cfg` condition name: `cfg`
-  --> $DIR/cfg_select.rs:201:5
+  --> $DIR/cfg_select.rs:202:5
    |
 LL |     cfg!() => {}
    |     ^^^

From f693a1df2b5fd345f73f78ac674c6b8c67314d61 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 14:12:42 +0100
Subject: [PATCH 1457/1843] Update diagnostics documentation

---
 .../src/diagnostics/diagnostic-structs.md     | 173 ++++++------------
 1 file changed, 53 insertions(+), 120 deletions(-)

diff --git a/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md b/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md
index 4e5c3413cb8a..2260b1ec4df1 100644
--- a/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md
+++ b/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md
@@ -12,7 +12,7 @@ customizing the rendering logic, or selecting messages at runtime, you will need
 the corresponding trait (`Diagnostic`, `LintDiagnostic`, or `Subdiagnostic`).
 This approach provides greater flexibility and is recommended for diagnostics that go beyond simple, static structures.
 
-Diagnostic can be translated into different languages and each has a slug that uniquely identifies the diagnostic.
+Diagnostic can be translated into different languages.
 
 ## `#[derive(Diagnostic)]` and `#[derive(LintDiagnostic)]`
 
@@ -21,13 +21,13 @@ shown below:
 
 ```rust,ignore
 #[derive(Diagnostic)]
-#[diag(hir_analysis_field_already_declared, code = E0124)]
+#[diag("field `{$field_name}` is already declared", code = E0124)]
 pub struct FieldAlreadyDeclared {
     pub field_name: Ident,
     #[primary_span]
-    #[label]
+    #[label("field already declared")]
     pub span: Span,
-    #[label(previous_decl_label)]
+    #[label("`{$field_name}` first declared here")]
     pub prev_span: Span,
 }
 ```
@@ -42,25 +42,10 @@ the `code` sub-attribute. Specifying a `code` isn't mandatory, but if you are
 porting a diagnostic that uses `Diag` to use `Diagnostic`
 then you should keep the code if there was one.
 
-`#[diag(..)]` must provide a slug as the first positional argument (a path to an
-item in `rustc_errors::fluent::*`). A slug uniquely identifies the diagnostic
-and is also how the compiler knows what error message to emit (in the default
-locale of the compiler, or in the locale requested by the user). See
+`#[diag(..)]` must provide a message as the first positional argument. 
+The message is written in English, but might be translated to the locale requested by the user. See
 [translation documentation](./translation.md) to learn more about how
-translatable error messages are written and how slug items are generated.
-
-In our example, the Fluent message for the "field already declared" diagnostic
-looks like this:
-
-```fluent
-hir_analysis_field_already_declared =
-    field `{$field_name}` is already declared
-    .label = field already declared
-    .previous_decl_label = `{$field_name}` first declared here
-```
-
-`hir_analysis_field_already_declared` is the slug from our example and is followed
-by the diagnostic message.
+translatable error messages are written and how they are generated.
 
 Every field of the `Diagnostic` which does not have an annotation is
 available in Fluent messages as a variable, like `field_name` in the example
@@ -76,13 +61,7 @@ specified on a `Diagnostic`.
 
 `#[label]`, `#[help]`, `#[warning]` and `#[note]` can all be applied to fields which have the
 type `Span`. Applying any of these attributes will create the corresponding
-subdiagnostic with that `Span`. These attributes will look for their
-diagnostic message in a Fluent attribute attached to the primary Fluent
-message. In our example, `#[label]` will look for
-`hir_analysis_field_already_declared.label` (which has the message "field already
-declared"). If there is more than one subdiagnostic of the same type, then
-these attributes can also take a value that is the attribute name to look for
-(e.g. `previous_decl_label` in our example).
+subdiagnostic with that `Span`. These attributes take a diagnostic message as an argument.
 
 Other types have special behavior when used in a `Diagnostic` derive:
 
@@ -99,17 +78,17 @@ represent optional `#[note]`/`#[help]`/`#[warning]` subdiagnostics.
 
 Suggestions can be emitted using one of four field attributes:
 
-- `#[suggestion(slug, code = "...", applicability = "...")]`
-- `#[suggestion_hidden(slug, code = "...", applicability = "...")]`
-- `#[suggestion_short(slug, code = "...", applicability = "...")]`
-- `#[suggestion_verbose(slug, code = "...", applicability = "...")]`
+- `#[suggestion("message", code = "...", applicability = "...")]`
+- `#[suggestion_hidden("message", code = "...", applicability = "...")]`
+- `#[suggestion_short("message", code = "...", applicability = "...")]`
+- `#[suggestion_verbose("message", code = "...", applicability = "...")]`
 
 Suggestions must be applied on either a `Span` field or a `(Span,
-MachineApplicability)` field. Similarly to other field attributes, the slug
-specifies the Fluent attribute with the message and defaults to the equivalent
-of `.suggestion`. `code` specifies the code that should be suggested as a
+MachineApplicability)` field. Similarly to other field attributes, a message
+needs to be provided which will be shown to the user.
+`code` specifies the code that should be suggested as a
 replacement and is a format string (e.g. `{field_name}` would be replaced by
-the value of the `field_name` field of the struct), not a Fluent identifier.
+the value of the `field_name` field of the struct).
 `applicability` can be used to specify the applicability in the attribute, it
 cannot be used when the field's type contains an `Applicability`.
 
@@ -119,15 +98,15 @@ In the end, the `Diagnostic` derive will generate an implementation of
 ```rust,ignore
 impl<'a, G: EmissionGuarantee> Diagnostic<'a> for FieldAlreadyDeclared {
     fn into_diag(self, dcx: &'a DiagCtxt, level: Level) -> Diag<'a, G> {
-        let mut diag = Diag::new(dcx, level, fluent::hir_analysis_field_already_declared);
+        let mut diag = Diag::new(dcx, level, "field `{$field_name}` is already declared");
         diag.set_span(self.span);
         diag.span_label(
             self.span,
-            fluent::hir_analysis_label
+            "field already declared"
         );
         diag.span_label(
             self.prev_span,
-            fluent::hir_analysis_previous_decl_label
+            "`{$field_name}` first declared here"
         );
         diag
     }
@@ -150,60 +129,40 @@ tcx.dcx().emit_err(FieldAlreadyDeclared {
 `#[derive(Diagnostic)]` and `#[derive(LintDiagnostic)]` support the
 following attributes:
 
-- `#[diag(slug, code = "...")]`
+- `#[diag("message", code = "...")]`
   - _Applied to struct or enum variant._
   - _Mandatory_
   - Defines the text and error code to be associated with the diagnostic.
-  - Slug (_Mandatory_)
-    - Uniquely identifies the diagnostic and corresponds to its Fluent message,
-      mandatory.
-    - A path to an item in `rustc_errors::fluent`, e.g.
-      `rustc_errors::fluent::hir_analysis_field_already_declared`
-      (`rustc_errors::fluent` is implicit in the attribute, so just
-      `hir_analysis_field_already_declared`).
+  - Message (_Mandatory_)
+    - The diagnostic message which will be shown to the user.
     - See [translation documentation](./translation.md).
   - `code = "..."` (_Optional_)
     - Specifies the error code.
-- `#[note]` or `#[note(slug)]` (_Optional_)
+- `#[note("message")]` (_Optional_)
   - _Applied to struct or struct fields of type `Span`, `Option<()>` or `()`._
   - Adds a note subdiagnostic.
-  - Value is a path to an item in `rustc_errors::fluent` for the note's
-    message.
-    - Defaults to equivalent of `.note`.
+  - Value is the note's message.
   - If applied to a `Span` field, creates a spanned note.
-- `#[help]` or `#[help(slug)]` (_Optional_)
+- `#[help("message")]` (_Optional_)
   - _Applied to struct or struct fields of type `Span`, `Option<()>` or `()`._
   - Adds a help subdiagnostic.
-  - Value is a path to an item in `rustc_errors::fluent` for the note's
-    message.
-    - Defaults to equivalent of `.help`.
+  - Value is the help message.
   - If applied to a `Span` field, creates a spanned help.
-- `#[label]` or `#[label(slug)]` (_Optional_)
+- `#[label("message")]` (_Optional_)
   - _Applied to `Span` fields._
   - Adds a label subdiagnostic.
-  - Value is a path to an item in `rustc_errors::fluent` for the note's
-    message.
-    - Defaults to equivalent of `.label`.
-- `#[warning]` or `#[warning(slug)]` (_Optional_)
+  - Value is the label's message.
+- `#[warning("message")]` (_Optional_)
   - _Applied to struct or struct fields of type `Span`, `Option<()>` or `()`._
   - Adds a warning subdiagnostic.
-  - Value is a path to an item in `rustc_errors::fluent` for the note's
-    message.
-    - Defaults to equivalent of `.warn`.
-- `#[suggestion{,_hidden,_short,_verbose}(slug, code = "...", applicability = "...")]`
+  - Value is the warning's message.
+- `#[suggestion{,_hidden,_short,_verbose}("message", code = "...", applicability = "...")]`
   (_Optional_)
   - _Applied to `(Span, MachineApplicability)` or `Span` fields._
   - Adds a suggestion subdiagnostic.
-  - Slug (_Mandatory_)
-    - A path to an item in `rustc_errors::fluent`, e.g.
-      `rustc_errors::fluent::hir_analysis_field_already_declared`
-      (`rustc_errors::fluent` is implicit in the attribute, so just
-      `hir_analysis_field_already_declared`). Fluent attributes for all messages
-      exist as top-level items in that module (so `hir_analysis_message.attr` is just
-      `attr`).
+  - Message (_Mandatory_)
+    - Value is the suggestion message that will be shown to the user.
     - See [translation documentation](./translation.md).
-    - Defaults to `rustc_errors::fluent::_subdiag::suggestion` (or
-    - `.suggestion` in Fluent).
   - `code = "..."`/`code("...", ...)` (_Mandatory_)
     - One or multiple format strings indicating the code to be suggested as a
       replacement. Multiple values signify multiple possible replacements.
@@ -235,12 +194,12 @@ shown below:
 ```rust
 #[derive(Subdiagnostic)]
 pub enum ExpectedReturnTypeLabel<'tcx> {
-    #[label(hir_analysis_expected_default_return_type)]
+    #[label("expected `()` because of default return type")]
     Unit {
         #[primary_span]
         span: Span,
     },
-    #[label(hir_analysis_expected_return_type)]
+    #[label("expected `{$expected}` because of return type")]
     Other {
         #[primary_span]
         span: Span,
@@ -260,21 +219,9 @@ attribute applied to the struct or each variant, one of:
 - `#[warning(..)]` for defining a warning
 - `#[suggestion{,_hidden,_short,_verbose}(..)]` for defining a suggestion
 
-All of the above must provide a slug as the first positional argument (a path
-to an item in `rustc_errors::fluent::*`). A slug uniquely identifies the
-diagnostic and is also how the compiler knows what error message to emit (in
-the default locale of the compiler, or in the locale requested by the user).
+All of the above must provide a diagnostic message as the first positional argument.
 See [translation documentation](./translation.md) to learn more about how
-translatable error messages are written and how slug items are generated.
-
-In our example, the Fluent message for the "expected return type" label
-looks like this:
-
-```fluent
-hir_analysis_expected_default_return_type = expected `()` because of default return type
-
-hir_analysis_expected_return_type = expected `{$expected}` because of return type
-```
+translatable error messages are generated.
 
 Using the `#[primary_span]` attribute on a field (with type `Span`) will denote
 the primary span of the subdiagnostic. A primary span is only necessary for a
@@ -289,17 +236,15 @@ Like `Diagnostic`, `Subdiagnostic` supports `Option` and
 
 Suggestions can be emitted using one of four attributes on the type/variant:
 
-- `#[suggestion(..., code = "...", applicability = "...")]`
-- `#[suggestion_hidden(..., code = "...", applicability = "...")]`
-- `#[suggestion_short(..., code = "...", applicability = "...")]`
-- `#[suggestion_verbose(..., code = "...", applicability = "...")]`
+- `#[suggestion("...", code = "...", applicability = "...")]`
+- `#[suggestion_hidden("...", code = "...", applicability = "...")]`
+- `#[suggestion_short("...", code = "...", applicability = "...")]`
+- `#[suggestion_verbose("...", code = "...", applicability = "...")]`
 
 Suggestions require `#[primary_span]` be set on a field and can have the
 following sub-attributes:
 
-- The first positional argument specifies the path to a item in
-  `rustc_errors::fluent` corresponding to the Fluent attribute with the message
-  and defaults to the equivalent of `.suggestion`.
+- The first positional argument specifies the message which will be shown to the user.
 - `code` specifies the code that should be suggested as a replacement and is a
   format string (e.g. `{field_name}` would be replaced by the value of the
   `field_name` field of the struct), not a Fluent identifier.
@@ -318,11 +263,11 @@ impl<'tcx> Subdiagnostic for ExpectedReturnTypeLabel<'tcx> {
         use rustc_errors::{Applicability, IntoDiagArg};
         match self {
             ExpectedReturnTypeLabel::Unit { span } => {
-                diag.span_label(span, rustc_errors::fluent::hir_analysis_expected_default_return_type)
+                diag.span_label(span, "expected `()` because of default return type")
             }
             ExpectedReturnTypeLabel::Other { span, expected } => {
                 diag.set_arg("expected", expected);
-                diag.span_label(span, rustc_errors::fluent::hir_analysis_expected_return_type)
+                diag.span_label(span, "expected `{$expected}` because of return type")
             }
         }
     }
@@ -354,7 +299,7 @@ If a subdiagnostic sets a argument with the same name as a arguments already in
 it will report an error at runtime unless both have exactly the same value.
 It has two benefits:
 - preserves the flexibility that arguments in the main diagnostic are allowed to appear in the attributes of the subdiagnostic.
-For example, There is an attribute `#[suggestion(code = "{new_vis}")]` in the subdiagnostic, but `new_vis` is the field in the main diagnostic struct.
+For example, There is an attribute `#[suggestion("...", code = "{new_vis}")]` in the subdiagnostic, but `new_vis` is the field in the main diagnostic struct.
 - prevents accidental overwriting or deletion of arguments required by the main diagnostic or other subdiagnostics.
 
 These rules guarantee that arguments injected by subdiagnostics are strictly scoped to their own rendering.
@@ -364,32 +309,20 @@ Additionally, subdiagnostics can access arguments from the main diagnostic with
 ### Reference for `#[derive(Subdiagnostic)]`
 `#[derive(Subdiagnostic)]` supports the following attributes:
 
-- `#[label(slug)]`, `#[help(slug)]`, `#[warning(slug)]` or `#[note(slug)]`
+- `#[label("message")]`, `#[help("message")]`, `#[warning("message")]` or `#[note("message")]`
   - _Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes._
   - _Mandatory_
   - Defines the type to be representing a label, help or note.
-  - Slug (_Mandatory_)
-    - Uniquely identifies the diagnostic and corresponds to its Fluent message,
-      mandatory.
-    - A path to an item in `rustc_errors::fluent`, e.g.
-      `rustc_errors::fluent::hir_analysis_field_already_declared`
-      (`rustc_errors::fluent` is implicit in the attribute, so just
-      `hir_analysis_field_already_declared`).
+  - Message (_Mandatory_)
+    - The diagnostic message that will be shown to the user.
     - See [translation documentation](./translation.md).
-- `#[suggestion{,_hidden,_short,_verbose}(slug, code = "...", applicability = "...")]`
+- `#[suggestion{,_hidden,_short,_verbose}("message", code = "...", applicability = "...")]`
   - _Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes._
   - _Mandatory_
   - Defines the type to be representing a suggestion.
-  - Slug (_Mandatory_)
-    - A path to an item in `rustc_errors::fluent`, e.g.
-      `rustc_errors::fluent::hir_analysis_field_already_declared`
-      (`rustc_errors::fluent` is implicit in the attribute, so just
-      `hir_analysis::field_already_declared`). Fluent attributes for all messages
-      exist as top-level items in that module (so `hir_analysis_message.attr` is just
-      `hir_analysis::attr`).
+  - Message (_Mandatory_)
+    - The diagnostic message that will be shown to the user.
     - See [translation documentation](./translation.md).
-    - Defaults to `rustc_errors::fluent::_subdiag::suggestion` (or
-    - `.suggestion` in Fluent).
   - `code = "..."`/`code("...", ...)` (_Mandatory_)
     - One or multiple format strings indicating the code to be suggested as a
       replacement. Multiple values signify multiple possible replacements.
@@ -401,11 +334,11 @@ Additionally, subdiagnostics can access arguments from the main diagnostic with
       - `maybe-incorrect`
       - `has-placeholders`
       - `unspecified`
-- `#[multipart_suggestion{,_hidden,_short,_verbose}(slug, applicability = "...")]`
+- `#[multipart_suggestion{,_hidden,_short,_verbose}("message", applicability = "...")]`
   - _Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes._
   - _Mandatory_
   - Defines the type to be representing a multipart suggestion.
-  - Slug (_Mandatory_): see `#[suggestion]`
+  - Message (_Mandatory_): see `#[suggestion]`
   - `applicability = "..."` (_Optional_): see `#[suggestion]`
 - `#[primary_span]` (_Mandatory_ for labels and suggestions; _optional_ otherwise; not applicable
 to multipart suggestions)

From 37a314b719af622c982c29fad4aeb992f5068035 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 14:12:46 +0100
Subject: [PATCH 1458/1843] Update translation documentation

---
 .../src/diagnostics/translation.md              | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/src/doc/rustc-dev-guide/src/diagnostics/translation.md b/src/doc/rustc-dev-guide/src/diagnostics/translation.md
index 58d75f54a005..fcb26fab667c 100644
--- a/src/doc/rustc-dev-guide/src/diagnostics/translation.md
+++ b/src/doc/rustc-dev-guide/src/diagnostics/translation.md
@@ -13,10 +13,7 @@ active redesign proposals (as of
 Please see the tracking issue 
 for status updates.
 
-We have downgraded the internal lints `untranslatable_diagnostic` and
-`diagnostic_outside_of_impl`. Those internal lints previously required new code
-to use the current translation infrastructure. However, because the translation
-infra is waiting for a yet-to-be-proposed redesign and thus rework, we are not
+The translation infra is waiting for a yet-to-be-proposed redesign and thus rework, we are not
 mandating usage of current translation infra. Use the infra if you *want to* or
 otherwise makes the code cleaner, but otherwise sidestep the translation infra
 if you need more flexibility.
@@ -40,11 +37,6 @@ There are two ways of writing translatable diagnostics:
 When adding or changing a translatable diagnostic,
 you don't need to worry about the translations.
 Only updating the original English message is required.
-Currently,
-each crate which defines translatable diagnostics has its own Fluent resource,
-which is a file named `messages.ftl`,
-located in the root of the crate
-(such as`compiler/rustc_expand/messages.ftl`).
 
 ## Fluent
 
@@ -118,11 +110,8 @@ information that needs to be provided by the code to do so.
 
 ### Compile-time validation and typed identifiers
 
-rustc's `fluent_messages` macro performs compile-time validation of Fluent
-resources and generates code to make it easier to refer to Fluent messages in
-diagnostics.
-
-Compile-time validation of Fluent resources will emit any parsing errors
+rustc's `#[derive(Diagnostic)]` macro performs compile-time validation of Fluent
+messages. Compile-time validation of Fluent resources will emit any parsing errors
 from Fluent resources while building the compiler, preventing invalid Fluent
 resources from causing panics in the compiler. Compile-time validation also
 emits an error if multiple Fluent messages have the same identifier.

From d8ff397cd69c50d735e3ff6e7e4cbfe18c1b01dc Mon Sep 17 00:00:00 2001
From: Adwin White 
Date: Sun, 8 Feb 2026 12:38:02 +0800
Subject: [PATCH 1459/1843] check stalled coroutine obligations eagerly

---
 compiler/rustc_borrowck/src/lib.rs     |  5 +++++
 compiler/rustc_interface/src/passes.rs | 18 +++++++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 361e98454e83..38b73a07a689 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -121,6 +121,11 @@ fn mir_borrowck(
     let (input_body, _) = tcx.mir_promoted(def);
     debug!("run query mir_borrowck: {}", tcx.def_path_str(def));
 
+    // We should eagerly check stalled coroutine obligations from HIR typeck.
+    // Not doing so leads to silent normalization failures later, which will
+    // fail to register opaque types in the next solver.
+    tcx.check_coroutine_obligations(def)?;
+
     let input_body: &Body<'_> = &input_body.borrow();
     if let Some(guar) = input_body.tainted_by_errors {
         debug!("Skipping borrowck because of tainted body");
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 35f8c44b685d..23c59602d296 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -1116,18 +1116,14 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
             {
                 tcx.ensure_ok().mir_drops_elaborated_and_const_checked(def_id);
             }
-            if tcx.is_coroutine(def_id.to_def_id()) {
-                tcx.ensure_ok().mir_coroutine_witnesses(def_id);
-                let _ = tcx.ensure_ok().check_coroutine_obligations(
-                    tcx.typeck_root_def_id(def_id.to_def_id()).expect_local(),
+            if tcx.is_coroutine(def_id.to_def_id())
+                && (!tcx.is_async_drop_in_place_coroutine(def_id.to_def_id()))
+            {
+                // Eagerly check the unsubstituted layout for cycles.
+                tcx.ensure_ok().layout_of(
+                    ty::TypingEnv::post_analysis(tcx, def_id.to_def_id())
+                        .as_query_input(tcx.type_of(def_id).instantiate_identity()),
                 );
-                if !tcx.is_async_drop_in_place_coroutine(def_id.to_def_id()) {
-                    // Eagerly check the unsubstituted layout for cycles.
-                    tcx.ensure_ok().layout_of(
-                        ty::TypingEnv::post_analysis(tcx, def_id.to_def_id())
-                            .as_query_input(tcx.type_of(def_id).instantiate_identity()),
-                    );
-                }
             }
         });
     });

From 8aa0e905f4a1806a9406eb7c34e0c58157dc2856 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 10:23:58 +0100
Subject: [PATCH 1460/1843] Fix suggestion on
 `CallToDeprecatedSafeFnRequiresUnsafe` diagnostic

---
 compiler/rustc_mir_build/src/errors.rs        |  2 +-
 .../unsafe-before_exec-suggestion.fixed       | 18 ++++++++++++++++
 .../unsafe-before_exec-suggestion.rs          | 17 +++++++++++++++
 .../unsafe-before_exec-suggestion.stderr      | 21 +++++++++++++++++++
 4 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/rust-2024/unsafe-before_exec-suggestion.fixed
 create mode 100644 tests/ui/rust-2024/unsafe-before_exec-suggestion.rs
 create mode 100644 tests/ui/rust-2024/unsafe-before_exec-suggestion.stderr

diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 13d8211999e6..2474b3c49a61 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -22,7 +22,7 @@ pub(crate) struct CallToDeprecatedSafeFnRequiresUnsafe {
 
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(
-    "you can wrap the call in an `unsafe` block if you can guarantee that the environment access only happens in single-threaded code",
+    "you can wrap the call in an `unsafe` block if you can guarantee {$guarantee}",
     applicability = "machine-applicable"
 )]
 pub(crate) struct CallToDeprecatedSafeFnRequiresUnsafeSub {
diff --git a/tests/ui/rust-2024/unsafe-before_exec-suggestion.fixed b/tests/ui/rust-2024/unsafe-before_exec-suggestion.fixed
new file mode 100644
index 000000000000..d850428cc1a5
--- /dev/null
+++ b/tests/ui/rust-2024/unsafe-before_exec-suggestion.fixed
@@ -0,0 +1,18 @@
+//@ edition:2015
+//@ only-unix
+//@ run-rustfix
+
+#![deny(deprecated_safe_2024)]
+
+use std::process::Command;
+use std::os::unix::process::CommandExt;
+
+#[allow(deprecated)]
+fn main() {
+    let mut cmd = Command::new("sleep");
+    // TODO: Audit that the closure is async-signal-safe.
+    unsafe { cmd.before_exec(|| Ok(())) };
+    //~^ ERROR call to deprecated safe function
+    //~| WARN this is accepted in the current edition
+    drop(cmd);
+}
diff --git a/tests/ui/rust-2024/unsafe-before_exec-suggestion.rs b/tests/ui/rust-2024/unsafe-before_exec-suggestion.rs
new file mode 100644
index 000000000000..c6894ea631ae
--- /dev/null
+++ b/tests/ui/rust-2024/unsafe-before_exec-suggestion.rs
@@ -0,0 +1,17 @@
+//@ edition:2015
+//@ only-unix
+//@ run-rustfix
+
+#![deny(deprecated_safe_2024)]
+
+use std::process::Command;
+use std::os::unix::process::CommandExt;
+
+#[allow(deprecated)]
+fn main() {
+    let mut cmd = Command::new("sleep");
+    cmd.before_exec(|| Ok(()));
+    //~^ ERROR call to deprecated safe function
+    //~| WARN this is accepted in the current edition
+    drop(cmd);
+}
diff --git a/tests/ui/rust-2024/unsafe-before_exec-suggestion.stderr b/tests/ui/rust-2024/unsafe-before_exec-suggestion.stderr
new file mode 100644
index 000000000000..33893f30ceca
--- /dev/null
+++ b/tests/ui/rust-2024/unsafe-before_exec-suggestion.stderr
@@ -0,0 +1,21 @@
+error: call to deprecated safe function `std::os::unix::process::CommandExt::before_exec` is unsafe and requires unsafe block
+  --> $DIR/unsafe-before_exec-suggestion.rs:13:5
+   |
+LL |     cmd.before_exec(|| Ok(()));
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^ call to unsafe function
+   |
+   = warning: this is accepted in the current edition (Rust 2015) but is a hard error in Rust 2024!
+   = note: for more information, see 
+note: the lint level is defined here
+  --> $DIR/unsafe-before_exec-suggestion.rs:5:9
+   |
+LL | #![deny(deprecated_safe_2024)]
+   |         ^^^^^^^^^^^^^^^^^^^^
+help: you can wrap the call in an `unsafe` block if you can guarantee that the closure is async-signal-safe
+   |
+LL +     // TODO: Audit that the closure is async-signal-safe.
+LL ~     unsafe { cmd.before_exec(|| Ok(())) };
+   |
+
+error: aborting due to 1 previous error
+

From d14c26f45a84eff6eabddd8c0e0580d851ade217 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 10:24:07 +0100
Subject: [PATCH 1461/1843] Fix help on `AmbiguousMissingKwForItemSub`
 diagnostic

---
 compiler/rustc_parse/src/errors.rs                      | 4 +++-
 tests/ui/macros/metavar-expressions/usage-errors.rs     | 2 ++
 tests/ui/macros/metavar-expressions/usage-errors.stderr | 8 ++++----
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 418d56f632bd..b54782ec592c 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -2266,7 +2266,9 @@ pub(crate) enum AmbiguousMissingKwForItemSub {
         span: Span,
         snippet: String,
     },
-    #[help("you likely meant to define an item, e.g., `{$vis} fn foo() {\"{}\"}`")]
+    #[help(
+        "if you meant to call a macro, remove the `pub` and add a trailing `!` after the identifier"
+    )]
     HelpMacro,
 }
 
diff --git a/tests/ui/macros/metavar-expressions/usage-errors.rs b/tests/ui/macros/metavar-expressions/usage-errors.rs
index feff02e2ce47..966ffadb32af 100644
--- a/tests/ui/macros/metavar-expressions/usage-errors.rs
+++ b/tests/ui/macros/metavar-expressions/usage-errors.rs
@@ -27,6 +27,7 @@ curly__rhs_dollar__no_round !(a);
 macro_rules! no_curly__no_rhs_dollar__round {
     ( $( $i:ident ),* ) => { count(i) };
     //~^ ERROR missing `fn` or `struct` for function or struct definition
+    //~| HELP if you meant to call a macro, try
 }
 no_curly__no_rhs_dollar__round !(a, b, c);
 
@@ -34,6 +35,7 @@ no_curly__no_rhs_dollar__round !(a, b, c);
 macro_rules! no_curly__no_rhs_dollar__no_round {
     ( $i:ident ) => { count(i) };
     //~^ ERROR missing `fn` or `struct` for function or struct definition
+    //~| HELP if you meant to call a macro, try
 }
 no_curly__no_rhs_dollar__no_round !(a);
 
diff --git a/tests/ui/macros/metavar-expressions/usage-errors.stderr b/tests/ui/macros/metavar-expressions/usage-errors.stderr
index f66f522e23b9..3d22e3ac4b30 100644
--- a/tests/ui/macros/metavar-expressions/usage-errors.stderr
+++ b/tests/ui/macros/metavar-expressions/usage-errors.stderr
@@ -26,7 +26,7 @@ LL |     ( $( $i:ident ),* ) => { count!(i) };
    |                                   +
 
 error: missing `fn` or `struct` for function or struct definition
-  --> $DIR/usage-errors.rs:35:23
+  --> $DIR/usage-errors.rs:36:23
    |
 LL |     ( $i:ident ) => { count(i) };
    |                       ^^^^^
@@ -41,13 +41,13 @@ LL |     ( $i:ident ) => { count!(i) };
    |                            +
 
 error: variable `i` is still repeating at this depth
-  --> $DIR/usage-errors.rs:42:36
+  --> $DIR/usage-errors.rs:44:36
    |
 LL |     ( $( $i:ident ),* ) => { count($i) };
    |                                    ^^
 
 error[E0425]: cannot find value `a` in this scope
-  --> $DIR/usage-errors.rs:52:49
+  --> $DIR/usage-errors.rs:54:49
    |
 LL |     ( $i:ident ) => { count($i) };
    |                             -- due to this macro variable
@@ -56,7 +56,7 @@ LL | const _: u32 = no_curly__rhs_dollar__no_round! (a);
    |                                                 ^ not found in this scope
 
 error[E0425]: cannot find function `count` in this scope
-  --> $DIR/usage-errors.rs:49:23
+  --> $DIR/usage-errors.rs:51:23
    |
 LL |     ( $i:ident ) => { count($i) };
    |                       ^^^^^ not found in this scope

From f605c0ab7c66506172600ad37a05be3603a20dd2 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 8 Feb 2026 22:14:06 +0530
Subject: [PATCH 1462/1843] self_param and impl_ implementation in
 syntax_factory

---
 .../src/ast/syntax_factory/constructors.rs    | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs
index 5fe419ad4eb7..ad9b9054a8f5 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs
@@ -1590,6 +1590,65 @@ impl SyntaxFactory {
         ast
     }
 
+    pub fn self_param(&self) -> ast::SelfParam {
+        let ast = make::self_param().clone_for_update();
+
+        if let Some(mut mapping) = self.mappings() {
+            let builder = SyntaxMappingBuilder::new(ast.syntax().clone());
+            builder.finish(&mut mapping);
+        }
+
+        ast
+    }
+
+    pub fn impl_(
+        &self,
+        attrs: impl IntoIterator,
+        generic_params: Option,
+        generic_args: Option,
+        path_type: ast::Type,
+        where_clause: Option,
+        body: Option,
+    ) -> ast::Impl {
+        let (attrs, attrs_input) = iterator_input(attrs);
+        let ast = make::impl_(
+            attrs,
+            generic_params.clone(),
+            generic_args.clone(),
+            path_type.clone(),
+            where_clause.clone(),
+            body.clone(),
+        )
+        .clone_for_update();
+
+        if let Some(mut mapping) = self.mappings() {
+            let mut builder = SyntaxMappingBuilder::new(ast.syntax().clone());
+            builder.map_children(attrs_input, ast.attrs().map(|attr| attr.syntax().clone()));
+            if let Some(generic_params) = generic_params {
+                builder.map_node(
+                    generic_params.syntax().clone(),
+                    ast.generic_param_list().unwrap().syntax().clone(),
+                );
+            }
+            builder.map_node(path_type.syntax().clone(), ast.self_ty().unwrap().syntax().clone());
+            if let Some(where_clause) = where_clause {
+                builder.map_node(
+                    where_clause.syntax().clone(),
+                    ast.where_clause().unwrap().syntax().clone(),
+                );
+            }
+            if let Some(body) = body {
+                builder.map_node(
+                    body.syntax().clone(),
+                    ast.assoc_item_list().unwrap().syntax().clone(),
+                );
+            }
+            builder.finish(&mut mapping);
+        }
+
+        ast
+    }
+
     pub fn ret_type(&self, ty: ast::Type) -> ast::RetType {
         let ast = make::ret_type(ty.clone()).clone_for_update();
 

From fa31e1759a65a3689a6a9ed23eff65da9cc8ce76 Mon Sep 17 00:00:00 2001
From: bit-aloo 
Date: Sun, 8 Feb 2026 22:14:30 +0530
Subject: [PATCH 1463/1843] migrate generate_getter_or_setter to syntaxeditor
 api

---
 .../src/handlers/generate_getter_or_setter.rs | 161 +++++++++++-------
 1 file changed, 100 insertions(+), 61 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_getter_or_setter.rs b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_getter_or_setter.rs
index 73e93a3fbf52..2db5e46bb093 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_getter_or_setter.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/handlers/generate_getter_or_setter.rs
@@ -2,13 +2,16 @@ use ide_db::{famous_defs::FamousDefs, source_change::SourceChangeBuilder};
 use stdx::{format_to, to_lower_snake_case};
 use syntax::{
     TextRange,
-    ast::{self, AstNode, HasName, HasVisibility, edit_in_place::Indent, make},
-    ted,
+    ast::{
+        self, AstNode, HasGenericParams, HasName, HasVisibility, edit_in_place::Indent,
+        syntax_factory::SyntaxFactory,
+    },
+    syntax_editor::Position,
 };
 
 use crate::{
     AssistContext, AssistId, Assists, GroupLabel,
-    utils::{convert_reference_type, find_struct_impl, generate_impl},
+    utils::{convert_reference_type, find_struct_impl},
 };
 
 // Assist: generate_setter
@@ -215,12 +218,14 @@ fn generate_getter_from_info(
     ctx: &AssistContext<'_>,
     info: &AssistInfo,
     record_field_info: &RecordFieldInfo,
+    syntax_factory: &SyntaxFactory,
 ) -> ast::Fn {
     let (ty, body) = if matches!(info.assist_type, AssistType::MutGet) {
+        let self_expr = syntax_factory.expr_path(syntax_factory.ident_path("self"));
         (
-            make::ty_ref(record_field_info.field_ty.clone(), true),
-            make::expr_ref(
-                make::expr_field(make::ext::expr_self(), &record_field_info.field_name.text()),
+            syntax_factory.ty_ref(record_field_info.field_ty.clone(), true),
+            syntax_factory.expr_ref(
+                syntax_factory.expr_field(self_expr, &record_field_info.field_name.text()).into(),
                 true,
             ),
         )
@@ -241,9 +246,14 @@ fn generate_getter_from_info(
         })()
         .unwrap_or_else(|| {
             (
-                make::ty_ref(record_field_info.field_ty.clone(), false),
-                make::expr_ref(
-                    make::expr_field(make::ext::expr_self(), &record_field_info.field_name.text()),
+                syntax_factory.ty_ref(record_field_info.field_ty.clone(), false),
+                syntax_factory.expr_ref(
+                    syntax_factory
+                        .expr_field(
+                            syntax_factory.expr_path(syntax_factory.ident_path("self")),
+                            &record_field_info.field_name.text(),
+                        )
+                        .into(),
                     false,
                 ),
             )
@@ -251,18 +261,18 @@ fn generate_getter_from_info(
     };
 
     let self_param = if matches!(info.assist_type, AssistType::MutGet) {
-        make::mut_self_param()
+        syntax_factory.mut_self_param()
     } else {
-        make::self_param()
+        syntax_factory.self_param()
     };
 
     let strukt = &info.strukt;
-    let fn_name = make::name(&record_field_info.fn_name);
-    let params = make::param_list(Some(self_param), []);
-    let ret_type = Some(make::ret_type(ty));
-    let body = make::block_expr([], Some(body));
+    let fn_name = syntax_factory.name(&record_field_info.fn_name);
+    let params = syntax_factory.param_list(Some(self_param), []);
+    let ret_type = Some(syntax_factory.ret_type(ty));
+    let body = syntax_factory.block_expr([], Some(body));
 
-    make::fn_(
+    syntax_factory.fn_(
         None,
         strukt.visibility(),
         fn_name,
@@ -278,28 +288,35 @@ fn generate_getter_from_info(
     )
 }
 
-fn generate_setter_from_info(info: &AssistInfo, record_field_info: &RecordFieldInfo) -> ast::Fn {
+fn generate_setter_from_info(
+    info: &AssistInfo,
+    record_field_info: &RecordFieldInfo,
+    syntax_factory: &SyntaxFactory,
+) -> ast::Fn {
     let strukt = &info.strukt;
     let field_name = &record_field_info.fn_name;
-    let fn_name = make::name(&format!("set_{field_name}"));
+    let fn_name = syntax_factory.name(&format!("set_{field_name}"));
     let field_ty = &record_field_info.field_ty;
 
     // Make the param list
     // `(&mut self, $field_name: $field_ty)`
-    let field_param =
-        make::param(make::ident_pat(false, false, make::name(field_name)).into(), field_ty.clone());
-    let params = make::param_list(Some(make::mut_self_param()), [field_param]);
+    let field_param = syntax_factory.param(
+        syntax_factory.ident_pat(false, false, syntax_factory.name(field_name)).into(),
+        field_ty.clone(),
+    );
+    let params = syntax_factory.param_list(Some(syntax_factory.mut_self_param()), [field_param]);
 
     // Make the assignment body
     // `self.$field_name = $field_name`
-    let self_expr = make::ext::expr_self();
-    let lhs = make::expr_field(self_expr, field_name);
-    let rhs = make::expr_path(make::ext::ident_path(field_name));
-    let assign_stmt = make::expr_stmt(make::expr_assignment(lhs, rhs).into());
-    let body = make::block_expr([assign_stmt.into()], None);
+    let self_expr = syntax_factory.expr_path(syntax_factory.ident_path("self"));
+    let lhs = syntax_factory.expr_field(self_expr, field_name);
+    let rhs = syntax_factory.expr_path(syntax_factory.ident_path(field_name));
+    let assign_stmt =
+        syntax_factory.expr_stmt(syntax_factory.expr_assignment(lhs.into(), rhs).into());
+    let body = syntax_factory.block_expr([assign_stmt.into()], None);
 
     // Make the setter fn
-    make::fn_(
+    syntax_factory.fn_(
         None,
         strukt.visibility(),
         fn_name,
@@ -403,47 +420,69 @@ fn build_source_change(
     info_of_record_fields: Vec,
     assist_info: AssistInfo,
 ) {
-    let record_fields_count = info_of_record_fields.len();
+    let syntax_factory = SyntaxFactory::without_mappings();
 
-    let impl_def = if let Some(impl_def) = &assist_info.impl_def {
+    let items: Vec = info_of_record_fields
+        .iter()
+        .map(|record_field_info| {
+            let method = match assist_info.assist_type {
+                AssistType::Set => {
+                    generate_setter_from_info(&assist_info, record_field_info, &syntax_factory)
+                }
+                _ => {
+                    generate_getter_from_info(ctx, &assist_info, record_field_info, &syntax_factory)
+                }
+            };
+            let new_fn = method.clone_for_update();
+            new_fn.indent(1.into());
+            new_fn.into()
+        })
+        .collect();
+
+    if let Some(impl_def) = &assist_info.impl_def {
         // We have an existing impl to add to
-        builder.make_mut(impl_def.clone())
-    } else {
-        // Generate a new impl to add the methods to
-        let impl_def = generate_impl(&ast::Adt::Struct(assist_info.strukt.clone()));
+        let mut editor = builder.make_editor(impl_def.syntax());
+        impl_def.assoc_item_list().unwrap().add_items(&mut editor, items.clone());
 
-        // Insert it after the adt
-        let strukt = builder.make_mut(assist_info.strukt.clone());
-
-        ted::insert_all_raw(
-            ted::Position::after(strukt.syntax()),
-            vec![make::tokens::blank_line().into(), impl_def.syntax().clone().into()],
-        );
-
-        impl_def
-    };
-
-    let assoc_item_list = impl_def.get_or_create_assoc_item_list();
-
-    for (i, record_field_info) in info_of_record_fields.iter().enumerate() {
-        // Make the new getter or setter fn
-        let new_fn = match assist_info.assist_type {
-            AssistType::Set => generate_setter_from_info(&assist_info, record_field_info),
-            _ => generate_getter_from_info(ctx, &assist_info, record_field_info),
-        }
-        .clone_for_update();
-        new_fn.indent(1.into());
-
-        // Insert a tabstop only for last method we generate
-        if i == record_fields_count - 1
-            && let Some(cap) = ctx.config.snippet_cap
-            && let Some(name) = new_fn.name()
+        if let Some(cap) = ctx.config.snippet_cap
+            && let Some(ast::AssocItem::Fn(fn_)) = items.last()
+            && let Some(name) = fn_.name()
         {
-            builder.add_tabstop_before(cap, name);
+            let tabstop = builder.make_tabstop_before(cap);
+            editor.add_annotation(name.syntax(), tabstop);
         }
 
-        assoc_item_list.add_item(new_fn.clone().into());
+        builder.add_file_edits(ctx.vfs_file_id(), editor);
+        return;
     }
+    let ty_params = assist_info.strukt.generic_param_list();
+    let ty_args = ty_params.as_ref().map(|it| it.to_generic_args());
+    let impl_def = syntax_factory.impl_(
+        None,
+        ty_params,
+        ty_args,
+        syntax_factory
+            .ty_path(syntax_factory.ident_path(&assist_info.strukt.name().unwrap().to_string()))
+            .into(),
+        None,
+        Some(syntax_factory.assoc_item_list(items)),
+    );
+    let mut editor = builder.make_editor(assist_info.strukt.syntax());
+    editor.insert_all(
+        Position::after(assist_info.strukt.syntax()),
+        vec![syntax_factory.whitespace("\n\n").into(), impl_def.syntax().clone().into()],
+    );
+
+    if let Some(cap) = ctx.config.snippet_cap
+        && let Some(assoc_list) = impl_def.assoc_item_list()
+        && let Some(ast::AssocItem::Fn(fn_)) = assoc_list.assoc_items().last()
+        && let Some(name) = fn_.name()
+    {
+        let tabstop = builder.make_tabstop_before(cap);
+        editor.add_annotation(name.syntax().clone(), tabstop);
+    }
+
+    builder.add_file_edits(ctx.vfs_file_id(), editor);
 }
 
 #[cfg(test)]

From 4d07a8f24c42c4cb7362ea8c4db8cfca62a846b2 Mon Sep 17 00:00:00 2001
From: Nicholas Bishop 
Date: Sun, 8 Feb 2026 12:29:59 -0500
Subject: [PATCH 1464/1843] std: Don't panic when removing a nonexistent UEFI
 var

`std::env::remove_var` does not say that deleting a nonexistent variable
is an error (and at least on Linux, it indeed does not cause an
error).

The UEFI Shell Protocol spec also doesn't say it's an error, but the
edk2 implementation delegates to the UEFI runtime `SetVariable`
function, which returns `EFI_NOT_FOUND` when trying to delete a
nonexistent variable.

Change the UEFI implementation to check for a `NotFound` error and treat
it as success.
---
 library/std/src/sys/env/uefi.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/library/std/src/sys/env/uefi.rs b/library/std/src/sys/env/uefi.rs
index af16a02642a4..bc2aed423179 100644
--- a/library/std/src/sys/env/uefi.rs
+++ b/library/std/src/sys/env/uefi.rs
@@ -43,7 +43,20 @@ mod uefi_env {
     pub(crate) fn unset(key: &OsStr) -> io::Result<()> {
         let mut key_ptr = helpers::os_string_to_raw(key)
             .ok_or(io::const_error!(io::ErrorKind::InvalidInput, "invalid key"))?;
-        unsafe { set_raw(key_ptr.as_mut_ptr(), crate::ptr::null_mut()) }
+        let r = unsafe { set_raw(key_ptr.as_mut_ptr(), crate::ptr::null_mut()) };
+
+        // The UEFI Shell spec only lists `EFI_SUCCESS` as a possible return value for
+        // `SetEnv`, but the edk2 implementation can return errors. Allow most of these
+        // errors to bubble up to the caller, but ignore `NotFound` errors; deleting a
+        // nonexistent variable is not listed as an error condition of
+        // `std::env::remove_var`.
+        if let Err(err) = &r
+            && err.kind() == io::ErrorKind::NotFound
+        {
+            Ok(())
+        } else {
+            r
+        }
     }
 
     pub(crate) fn get_all() -> io::Result> {

From 9a30ec81498592914950d7a63253965e61d8cc05 Mon Sep 17 00:00:00 2001
From: Noah Lev 
Date: Wed, 7 Jan 2026 15:22:47 -0800
Subject: [PATCH 1465/1843] Implement MVP for opaque generic const arguments

This is meant to be the interim successor to generic const expressions.
Essentially, const item RHS's will be allowed to do arbitrary const
operations using generics. The limitation is that these const items will
be treated opaquely, like ADTs in nominal typing, such that uses of them
will only be equal if the same const item is referenced. In other words,
two const items with the exact same RHS will not be considered equal.

I also added some logic to check feature gates that depend on others
being enabled (like oGCA depending on mGCA).

= Coherence =

During coherence, OGCA consts should be normalized ambiguously because
they are opaque but eventually resolved to a real value. We don't want
two OGCAs that have the same value to be treated as distinct for
coherence purposes. (Just like opaque types.)

This actually doesn't work yet because there are pre-existing
fundamental issues with equate relations involving consts that need to
be normalized. The problem is that we normalize only one layer of the
const item and don't actually process the resulting anon const. Normally
the created inference variable should be handled, which in this case
would cause us to hit the anon const, but that's not happening.
Specifically, `visit_const` on `Generalizer` should be updated to be
similar to `visit_ty`.
---
 compiler/rustc_ast_passes/src/errors.rs       | 10 +++
 compiler/rustc_ast_passes/src/feature_gate.rs | 22 +++++++
 compiler/rustc_ast_passes/src/lib.rs          |  1 +
 compiler/rustc_feature/src/lib.rs             |  3 +-
 compiler/rustc_feature/src/unstable.rs        |  8 +++
 compiler/rustc_hir_analysis/src/collect.rs    | 64 ++++++++++++++++++-
 .../src/collect/generics_of.rs                |  2 +
 .../src/hir_ty_lowering/mod.rs                |  5 ++
 compiler/rustc_middle/src/ty/consts.rs        | 15 +----
 compiler/rustc_middle/src/ty/context.rs       |  3 +
 compiler/rustc_middle/src/ty/mod.rs           |  4 +-
 .../src/solve/normalizes_to/anon_const.rs     | 15 ++++-
 compiler/rustc_parse/src/parser/item.rs       |  4 +-
 compiler/rustc_span/src/symbol.rs             |  1 +
 .../rustc_trait_selection/src/traits/mod.rs   | 14 ++++
 compiler/rustc_type_ir/src/const_kind.rs      | 20 ++++++
 compiler/rustc_type_ir/src/interner.rs        |  1 +
 .../non_valtreeable_const_arg-2.rs            |  2 +-
 .../non_valtreeable_const_arg-2.stderr        |  6 +-
 .../associated-const-bindings/ambiguity.rs    |  2 +-
 .../coexisting-with-type-binding.rs           |  2 +-
 ...dyn-compat-assoc-const-ty-mentions-self.rs |  1 +
 ...compat-assoc-const-ty-mentions-self.stderr |  4 +-
 ...elf-const-projections-in-assoc-const-ty.rs | 13 +++-
 ...const-projections-in-assoc-const-ty.stderr |  8 +--
 .../esc-bound-var-in-ty.rs                    |  1 +
 .../esc-bound-var-in-ty.stderr                |  2 +-
 .../mgca/adt_expr_arg_simple.stderr           |  2 +
 .../mgca/early-bound-param-lt-bad.stderr      |  2 +
 .../mgca/explicit_anon_consts.stderr          | 14 ++++
 .../mgca/selftyalias-containing-param.stderr  |  1 +
 .../ui/const-generics/mgca/selftyparam.stderr |  2 +
 .../size-of-generic-ptr-in-array-len.stderr   |  2 +
 .../mgca/tuple_ctor_complex_args.stderr       |  2 +
 .../mgca/tuple_expr_arg_complex.stderr        |  2 +
 .../mgca/type_const-on-generic-expr.stderr    |  4 ++
 .../mgca/type_const-on-generic_expr-2.stderr  |  6 ++
 .../unbraced_const_block_const_arg_gated.rs   |  1 -
 ...nbraced_const_block_const_arg_gated.stderr | 12 +---
 tests/ui/const-generics/ogca/basic-fail.rs    | 22 +++++++
 .../ui/const-generics/ogca/basic-fail.stderr  |  9 +++
 tests/ui/const-generics/ogca/basic.rs         | 22 +++++++
 .../ogca/coherence-ambiguous.rs               | 21 ++++++
 .../const-generics/ogca/rhs-but-not-root.rs   | 14 ++++
 .../ogca/rhs-but-not-root.stderr              |  8 +++
 .../feature-gate-opaque-generic-const-args.rs |  9 +++
 ...ture-gate-opaque-generic-const-args.stderr | 10 +++
 .../assoc-const-no-infer-ice-115806.rs        |  2 +-
 48 files changed, 347 insertions(+), 53 deletions(-)
 create mode 100644 tests/ui/const-generics/ogca/basic-fail.rs
 create mode 100644 tests/ui/const-generics/ogca/basic-fail.stderr
 create mode 100644 tests/ui/const-generics/ogca/basic.rs
 create mode 100644 tests/ui/const-generics/ogca/coherence-ambiguous.rs
 create mode 100644 tests/ui/const-generics/ogca/rhs-but-not-root.rs
 create mode 100644 tests/ui/const-generics/ogca/rhs-but-not-root.stderr
 create mode 100644 tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr

diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index baf6f6beaeed..7282a5168213 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -950,6 +950,16 @@ pub(crate) struct IncompatibleFeatures {
     pub f2: Symbol,
 }
 
+#[derive(Diagnostic)]
+#[diag("`{$parent}` requires {$missing} to be enabled")]
+#[help("enable all of these features")]
+pub(crate) struct MissingDependentFeatures {
+    #[primary_span]
+    pub parent_span: Span,
+    pub parent: Symbol,
+    pub missing: String,
+}
+
 #[derive(Diagnostic)]
 #[diag("negative bounds are not supported")]
 pub(crate) struct NegativeBoundUnsupported {
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index 1b1bbb1564c4..ce9001d65931 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -441,6 +441,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
     maybe_stage_features(sess, features, krate);
     check_incompatible_features(sess, features);
+    check_dependent_features(sess, features);
     check_new_solver_banned_features(sess, features);
 
     let mut visitor = PostExpansionVisitor { sess, features };
@@ -649,6 +650,27 @@ fn check_incompatible_features(sess: &Session, features: &Features) {
     }
 }
 
+fn check_dependent_features(sess: &Session, features: &Features) {
+    for &(parent, children) in
+        rustc_feature::DEPENDENT_FEATURES.iter().filter(|(parent, _)| features.enabled(*parent))
+    {
+        if children.iter().any(|f| !features.enabled(*f)) {
+            let parent_span = features
+                .enabled_features_iter_stable_order()
+                .find_map(|(name, span)| (name == parent).then_some(span))
+                .unwrap();
+            // FIXME: should probably format this in fluent instead of here
+            let missing = children
+                .iter()
+                .filter(|f| !features.enabled(**f))
+                .map(|s| format!("`{}`", s.as_str()))
+                .intersperse(String::from(", "))
+                .collect();
+            sess.dcx().emit_err(errors::MissingDependentFeatures { parent_span, parent, missing });
+        }
+    }
+}
+
 fn check_new_solver_banned_features(sess: &Session, features: &Features) {
     if !sess.opts.unstable_opts.next_solver.globally {
         return;
diff --git a/compiler/rustc_ast_passes/src/lib.rs b/compiler/rustc_ast_passes/src/lib.rs
index 2630348c49c1..7793f786cefe 100644
--- a/compiler/rustc_ast_passes/src/lib.rs
+++ b/compiler/rustc_ast_passes/src/lib.rs
@@ -5,6 +5,7 @@
 // tidy-alphabetical-start
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
+#![feature(iter_intersperse)]
 #![feature(iter_is_partitioned)]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index 4ba50d03d707..619726f0d5d8 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -136,5 +136,6 @@ pub use builtin_attrs::{
 };
 pub use removed::REMOVED_LANG_FEATURES;
 pub use unstable::{
-    EnabledLangFeature, EnabledLibFeature, Features, INCOMPATIBLE_FEATURES, UNSTABLE_LANG_FEATURES,
+    DEPENDENT_FEATURES, EnabledLangFeature, EnabledLibFeature, Features, INCOMPATIBLE_FEATURES,
+    UNSTABLE_LANG_FEATURES,
 };
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 2ab7714f22ca..a941eb1f459e 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -605,6 +605,8 @@ declare_features! (
     (unstable, offset_of_enum, "1.75.0", Some(120141)),
     /// Allows using fields with slice type in offset_of!
     (unstable, offset_of_slice, "1.81.0", Some(126151)),
+    /// Allows using generics in more complex const expressions, based on definitional equality.
+    (unstable, opaque_generic_const_args, "CURRENT_RUSTC_VERSION", Some(151972)),
     /// Allows using `#[optimize(X)]`.
     (unstable, optimize_attribute, "1.34.0", Some(54882)),
     /// Allows specifying nop padding on functions for dynamic patching.
@@ -782,3 +784,9 @@ pub const INCOMPATIBLE_FEATURES: &[(Symbol, Symbol)] = &[
     // boolean logic required to tell which typing rules to use.
     (sym::ref_pat_eat_one_layer_2024, sym::ref_pat_eat_one_layer_2024_structural),
 ];
+
+/// Some features require one or more other features to be enabled.
+pub const DEPENDENT_FEATURES: &[(Symbol, &[Symbol])] = &[
+    (sym::opaque_generic_const_args, &[sym::min_generic_const_args]),
+    (sym::unsized_const_params, &[sym::adt_const_params]),
+];
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index f50aff187f25..f3086266eec0 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -16,7 +16,7 @@
 
 use std::cell::Cell;
 use std::iter;
-use std::ops::Bound;
+use std::ops::{Bound, ControlFlow};
 
 use rustc_abi::{ExternAbi, Size};
 use rustc_ast::Recovered;
@@ -26,12 +26,13 @@ use rustc_errors::{
     Applicability, Diag, DiagCtxtHandle, E0228, ErrorGuaranteed, StashKey, struct_span_code_err,
 };
 use rustc_hir::attrs::AttributeKind;
-use rustc_hir::def::DefKind;
+use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::intravisit::{InferKind, Visitor, VisitorExt};
+use rustc_hir::intravisit::{self, InferKind, Visitor, VisitorExt};
 use rustc_hir::{self as hir, GenericParamKind, HirId, Node, PreciseCapturingArgKind, find_attr};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::{DynCompatibilityViolation, ObligationCause};
+use rustc_middle::hir::nested_filter;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::util::{Discr, IntTypeExt};
 use rustc_middle::ty::{
@@ -1511,6 +1512,20 @@ fn anon_const_kind<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> ty::AnonConstKin
             let parent_hir_node = tcx.hir_node(tcx.parent_hir_id(const_arg_id));
             if tcx.features().generic_const_exprs() {
                 ty::AnonConstKind::GCE
+            } else if tcx.features().opaque_generic_const_args() {
+                // Only anon consts that are the RHS of a const item can be OGCA.
+                // Note: We can't just check tcx.parent because it needs to be EXACTLY
+                // the RHS, not just part of the RHS.
+                if !is_anon_const_rhs_of_const_item(tcx, def) {
+                    return ty::AnonConstKind::MCG;
+                }
+
+                let body = tcx.hir_body_owned_by(def);
+                let mut visitor = OGCAParamVisitor(tcx);
+                match visitor.visit_body(body) {
+                    ControlFlow::Break(UsesParam) => ty::AnonConstKind::OGCA,
+                    ControlFlow::Continue(()) => ty::AnonConstKind::MCG,
+                }
             } else if tcx.features().min_generic_const_args() {
                 ty::AnonConstKind::MCG
             } else if let hir::Node::Expr(hir::Expr {
@@ -1528,6 +1543,49 @@ fn anon_const_kind<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> ty::AnonConstKin
     }
 }
 
+fn is_anon_const_rhs_of_const_item<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> bool {
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
+    let Some((_, grandparent_node)) = tcx.hir_parent_iter(hir_id).nth(1) else { return false };
+    let (Node::Item(hir::Item { kind: hir::ItemKind::Const(_, _, _, ct_rhs), .. })
+    | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(_, ct_rhs), .. })
+    | Node::TraitItem(hir::TraitItem {
+        kind: hir::TraitItemKind::Const(_, Some(ct_rhs)), ..
+    })) = grandparent_node
+    else {
+        return false;
+    };
+    let hir::ConstItemRhs::TypeConst(hir::ConstArg {
+        kind: hir::ConstArgKind::Anon(rhs_anon), ..
+    }) = ct_rhs
+    else {
+        return false;
+    };
+    def_id == rhs_anon.def_id
+}
+
+struct OGCAParamVisitor<'tcx>(TyCtxt<'tcx>);
+
+struct UsesParam;
+
+impl<'tcx> Visitor<'tcx> for OGCAParamVisitor<'tcx> {
+    type NestedFilter = nested_filter::OnlyBodies;
+    type Result = ControlFlow;
+
+    fn maybe_tcx(&mut self) -> TyCtxt<'tcx> {
+        self.0
+    }
+
+    fn visit_path(&mut self, path: &hir::Path<'tcx>, _id: HirId) -> ControlFlow {
+        if let Res::Def(DefKind::TyParam | DefKind::ConstParam | DefKind::LifetimeParam, _) =
+            path.res
+        {
+            return ControlFlow::Break(UsesParam);
+        }
+
+        intravisit::walk_path(self, path)
+    }
+}
+
 #[instrument(level = "debug", skip(tcx), ret)]
 fn const_of_item<'tcx>(
     tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index 511d513216eb..0723418d8ddf 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -92,6 +92,8 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
             match tcx.anon_const_kind(def_id) {
                 // Stable: anon consts are not able to use any generic parameters...
                 ty::AnonConstKind::MCG => None,
+                // OGCA anon consts inherit their parent's generics.
+                ty::AnonConstKind::OGCA => Some(parent_did),
                 // we provide generics to repeat expr counts as a backwards compatibility hack. #76200
                 ty::AnonConstKind::RepeatExprCount => Some(parent_did),
 
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index a79170452fae..7a03d8b220c5 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -404,6 +404,11 @@ impl<'tcx> ForbidMCGParamUsesFolder<'tcx> {
                 diag.span_note(impl_.self_ty.span, "not a concrete type");
             }
         }
+        if self.tcx.features().min_generic_const_args()
+            && !self.tcx.features().opaque_generic_const_args()
+        {
+            diag.help("add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items");
+        }
         diag.emit()
     }
 }
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index 5581ad5669aa..3e17943a8857 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -2,7 +2,7 @@ use std::borrow::Cow;
 
 use rustc_data_structures::intern::Interned;
 use rustc_error_messages::MultiSpan;
-use rustc_macros::{HashStable, TyDecodable, TyEncodable};
+use rustc_macros::HashStable;
 use rustc_type_ir::walk::TypeWalker;
 use rustc_type_ir::{self as ir, TypeFlags, WithCachedTypeInfo};
 
@@ -335,16 +335,3 @@ impl<'tcx> Const<'tcx> {
         TypeWalker::new(self.into())
     }
 }
-
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable, HashStable)]
-pub enum AnonConstKind {
-    /// `feature(generic_const_exprs)` anon consts are allowed to use arbitrary generic parameters in scope
-    GCE,
-    /// stable `min_const_generics` anon consts are not allowed to use any generic parameters
-    MCG,
-    /// anon consts used as the length of a repeat expr are syntactically allowed to use generic parameters
-    /// but must not depend on the actual instantiation. See #76200 for more information
-    RepeatExprCount,
-    /// anon consts outside of the type system, e.g. enum discriminants
-    NonTypeSystem,
-}
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index e43a30091f50..1bb83530487d 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -238,6 +238,9 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     fn const_of_item(self, def_id: DefId) -> ty::EarlyBinder<'tcx, Const<'tcx>> {
         self.const_of_item(def_id)
     }
+    fn anon_const_kind(self, def_id: DefId) -> ty::AnonConstKind {
+        self.anon_const_kind(def_id)
+    }
 
     type AdtDef = ty::AdtDef<'tcx>;
     fn adt_def(self, adt_def_id: DefId) -> Self::AdtDef {
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 0220531b09fa..37871aad4a77 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -76,8 +76,8 @@ pub use self::closure::{
     place_to_string_for_capture,
 };
 pub use self::consts::{
-    AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, ConstToValTreeResult, Expr,
-    ExprKind, ScalarInt, SimdAlign, UnevaluatedConst, ValTree, ValTreeKindExt, Value,
+    AtomicOrdering, Const, ConstInt, ConstKind, ConstToValTreeResult, Expr, ExprKind, ScalarInt,
+    SimdAlign, UnevaluatedConst, ValTree, ValTreeKindExt, Value,
 };
 pub use self::context::{
     CtxtInterners, CurrentGcx, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed, tls,
diff --git a/compiler/rustc_next_trait_solver/src/solve/normalizes_to/anon_const.rs b/compiler/rustc_next_trait_solver/src/solve/normalizes_to/anon_const.rs
index 46312be5ea9a..eb6a1b51421c 100644
--- a/compiler/rustc_next_trait_solver/src/solve/normalizes_to/anon_const.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/normalizes_to/anon_const.rs
@@ -1,4 +1,4 @@
-use rustc_type_ir::{self as ty, Interner};
+use rustc_type_ir::{self as ty, Interner, TypingMode};
 use tracing::instrument;
 
 use crate::delegate::SolverDelegate;
@@ -14,7 +14,18 @@ where
         &mut self,
         goal: Goal>,
     ) -> QueryResult {
-        if let Some(normalized_const) = self.evaluate_const(
+        if self.typing_mode() == TypingMode::Coherence
+            && self.cx().anon_const_kind(goal.predicate.alias.def_id) == ty::AnonConstKind::OGCA
+        {
+            // During coherence, OGCA consts should be normalized ambiguously
+            // because they are opaque but eventually resolved to a real value.
+            // We don't want two OGCAs that have the same value to be treated
+            // as distinct for coherence purposes. (Just like opaque types.)
+            //
+            // We can't rely on evaluate_const below because that particular wrapper
+            // treats too-generic consts as a successful evaluation.
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+        } else if let Some(normalized_const) = self.evaluate_const(
             goal.param_env,
             ty::UnevaluatedConst::new(
                 goal.predicate.alias.def_id.try_into().unwrap(),
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index c35ddc4dc943..9ac9bd5c0e46 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -1551,7 +1551,9 @@ impl<'a> Parser<'a> {
 
         let rhs = if self.eat(exp!(Eq)) {
             if attr::contains_name(attrs, sym::type_const) {
-                Some(ConstItemRhs::TypeConst(self.parse_const_arg()?))
+                let ct =
+                    self.parse_expr_anon_const(|this, expr| this.mgca_direct_lit_hack(expr))?;
+                Some(ConstItemRhs::TypeConst(ct))
             } else {
                 Some(ConstItemRhs::Body(self.parse_expr()?))
             }
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 1915ff0380fd..6aa2eae556e2 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1628,6 +1628,7 @@ symbols! {
         on_const,
         on_unimplemented,
         opaque,
+        opaque_generic_const_args,
         opaque_module_name_placeholder: "",
         open_options_new,
         ops,
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index 4e027a301cc8..08f1d947dfb5 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -681,6 +681,20 @@ pub fn try_evaluate_const<'tcx>(
 
                     (args, typing_env)
                 }
+                Some(ty::AnonConstKind::OGCA) => {
+                    if infcx.typing_mode() != TypingMode::PostAnalysis {
+                        // OGCA anon consts should be treated as always having generics
+                        // during anything before codegen (or maybe MIR opts too).
+                        return Err(EvaluateConstErr::HasGenericsOrInfers);
+                    }
+
+                    if uv.args.has_non_region_param() || uv.args.has_non_region_infer() {
+                        return Err(EvaluateConstErr::HasGenericsOrInfers);
+                    }
+
+                    let typing_env = ty::TypingEnv::fully_monomorphized();
+                    (uv.args, typing_env)
+                }
                 Some(ty::AnonConstKind::MCG) | Some(ty::AnonConstKind::NonTypeSystem) | None => {
                     // We are only dealing with "truly" generic/uninferred constants here:
                     // - GCEConsts have been handled separately
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index b215230ea443..1c39f31469b1 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -200,3 +200,23 @@ impl ValTreeKind {
         }
     }
 }
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[cfg_attr(
+    feature = "nightly",
+    derive(Encodable_NoContext, Decodable_NoContext, HashStable_NoContext)
+)]
+pub enum AnonConstKind {
+    /// `feature(generic_const_exprs)` anon consts are allowed to use arbitrary generic parameters in scope
+    GCE,
+    /// stable `min_const_generics` anon consts are not allowed to use any generic parameters
+    MCG,
+    /// `feature(opaque_generic_const_args)` anon consts are allowed to use arbitrary
+    /// generic parameters in scope, but only if they syntactically reference them.
+    OGCA,
+    /// anon consts used as the length of a repeat expr are syntactically allowed to use generic parameters
+    /// but must not depend on the actual instantiation. See #76200 for more information
+    RepeatExprCount,
+    /// anon consts outside of the type system, e.g. enum discriminants
+    NonTypeSystem,
+}
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index 59ae6733fb84..8f446cdfba6d 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -205,6 +205,7 @@ pub trait Interner:
     fn type_of_opaque_hir_typeck(self, def_id: Self::LocalDefId)
     -> ty::EarlyBinder;
     fn const_of_item(self, def_id: Self::DefId) -> ty::EarlyBinder;
+    fn anon_const_kind(self, def_id: Self::DefId) -> ty::AnonConstKind;
 
     type AdtDef: AdtDef;
     fn adt_def(self, adt_def_id: Self::AdtId) -> Self::AdtDef;
diff --git a/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.rs b/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.rs
index 6b87ad86d4bc..e364368b8a4c 100644
--- a/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.rs
+++ b/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.rs
@@ -1,4 +1,4 @@
-#![feature(generic_const_exprs, unsized_const_params)]
+#![feature(adt_const_params, generic_const_exprs, unsized_const_params)]
 #![allow(incomplete_features)]
 
 // Regression test for 128232
diff --git a/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.stderr b/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.stderr
index 72dfda50ea5c..b13f76eabadb 100644
--- a/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.stderr
+++ b/tests/ui/const-generics/adt_const_params/non_valtreeable_const_arg-2.stderr
@@ -9,13 +9,11 @@ help: you might be missing a const parameter
 LL | impl Wrapper<{ bar() }> {
    |     +++++++++++++++++++++++
 
-error: using function pointers as const generic parameters is forbidden
+error[E0741]: using function pointers as const generic parameters is forbidden
   --> $DIR/non_valtreeable_const_arg-2.rs:8:25
    |
 LL | struct Wrapper;
    |                         ^^^^
-   |
-   = note: the only supported types are integers, `bool`, and `char`
 
 error[E0599]: the function or associated item `call` exists for struct `Wrapper`, but its trait bounds were not satisfied
   --> $DIR/non_valtreeable_const_arg-2.rs:17:26
@@ -37,5 +35,5 @@ note: the trait `Fn` must be implemented
 
 error: aborting due to 3 previous errors
 
-Some errors have detailed explanations: E0425, E0599.
+Some errors have detailed explanations: E0425, E0599, E0741.
 For more information about an error, try `rustc --explain E0425`.
diff --git a/tests/ui/const-generics/associated-const-bindings/ambiguity.rs b/tests/ui/const-generics/associated-const-bindings/ambiguity.rs
index 6bc2a6d5d153..785d43e51b3d 100644
--- a/tests/ui/const-generics/associated-const-bindings/ambiguity.rs
+++ b/tests/ui/const-generics/associated-const-bindings/ambiguity.rs
@@ -1,7 +1,7 @@
 // We used to say "ambiguous associated type" on ambiguous associated consts.
 // Ensure that we now use the correct label.
 
-#![feature(min_generic_const_args, unsized_const_params)]
+#![feature(adt_const_params, min_generic_const_args, unsized_const_params)]
 #![allow(incomplete_features)]
 
 trait Trait0: Parent0 + Parent0 {}
diff --git a/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs b/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs
index 004215986711..0af91bfe7da8 100644
--- a/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs
+++ b/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs
@@ -5,7 +5,7 @@
 
 //@ check-pass
 
-#![feature(min_generic_const_args, unsized_const_params)]
+#![feature(adt_const_params, min_generic_const_args, unsized_const_params)]
 #![allow(incomplete_features)]
 
 trait Trait: SuperTrait {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs
index d19e7acbaff0..bd26acce681f 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs
@@ -3,6 +3,7 @@
 
 //@ dont-require-annotations: NOTE
 
+#![feature(adt_const_params)]
 #![feature(generic_const_items)]
 #![feature(generic_const_parameter_types)]
 #![feature(min_generic_const_args)]
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr
index dba1643d2c5c..dedbdd7f82bb 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr
@@ -1,12 +1,12 @@
 error[E0038]: the trait `Trait` is not dyn compatible
-  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:37:16
+  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:38:16
    |
 LL |     let _: dyn Trait;
    |                ^^^^^ `Trait` is not dyn compatible
    |
 note: for a trait to be dyn compatible it needs to allow building a vtable
       for more information, visit 
-  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:17:11
+  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:18:11
    |
 LL | trait Trait {
    |       ----- this trait is not dyn compatible...
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs
index 936556e957ca..fc3bd1d2e753 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs
@@ -4,17 +4,24 @@
 //              to the rest of the compiler and by extension the user via diagnostics.
 //@ known-bug: unknown
 
-#![feature(min_generic_const_args, unsized_const_params, generic_const_parameter_types)]
+#![feature(
+    adt_const_params,
+    min_generic_const_args,
+    unsized_const_params,
+    generic_const_parameter_types
+)]
 #![expect(incomplete_features)]
 
 trait A {
     type Ty: std::marker::ConstParamTy_;
-    #[type_const] const CT: Self::Ty;
+    #[type_const]
+    const CT: Self::Ty;
 }
 
 impl A for () {
     type Ty = i32;
-    #[type_const] const CT: i32 = 0;
+    #[type_const]
+    const CT: i32 = 0;
 }
 
 fn main() {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr
index 8ee231ec070f..0b8dae1aac43 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr
@@ -1,23 +1,23 @@
 error[E0277]: the trait bound `FreshTy(0): A` is not satisfied
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:27:33
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:34:33
    |
 LL |     let _: dyn A;
    |                                 ^ the trait `A` is not implemented for `FreshTy(0)`
    |
 help: the trait `A` is implemented for `()`
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:15:1
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:21:1
    |
 LL | impl A for () {
    | ^^^^^^^^^^^^^
 
 error[E0277]: the trait bound `FreshTy(0): A` is not satisfied
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:29:34
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:36:34
    |
 LL |     let _: &dyn A = &();
    |                                  ^ the trait `A` is not implemented for `FreshTy(0)`
    |
 help: the trait `A` is implemented for `()`
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:15:1
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:21:1
    |
 LL | impl A for () {
    | ^^^^^^^^^^^^^
diff --git a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs
index d32737fcb62f..4d06cdc7620f 100644
--- a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs
+++ b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs
@@ -1,6 +1,7 @@
 // Detect and reject escaping late-bound generic params in
 // the type of assoc consts used in an equality bound.
 #![feature(
+    adt_const_params,
     min_generic_const_args,
     unsized_const_params,
     generic_const_parameter_types,
diff --git a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr
index 122893662933..3966483aa600 100644
--- a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr
@@ -1,5 +1,5 @@
 error: the type of the associated constant `K` cannot capture late-bound generic parameters
-  --> $DIR/esc-bound-var-in-ty.rs:15:35
+  --> $DIR/esc-bound-var-in-ty.rs:16:35
    |
 LL | fn take(_: impl for<'r> Trait<'r, K = const { &() }>) {}
    |                     --            ^ its type cannot capture the late-bound lifetime parameter `'r`
diff --git a/tests/ui/const-generics/mgca/adt_expr_arg_simple.stderr b/tests/ui/const-generics/mgca/adt_expr_arg_simple.stderr
index 8b9c228c1ee5..f1d5e5c67475 100644
--- a/tests/ui/const-generics/mgca/adt_expr_arg_simple.stderr
+++ b/tests/ui/const-generics/mgca/adt_expr_arg_simple.stderr
@@ -9,6 +9,8 @@ error: generic parameters may not be used in const operations
    |
 LL |     foo::<{ Some:: { 0: const { N + 1 } } }>();
    |                                      ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/mgca/early-bound-param-lt-bad.stderr b/tests/ui/const-generics/mgca/early-bound-param-lt-bad.stderr
index 461a26e33a3c..e0804158952d 100644
--- a/tests/ui/const-generics/mgca/early-bound-param-lt-bad.stderr
+++ b/tests/ui/const-generics/mgca/early-bound-param-lt-bad.stderr
@@ -3,6 +3,8 @@ error: generic parameters may not be used in const operations
    |
 LL |     T: Trait
    |                              ^^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/explicit_anon_consts.stderr b/tests/ui/const-generics/mgca/explicit_anon_consts.stderr
index 551815c4c31a..1251f4415171 100644
--- a/tests/ui/const-generics/mgca/explicit_anon_consts.stderr
+++ b/tests/ui/const-generics/mgca/explicit_anon_consts.stderr
@@ -39,42 +39,56 @@ error: generic parameters may not be used in const operations
    |
 LL | const ITEM3: usize = const { N };
    |                                              ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/explicit_anon_consts.rs:60:31
    |
 LL |     T3: Trait,
    |                               ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/explicit_anon_consts.rs:69:58
    |
 LL | struct Default3;
    |                                                          ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/explicit_anon_consts.rs:28:27
    |
 LL |     let _3 = [(); const { N }];
    |                           ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/explicit_anon_consts.rs:33:26
    |
 LL |     let _6: [(); const { N }] = todo!();
    |                          ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/explicit_anon_consts.rs:11:41
    |
 LL | type Adt3 = Foo;
    |                                         ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/explicit_anon_consts.rs:19:42
    |
 LL | type Arr3 = [(); const { N }];
    |                                          ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 13 previous errors
 
diff --git a/tests/ui/const-generics/mgca/selftyalias-containing-param.stderr b/tests/ui/const-generics/mgca/selftyalias-containing-param.stderr
index fdd3e6efdf65..1c841e39e673 100644
--- a/tests/ui/const-generics/mgca/selftyalias-containing-param.stderr
+++ b/tests/ui/const-generics/mgca/selftyalias-containing-param.stderr
@@ -9,6 +9,7 @@ note: not a concrete type
    |
 LL | impl S {
    |                      ^^^^
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/selftyparam.stderr b/tests/ui/const-generics/mgca/selftyparam.stderr
index 376e63da9a75..c3e0770fb978 100644
--- a/tests/ui/const-generics/mgca/selftyparam.stderr
+++ b/tests/ui/const-generics/mgca/selftyparam.stderr
@@ -3,6 +3,8 @@ error: generic parameters may not be used in const operations
    |
 LL |     fn foo() -> [(); const { let _: Self; 1 }];
    |                                     ^^^^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
index 913d8195fe21..6d8d3b4054d3 100644
--- a/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
+++ b/tests/ui/const-generics/mgca/size-of-generic-ptr-in-array-len.stderr
@@ -9,6 +9,8 @@ error: generic parameters may not be used in const operations
    |
 LL |     [0; const { size_of::<*mut T>() }];
    |                                ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/mgca/tuple_ctor_complex_args.stderr b/tests/ui/const-generics/mgca/tuple_ctor_complex_args.stderr
index e0ea3fd5560c..2961c714d141 100644
--- a/tests/ui/const-generics/mgca/tuple_ctor_complex_args.stderr
+++ b/tests/ui/const-generics/mgca/tuple_ctor_complex_args.stderr
@@ -9,6 +9,8 @@ error: generic parameters may not be used in const operations
    |
 LL |     with_point::<{ Point(const { N + 1 }, N) }>();
    |                                  ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/mgca/tuple_expr_arg_complex.stderr b/tests/ui/const-generics/mgca/tuple_expr_arg_complex.stderr
index b294e1032ce8..b4853d3c2e38 100644
--- a/tests/ui/const-generics/mgca/tuple_expr_arg_complex.stderr
+++ b/tests/ui/const-generics/mgca/tuple_expr_arg_complex.stderr
@@ -21,6 +21,8 @@ error: generic parameters may not be used in const operations
    |
 LL |     takes_nested_tuple::<{ (N, (N, const { N + 1 })) }>();
    |                                            ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 4 previous errors
 
diff --git a/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr b/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr
index f339e82beeed..8d43f2177562 100644
--- a/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr
+++ b/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr
@@ -3,12 +3,16 @@ error: generic parameters may not be used in const operations
    |
 LL | const FREE1: usize = const { std::mem::size_of::() };
    |                                                     ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/type_const-on-generic-expr.rs:8:46
    |
 LL | const FREE2: usize = const { I + 1 };
    |                                              ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr b/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr
index 9d4e99ca0aaf..1a6097752e33 100644
--- a/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr
+++ b/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr
@@ -3,18 +3,24 @@ error: generic parameters may not be used in const operations
    |
 LL |     const N1: usize = const { std::mem::size_of::() };
    |                                                      ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/type_const-on-generic_expr-2.rs:20:47
    |
 LL |     const N2: usize = const { I + 1 };
    |                                               ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
   --> $DIR/type_const-on-generic_expr-2.rs:23:35
    |
 LL |     const N3: usize = const { 2 & X };
    |                                   ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs
index 588fa2f913b6..c01fb47002b2 100644
--- a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs
+++ b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs
@@ -35,7 +35,6 @@ const NON_TYPE_CONST: usize = const { 1 };
 #[type_const]
 //~^ ERROR: the `#[type_const]` attribute is an experimental feature
 const TYPE_CONST: usize = const { 1 };
-//~^ ERROR: unbraced const blocks as const args are experimental
 
 static STATIC: usize = const { 1 };
 
diff --git a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr
index 00db630c27e9..bbcd9f9897a6 100644
--- a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr
+++ b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr
@@ -48,16 +48,6 @@ LL |         generic::();
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error[E0658]: unbraced const blocks as const args are experimental
-  --> $DIR/unbraced_const_block_const_arg_gated.rs:37:27
-   |
-LL | const TYPE_CONST: usize = const { 1 };
-   |                           ^^^^^^^^^^^
-   |
-   = note: see issue #132980  for more information
-   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
-   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
-
 error[E0658]: the `#[type_const]` attribute is an experimental feature
   --> $DIR/unbraced_const_block_const_arg_gated.rs:35:1
    |
@@ -68,6 +58,6 @@ LL | #[type_const]
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error: aborting due to 7 previous errors
+error: aborting due to 6 previous errors
 
 For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/const-generics/ogca/basic-fail.rs b/tests/ui/const-generics/ogca/basic-fail.rs
new file mode 100644
index 000000000000..87176c7b067e
--- /dev/null
+++ b/tests/ui/const-generics/ogca/basic-fail.rs
@@ -0,0 +1,22 @@
+#![feature(generic_const_items)]
+#![feature(min_generic_const_args)]
+#![feature(opaque_generic_const_args)]
+#![expect(incomplete_features)]
+
+#[type_const]
+const ADD1: usize = const { N + 1 };
+
+#[type_const]
+const INC: usize = const { N + 1 };
+
+#[type_const]
+const ONE: usize = ADD1::<0>;
+
+#[type_const]
+const OTHER_ONE: usize = INC::<0>;
+
+// Not definitionally equal.
+const ARR: [(); ADD1::<0>] = [(); INC::<0>];
+//~^ ERROR mismatched types
+
+fn main() {}
diff --git a/tests/ui/const-generics/ogca/basic-fail.stderr b/tests/ui/const-generics/ogca/basic-fail.stderr
new file mode 100644
index 000000000000..05de01b132c6
--- /dev/null
+++ b/tests/ui/const-generics/ogca/basic-fail.stderr
@@ -0,0 +1,9 @@
+error[E0308]: mismatched types
+  --> $DIR/basic-fail.rs:19:30
+   |
+LL | const ARR: [(); ADD1::<0>] = [(); INC::<0>];
+   |                              ^^^^^^^^^^^^^^ expected an array with a size of const { N + 1 }, found one with a size of const { N + 1 }
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0308`.
diff --git a/tests/ui/const-generics/ogca/basic.rs b/tests/ui/const-generics/ogca/basic.rs
new file mode 100644
index 000000000000..c8aec9ee24ce
--- /dev/null
+++ b/tests/ui/const-generics/ogca/basic.rs
@@ -0,0 +1,22 @@
+//@ check-pass
+
+#![feature(generic_const_items)]
+#![feature(min_generic_const_args)]
+#![feature(opaque_generic_const_args)]
+#![expect(incomplete_features)]
+
+#[type_const]
+const ADD1: usize = const { N + 1 };
+
+#[type_const]
+const INC: usize = ADD1::;
+
+#[type_const]
+const ONE: usize = ADD1::<0>;
+
+#[type_const]
+const OTHER_ONE: usize = INC::<0>;
+
+const ARR: [(); ADD1::<0>] = [(); INC::<0>];
+
+fn main() {}
diff --git a/tests/ui/const-generics/ogca/coherence-ambiguous.rs b/tests/ui/const-generics/ogca/coherence-ambiguous.rs
new file mode 100644
index 000000000000..c8c088b1e8fc
--- /dev/null
+++ b/tests/ui/const-generics/ogca/coherence-ambiguous.rs
@@ -0,0 +1,21 @@
+// FIXME(ogca): this should ERROR not pass!!
+//@ check-pass
+
+#![feature(generic_const_items, min_generic_const_args, opaque_generic_const_args)]
+#![expect(incomplete_features)]
+
+#[type_const]
+const FOO: usize = const { N + 1 };
+
+#[type_const]
+const BAR: usize = const { N + 1 };
+
+trait Trait {}
+
+impl Trait for [(); FOO::<1>] {}
+impl Trait for [(); BAR::<1>] {}
+// FIXME(ogca): this should ERROR!
+impl Trait for [(); BAR::<2>] {}
+// FIXME(ogca): this should ERROR!
+
+fn main() {}
diff --git a/tests/ui/const-generics/ogca/rhs-but-not-root.rs b/tests/ui/const-generics/ogca/rhs-but-not-root.rs
new file mode 100644
index 000000000000..4ed136f04ced
--- /dev/null
+++ b/tests/ui/const-generics/ogca/rhs-but-not-root.rs
@@ -0,0 +1,14 @@
+#![feature(generic_const_items)]
+#![feature(min_generic_const_args)]
+#![feature(opaque_generic_const_args)]
+#![expect(incomplete_features)]
+
+// Anon consts must be the root of the RHS to be OGCA.
+#[type_const]
+const FOO: usize = ID::;
+//~^ ERROR generic parameters may not be used in const operations
+
+#[type_const]
+const ID: usize = N;
+
+fn main() {}
diff --git a/tests/ui/const-generics/ogca/rhs-but-not-root.stderr b/tests/ui/const-generics/ogca/rhs-but-not-root.stderr
new file mode 100644
index 000000000000..f4fdf5fb981b
--- /dev/null
+++ b/tests/ui/const-generics/ogca/rhs-but-not-root.stderr
@@ -0,0 +1,8 @@
+error: generic parameters may not be used in const operations
+  --> $DIR/rhs-but-not-root.rs:8:49
+   |
+LL | const FOO: usize = ID::;
+   |                                                 ^
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs
new file mode 100644
index 000000000000..0b78911158ba
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs
@@ -0,0 +1,9 @@
+#![feature(generic_const_items, min_generic_const_args)]
+#![expect(incomplete_features)]
+
+#[type_const]
+const INC: usize = const { N + 1 };
+//~^ ERROR generic parameters may not be used in const operations
+//~| HELP add `#![feature(opaque_generic_const_args)]`
+
+fn main() {}
diff --git a/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr
new file mode 100644
index 000000000000..1b87443aa3cb
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr
@@ -0,0 +1,10 @@
+error: generic parameters may not be used in const operations
+  --> $DIR/feature-gate-opaque-generic-const-args.rs:5:44
+   |
+LL | const INC: usize = const { N + 1 };
+   |                                            ^
+   |
+   = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs b/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs
index fb2506daf078..518ed6452369 100644
--- a/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs
+++ b/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs
@@ -1,6 +1,6 @@
 // ICE: assertion failed: !value.has_infer()
 // issue: rust-lang/rust#115806
-#![feature(min_generic_const_args, unsized_const_params)]
+#![feature(adt_const_params, min_generic_const_args, unsized_const_params)]
 #![allow(incomplete_features)]
 
 pub struct NoPin;

From 875c5908c747f5e97227aab367d4bcc7c161c892 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sun, 8 Feb 2026 20:39:02 +0100
Subject: [PATCH 1466/1843] Update examples for diagnostics changes

Co-authored-by: bjorn3 
---
 .../examples/rustc-interface-example.rs           | 15 +++++----------
 .../rustc-interface-getting-diagnostics.rs        |  8 +++-----
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/src/doc/rustc-dev-guide/examples/rustc-interface-example.rs b/src/doc/rustc-dev-guide/examples/rustc-interface-example.rs
index 360f70c8e865..b4439504650a 100644
--- a/src/doc/rustc-dev-guide/examples/rustc-interface-example.rs
+++ b/src/doc/rustc-dev-guide/examples/rustc-interface-example.rs
@@ -4,14 +4,12 @@
 
 extern crate rustc_driver;
 extern crate rustc_error_codes;
-extern crate rustc_errors;
 extern crate rustc_hash;
 extern crate rustc_hir;
 extern crate rustc_interface;
 extern crate rustc_session;
 extern crate rustc_span;
 
-use rustc_errors::registry;
 use rustc_hash::FxHashMap;
 use rustc_session::config;
 
@@ -30,12 +28,11 @@ fn main() {
     println!("{HELLO}");
 }
 "#
-            .into(),
+                .into(),
         },
-        output_dir: None,  // Option
-        output_file: None, // Option
-        file_loader: None, // Option>
-        locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_owned(),
+        output_dir: None,                // Option
+        output_file: None,               // Option
+        file_loader: None,               // Option>
         lint_caps: FxHashMap::default(), // FxHashMap
         // This is a callback from the driver that is called when [`ParseSess`] is created.
         psess_created: None, //Option>
@@ -50,8 +47,6 @@ fn main() {
         //
         // The second parameter is local providers and the third parameter is external providers.
         override_queries: None, // Option, &mut ty::query::Providers<'_>)>
-        // Registry of diagnostics codes.
-        registry: registry::Registry::new(rustc_errors::codes::DIAGNOSTICS),
         make_codegen_backend: None,
         expanded_args: Vec::new(),
         ice_file: None,
@@ -77,4 +72,4 @@ fn main() {
             }
         });
     });
-}
+}
\ No newline at end of file
diff --git a/src/doc/rustc-dev-guide/examples/rustc-interface-getting-diagnostics.rs b/src/doc/rustc-dev-guide/examples/rustc-interface-getting-diagnostics.rs
index 2512ba3c3f92..342316ba670a 100644
--- a/src/doc/rustc-dev-guide/examples/rustc-interface-getting-diagnostics.rs
+++ b/src/doc/rustc-dev-guide/examples/rustc-interface-getting-diagnostics.rs
@@ -15,7 +15,7 @@ extern crate rustc_span;
 use std::sync::{Arc, Mutex};
 
 use rustc_errors::emitter::Emitter;
-use rustc_errors::registry::{self, Registry};
+use rustc_errors::registry::Registry;
 use rustc_errors::translation::Translate;
 use rustc_errors::{DiagInner, FluentBundle};
 use rustc_session::config;
@@ -59,14 +59,13 @@ fn main() {
     let x: &str = 1;
 }
 "
-            .into(),
+                .into(),
         },
         crate_cfg: Vec::new(),
         crate_check_cfg: Vec::new(),
         output_dir: None,
         output_file: None,
         file_loader: None,
-        locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_owned(),
         lint_caps: rustc_hash::FxHashMap::default(),
         psess_created: Some(Box::new(|parse_sess| {
             parse_sess.dcx().set_emitter(Box::new(DebugEmitter {
@@ -76,7 +75,6 @@ fn main() {
         })),
         register_lints: None,
         override_queries: None,
-        registry: registry::Registry::new(rustc_errors::codes::DIAGNOSTICS),
         make_codegen_backend: None,
         expanded_args: Vec::new(),
         ice_file: None,
@@ -99,4 +97,4 @@ fn main() {
     buffer.lock().unwrap().iter().for_each(|diagnostic| {
         println!("{diagnostic:#?}");
     });
-}
+}
\ No newline at end of file

From 4a979d546bc779f97e6224e8ab3d26f8bd69a6e9 Mon Sep 17 00:00:00 2001
From: Mark Rousskov 
Date: Sun, 8 Feb 2026 19:54:03 +0000
Subject: [PATCH 1467/1843] Stop having two different alignment constants

* Stop having two different alignment constants
* Update library/core/src/alloc/global.rs
---
 library/core/src/alloc/global.rs                           | 7 ++++---
 library/core/src/mem/mod.rs                                | 5 ++++-
 library/core/src/ptr/alignment.rs                          | 3 +--
 .../default_boxed_slice.main.GVN.32bit.panic-abort.diff    | 2 +-
 .../default_boxed_slice.main.GVN.32bit.panic-unwind.diff   | 2 +-
 .../default_boxed_slice.main.GVN.64bit.panic-abort.diff    | 2 +-
 .../default_boxed_slice.main.GVN.64bit.panic-unwind.diff   | 2 +-
 tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff       | 2 +-
 8 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/library/core/src/alloc/global.rs b/library/core/src/alloc/global.rs
index d18e1f525d10..bf3d3e0a5aca 100644
--- a/library/core/src/alloc/global.rs
+++ b/library/core/src/alloc/global.rs
@@ -284,9 +284,10 @@ pub unsafe trait GlobalAlloc {
     /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html
     #[stable(feature = "global_alloc", since = "1.28.0")]
     unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
-        // SAFETY: the caller must ensure that the `new_size` does not overflow.
-        // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid.
-        let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
+        let alignment = layout.alignment();
+        // SAFETY: the caller must ensure that the `new_size` does not overflow
+        // when rounded up to the next multiple of `alignment`.
+        let new_layout = unsafe { Layout::from_size_alignment_unchecked(new_size, alignment) };
         // SAFETY: the caller must ensure that `new_layout` is greater than zero.
         let new_ptr = unsafe { self.alloc(new_layout) };
         if !new_ptr.is_null() {
diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs
index 7c486875a826..eb6f8f975721 100644
--- a/library/core/src/mem/mod.rs
+++ b/library/core/src/mem/mod.rs
@@ -1260,7 +1260,10 @@ pub trait SizedTypeProperties: Sized {
 
     #[doc(hidden)]
     #[unstable(feature = "ptr_alignment_type", issue = "102070")]
-    const ALIGNMENT: Alignment = Alignment::of::();
+    const ALIGNMENT: Alignment = {
+        // This can't panic since type alignment is always a power of two.
+        Alignment::new(Self::ALIGN).unwrap()
+    };
 
     /// `true` if this type requires no storage.
     /// `false` if its [size](size_of) is greater than zero.
diff --git a/library/core/src/ptr/alignment.rs b/library/core/src/ptr/alignment.rs
index 7c34b026e14b..b27930de4e66 100644
--- a/library/core/src/ptr/alignment.rs
+++ b/library/core/src/ptr/alignment.rs
@@ -52,8 +52,7 @@ impl Alignment {
     #[inline]
     #[must_use]
     pub const fn of() -> Self {
-        // This can't actually panic since type alignment is always a power of two.
-        const { Alignment::new(align_of::()).unwrap() }
+        ::ALIGNMENT
     }
 
     /// Returns the [ABI]-required minimum alignment of the type of the value that `val` points to.
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff
index 7d66d3129115..308f19ea759d 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff
@@ -44,7 +44,7 @@
           StorageLive(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = const std::ptr::Alignment::of::<[bool; 0]>::{constant#0} as *const [bool; 0] (Transmute);
+-         _6 = const <[bool; 0] as std::mem::SizedTypeProperties>::ALIGNMENT as *const [bool; 0] (Transmute);
 -         _5 = NonNull::<[bool; 0]> { pointer: copy _6 };
 +         _6 = const {0x1 as *const [bool; 0]};
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff
index cc00bd300a3c..819ad6054df8 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff
@@ -44,7 +44,7 @@
           StorageLive(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = const std::ptr::Alignment::of::<[bool; 0]>::{constant#0} as *const [bool; 0] (Transmute);
+-         _6 = const <[bool; 0] as std::mem::SizedTypeProperties>::ALIGNMENT as *const [bool; 0] (Transmute);
 -         _5 = NonNull::<[bool; 0]> { pointer: copy _6 };
 +         _6 = const {0x1 as *const [bool; 0]};
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff
index 9380cdd6ccb4..7029e02a857a 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff
@@ -44,7 +44,7 @@
           StorageLive(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = const std::ptr::Alignment::of::<[bool; 0]>::{constant#0} as *const [bool; 0] (Transmute);
+-         _6 = const <[bool; 0] as std::mem::SizedTypeProperties>::ALIGNMENT as *const [bool; 0] (Transmute);
 -         _5 = NonNull::<[bool; 0]> { pointer: copy _6 };
 +         _6 = const {0x1 as *const [bool; 0]};
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff
index bea564376274..23a134f3666b 100644
--- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff
@@ -44,7 +44,7 @@
           StorageLive(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = const std::ptr::Alignment::of::<[bool; 0]>::{constant#0} as *const [bool; 0] (Transmute);
+-         _6 = const <[bool; 0] as std::mem::SizedTypeProperties>::ALIGNMENT as *const [bool; 0] (Transmute);
 -         _5 = NonNull::<[bool; 0]> { pointer: copy _6 };
 +         _6 = const {0x1 as *const [bool; 0]};
 +         _5 = const NonNull::<[bool; 0]> {{ pointer: {0x1 as *const [bool; 0]} }};
diff --git a/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff b/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff
index 9e543699da70..23889b266e4a 100644
--- a/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff
+++ b/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff
@@ -40,7 +40,7 @@
           StorageLive(_1);
           StorageLive(_2);
           StorageLive(_3);
--         _3 = const std::ptr::Alignment::of::::{constant#0};
+-         _3 = const ::ALIGNMENT;
 -         _2 = copy _3 as *mut u8 (Transmute);
 +         _3 = const std::ptr::Alignment {{ _inner_repr_trick: std::ptr::alignment::AlignmentEnum::_Align1Shl0 }};
 +         _2 = const {0x1 as *mut u8};

From 59111f7a73fda12e0401cb35ccab7fbcd80ad272 Mon Sep 17 00:00:00 2001
From: Vincent Esche 
Date: Wed, 4 Feb 2026 09:01:24 +0100
Subject: [PATCH 1468/1843] Add `expression_types()`, `pattern_types()`,
 `binding_types()` to `DefWithBody`

---
 src/tools/rust-analyzer/crates/hir/src/lib.rs | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/src/tools/rust-analyzer/crates/hir/src/lib.rs b/src/tools/rust-analyzer/crates/hir/src/lib.rs
index 4b615665167c..0167ccaa1002 100644
--- a/src/tools/rust-analyzer/crates/hir/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/hir/src/lib.rs
@@ -2244,6 +2244,39 @@ impl DefWithBody {
             acc.push(diag.into())
         }
     }
+
+    /// Returns an iterator over the inferred types of all expressions in this body.
+    pub fn expression_types<'db>(
+        self,
+        db: &'db dyn HirDatabase,
+    ) -> impl Iterator> {
+        self.id().into_iter().flat_map(move |def_id| {
+            let infer = InferenceResult::for_body(db, def_id);
+            let resolver = def_id.resolver(db);
+
+            infer.expression_types().map(move |(_, ty)| Type::new_with_resolver(db, &resolver, ty))
+        })
+    }
+
+    /// Returns an iterator over the inferred types of all patterns in this body.
+    pub fn pattern_types<'db>(self, db: &'db dyn HirDatabase) -> impl Iterator> {
+        self.id().into_iter().flat_map(move |def_id| {
+            let infer = InferenceResult::for_body(db, def_id);
+            let resolver = def_id.resolver(db);
+
+            infer.pattern_types().map(move |(_, ty)| Type::new_with_resolver(db, &resolver, ty))
+        })
+    }
+
+    /// Returns an iterator over the inferred types of all bindings in this body.
+    pub fn binding_types<'db>(self, db: &'db dyn HirDatabase) -> impl Iterator> {
+        self.id().into_iter().flat_map(move |def_id| {
+            let infer = InferenceResult::for_body(db, def_id);
+            let resolver = def_id.resolver(db);
+
+            infer.binding_types().map(move |(_, ty)| Type::new_with_resolver(db, &resolver, ty))
+        })
+    }
 }
 
 fn expr_store_diagnostics<'db>(

From f1b4c2a0e64ccb5c1b4b2b2391ac91f40d143302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 14:42:17 +0100
Subject: [PATCH 1469/1843] Port rustc_insignificant_dtor to the new attribute
 parser

---
 .../src/attributes/rustc_internal.rs                | 13 +++++++++++++
 compiler/rustc_attr_parsing/src/context.rs          |  1 +
 compiler/rustc_hir/src/attrs/data_structures.rs     |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs  |  1 +
 compiler/rustc_passes/src/check_attr.rs             |  2 +-
 compiler/rustc_ty_utils/src/needs_drop.rs           |  6 +++---
 6 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index a8a0ee3e5e22..4740cc107353 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -739,6 +739,19 @@ impl CombineAttributeParser for RustcThenThisWouldNeedParser {
     }
 }
 
+pub(crate) struct RustcInsignificantDtorParser;
+
+impl NoArgsAttributeParser for RustcInsignificantDtorParser {
+    const PATH: &[Symbol] = &[sym::rustc_insignificant_dtor];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Enum),
+        Allow(Target::Struct),
+        Allow(Target::ForeignTy),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcInsignificantDtor;
+}
+
 pub(crate) struct RustcEffectiveVisibilityParser;
 
 impl NoArgsAttributeParser for RustcEffectiveVisibilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index ad39b0ec26d9..7921928ce7c5 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -270,6 +270,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 79452913c259..aaeebcc5c66d 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1120,6 +1120,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_if_this_changed]`
     RustcIfThisChanged(Span, Option),
 
+    /// Represents `#[rustc_insignificant_dtor]`
+    RustcInsignificantDtor,
+
     /// Represents `#[rustc_intrinsic]`
     RustcIntrinsic,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index ba351d80544c..4fb20599a026 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -118,6 +118,7 @@ impl AttributeKind {
             RustcHasIncoherentInherentImpls => Yes,
             RustcHiddenTypeOfOpaques => No,
             RustcIfThisChanged(..) => No,
+            RustcInsignificantDtor => Yes,
             RustcIntrinsic => Yes,
             RustcIntrinsicConstStableIndirect => No,
             RustcLayout(..) => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 6c4b76a08ad2..2f1695f0712a 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -313,6 +313,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcHasIncoherentInherentImpls
                     | AttributeKind::RustcHiddenTypeOfOpaques
                     | AttributeKind::RustcIfThisChanged(..)
+                    | AttributeKind::RustcInsignificantDtor
                     | AttributeKind::RustcIntrinsic
                     | AttributeKind::RustcIntrinsicConstStableIndirect
                     | AttributeKind::RustcLayout(..)
@@ -386,7 +387,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::default_lib_allocator
                             | sym::rustc_diagnostic_item
                             | sym::rustc_no_mir_inline
-                            | sym::rustc_insignificant_dtor
                             | sym::rustc_nonnull_optimization_guaranteed
                             | sym::rustc_inherit_overflow_checks
                             | sym::rustc_trivial_field_reads
diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs
index 0ef435b1a0e2..06eef7e95145 100644
--- a/compiler/rustc_ty_utils/src/needs_drop.rs
+++ b/compiler/rustc_ty_utils/src/needs_drop.rs
@@ -1,13 +1,14 @@
 //! Check whether a type has (potentially) non-trivial drop glue.
 
 use rustc_data_structures::fx::FxHashSet;
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def_id::DefId;
+use rustc_hir::find_attr;
 use rustc_hir::limit::Limit;
 use rustc_middle::bug;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::util::{AlwaysRequiresDrop, needs_drop_components};
 use rustc_middle::ty::{self, EarlyBinder, GenericArgsRef, Ty, TyCtxt};
-use rustc_span::sym;
 use tracing::{debug, instrument};
 
 use crate::errors::NeedsDropOverflow;
@@ -396,8 +397,7 @@ fn adt_consider_insignificant_dtor<'tcx>(
     tcx: TyCtxt<'tcx>,
 ) -> impl Fn(ty::AdtDef<'tcx>) -> Option {
     move |adt_def: ty::AdtDef<'tcx>| {
-        let is_marked_insig = tcx.has_attr(adt_def.did(), sym::rustc_insignificant_dtor);
-        if is_marked_insig {
+        if find_attr!(tcx.get_all_attrs(adt_def.did()), AttributeKind::RustcInsignificantDtor) {
             // In some cases like `std::collections::HashMap` where the struct is a wrapper around
             // a type that is a Drop type, and the wrapped type (eg: `hashbrown::HashMap`) lies
             // outside stdlib, we might choose to still annotate the wrapper (std HashMap) with

From 9d6e120939cad8e87c3ccd4e2f33d3591b82edf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 15:06:32 +0100
Subject: [PATCH 1470/1843] remove from impl block in std

---
 library/alloc/src/ffi/c_str.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/alloc/src/ffi/c_str.rs b/library/alloc/src/ffi/c_str.rs
index d6dcba7107a9..fba967c04895 100644
--- a/library/alloc/src/ffi/c_str.rs
+++ b/library/alloc/src/ffi/c_str.rs
@@ -103,6 +103,7 @@ use crate::vec::Vec;
 /// and other memory errors.
 #[derive(PartialEq, PartialOrd, Eq, Ord, Hash, Clone)]
 #[rustc_diagnostic_item = "cstring_type"]
+#[rustc_insignificant_dtor]
 #[stable(feature = "alloc_c_string", since = "1.64.0")]
 pub struct CString {
     // Invariant 1: the slice ends with a zero byte and has a length of at least one.
@@ -694,7 +695,6 @@ impl CString {
 // memory-unsafe code from working by accident. Inline
 // to prevent LLVM from optimizing it away in debug builds.
 #[stable(feature = "cstring_drop", since = "1.13.0")]
-#[rustc_insignificant_dtor]
 impl Drop for CString {
     #[inline]
     fn drop(&mut self) {

From 2dda303229b936641c29db1f277a9940473ebc70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Thu, 5 Feb 2026 15:12:05 +0100
Subject: [PATCH 1471/1843] Port rustc_strict_coherence to the new attribute
 parser

---
 .../src/attributes/rustc_internal.rs          | 15 +++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs    |  1 +
 .../rustc_hir/src/attrs/data_structures.rs    |  4 ++++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 compiler/rustc_middle/src/error.rs            |  2 +-
 .../src/traits/specialization_graph.rs        | 19 ++++++-------------
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 7 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index a8a0ee3e5e22..c4c0a04c44e4 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -845,3 +845,18 @@ impl NoArgsAttributeParser for RustcIntrinsicConstStableIndirectPar
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcIntrinsicConstStableIndirect;
 }
+
+pub(crate) struct RustcStrictCoherenceParser;
+
+impl NoArgsAttributeParser for RustcStrictCoherenceParser {
+    const PATH: &[Symbol] = &[sym::rustc_strict_coherence];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Trait),
+        Allow(Target::Struct),
+        Allow(Target::Enum),
+        Allow(Target::Union),
+        Allow(Target::ForeignTy),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = AttributeKind::RustcStrictCoherence;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index ad39b0ec26d9..16a3403b5243 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -286,6 +286,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 79452913c259..de7b2bd06546 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1230,6 +1230,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_std_internal_symbol]`.
     RustcStdInternalSymbol(Span),
 
+    /// Represents `#[rustc_strict_coherence]`.
+    RustcStrictCoherence(Span),
+
     /// Represents `#[rustc_symbol_name]`
     RustcSymbolName(Span),
 
@@ -1266,6 +1269,7 @@ pub enum AttributeKind {
         /// Span of the attribute.
         span: Span,
     },
+
     /// Represents `#[target_feature(enable = "...")]` and
     /// `#[unsafe(force_target_feature(enable = "...")]`.
     TargetFeature { features: ThinVec<(Symbol, Span)>, attr_span: Span, was_forced: bool },
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index ba351d80544c..954c72ca86fe 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -153,6 +153,7 @@ impl AttributeKind {
             RustcSkipDuringMethodDispatch { .. } => No,
             RustcSpecializationTrait(..) => No,
             RustcStdInternalSymbol(..) => No,
+            RustcStrictCoherence(..) => Yes,
             RustcSymbolName(..) => Yes,
             RustcThenThisWouldNeed(..) => No,
             RustcUnsafeSpecializationMarker(..) => No,
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
index a5d5978916eb..d487caf4d3ac 100644
--- a/compiler/rustc_middle/src/error.rs
+++ b/compiler/rustc_middle/src/error.rs
@@ -93,7 +93,7 @@ pub(crate) struct StrictCoherenceNeedsNegativeCoherence {
     #[primary_span]
     pub span: Span,
     #[label("due to this attribute")]
-    pub attr_span: Option,
+    pub attr_span: Span,
 }
 
 #[derive(Diagnostic)]
diff --git a/compiler/rustc_middle/src/traits/specialization_graph.rs b/compiler/rustc_middle/src/traits/specialization_graph.rs
index 05c19db4caa8..84415a592a2e 100644
--- a/compiler/rustc_middle/src/traits/specialization_graph.rs
+++ b/compiler/rustc_middle/src/traits/specialization_graph.rs
@@ -1,8 +1,9 @@
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::ErrorGuaranteed;
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def_id::{DefId, DefIdMap};
+use rustc_hir::find_attr;
 use rustc_macros::{HashStable, TyDecodable, TyEncodable};
-use rustc_span::sym;
 
 use crate::error::StrictCoherenceNeedsNegativeCoherence;
 use crate::ty::fast_reject::SimplifiedType;
@@ -61,23 +62,15 @@ pub enum OverlapMode {
 impl OverlapMode {
     pub fn get(tcx: TyCtxt<'_>, trait_id: DefId) -> OverlapMode {
         let with_negative_coherence = tcx.features().with_negative_coherence();
-        let strict_coherence = tcx.has_attr(trait_id, sym::rustc_strict_coherence);
+        let strict_coherence = find_attr!(tcx.get_all_attrs(trait_id), AttributeKind::RustcStrictCoherence(span) => *span);
 
         if with_negative_coherence {
-            if strict_coherence { OverlapMode::Strict } else { OverlapMode::WithNegative }
+            if strict_coherence.is_some() { OverlapMode::Strict } else { OverlapMode::WithNegative }
         } else {
-            if strict_coherence {
-                let attr_span = trait_id
-                    .as_local()
-                    .into_iter()
-                    .flat_map(|local_def_id| {
-                        tcx.hir_attrs(tcx.local_def_id_to_hir_id(local_def_id))
-                    })
-                    .find(|attr| attr.has_name(sym::rustc_strict_coherence))
-                    .map(|attr| attr.span());
+            if let Some(span) = strict_coherence {
                 tcx.dcx().emit_err(StrictCoherenceNeedsNegativeCoherence {
                     span: tcx.def_span(trait_id),
-                    attr_span,
+                    attr_span: span,
                 });
             }
             OverlapMode::Stable
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 6c4b76a08ad2..c29e4b2b1afe 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -344,6 +344,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcSkipDuringMethodDispatch { .. }
                     | AttributeKind::RustcSpecializationTrait(..)
                     | AttributeKind::RustcStdInternalSymbol (..)
+                    | AttributeKind::RustcStrictCoherence(..)
                     | AttributeKind::RustcSymbolName(..)
                     | AttributeKind::RustcThenThisWouldNeed(..)
                     | AttributeKind::RustcUnsafeSpecializationMarker(..)
@@ -402,7 +403,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_never_type_options
                             | sym::rustc_autodiff
                             | sym::rustc_capture_analysis
-                            | sym::rustc_strict_coherence
                             | sym::rustc_mir
                             | sym::rustc_outlives
                             | sym::rustc_evaluate_where_clauses

From dfef1209b1ed4d4bb9eee0d00fe3df1154e3edf7 Mon Sep 17 00:00:00 2001
From: Folkert de Vries 
Date: Sat, 31 Jan 2026 22:33:56 +0100
Subject: [PATCH 1472/1843] x86: use `intrinsics::simd` for masked truncated
 saturating stores

---
 .../crates/core_arch/src/x86/avx512bw.rs      | 43 +++++++++++++------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs
index 8e074fdcfa48..3ba171c0fa50 100644
--- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs
+++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs
@@ -12476,7 +12476,14 @@ pub unsafe fn _mm512_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask32,
 #[stable(feature = "stdarch_x86_avx512", since = "1.89")]
 #[cfg_attr(test, assert_instr(vpmovswb))]
 pub unsafe fn _mm256_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m256i) {
-    vpmovswbmem256(mem_addr, a.as_i16x16(), k);
+    let mask = simd_select_bitmask(k, i16x16::splat(!0), i16x16::ZERO);
+
+    let max = simd_splat(i16::from(i8::MAX));
+    let min = simd_splat(i16::from(i8::MIN));
+
+    let v = simd_imax(simd_imin(a.as_i16x16(), max), min);
+    let truncated: i8x16 = simd_cast(v);
+    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, truncated);
 }
 
 /// Convert packed signed 16-bit integers in a to packed 8-bit integers with signed saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr.
@@ -12487,7 +12494,14 @@ pub unsafe fn _mm256_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16,
 #[stable(feature = "stdarch_x86_avx512", since = "1.89")]
 #[cfg_attr(test, assert_instr(vpmovswb))]
 pub unsafe fn _mm_mask_cvtsepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) {
-    vpmovswbmem128(mem_addr, a.as_i16x8(), k);
+    let mask = simd_select_bitmask(k, i16x8::splat(!0), i16x8::ZERO);
+
+    let max = simd_splat(i16::from(i8::MAX));
+    let min = simd_splat(i16::from(i8::MIN));
+
+    let v = simd_imax(simd_imin(a.as_i16x8(), max), min);
+    let truncated: i8x8 = simd_cast(v);
+    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, truncated);
 }
 
 /// Convert packed 16-bit integers in a to packed 8-bit integers with truncation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr.
@@ -12555,7 +12569,12 @@ pub unsafe fn _mm512_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask32
 #[stable(feature = "stdarch_x86_avx512", since = "1.89")]
 #[cfg_attr(test, assert_instr(vpmovuswb))]
 pub unsafe fn _mm256_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16, a: __m256i) {
-    vpmovuswbmem256(mem_addr, a.as_i16x16(), k);
+    let mask = simd_select_bitmask(k, i16x16::splat(!0), i16x16::ZERO);
+    let mem_addr = mem_addr.cast::();
+    let max = simd_splat(u16::from(u8::MAX));
+
+    let truncated: u8x16 = simd_cast(simd_imin(a.as_u16x16(), max));
+    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, truncated);
 }
 
 /// Convert packed unsigned 16-bit integers in a to packed unsigned 8-bit integers with unsigned saturation, and store the active results (those with their respective bit set in writemask k) to unaligned memory at base_addr.
@@ -12566,7 +12585,15 @@ pub unsafe fn _mm256_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask16
 #[stable(feature = "stdarch_x86_avx512", since = "1.89")]
 #[cfg_attr(test, assert_instr(vpmovuswb))]
 pub unsafe fn _mm_mask_cvtusepi16_storeu_epi8(mem_addr: *mut i8, k: __mmask8, a: __m128i) {
-    vpmovuswbmem128(mem_addr, a.as_i16x8(), k);
+    let mask = simd_select_bitmask(k, i16x8::splat(!0), i16x8::ZERO);
+    let mem_addr = mem_addr.cast::();
+    let max = simd_splat(u16::from(u8::MAX));
+
+    let v = a.as_u16x8();
+    let v = simd_imin(v, max);
+
+    let truncated: u8x8 = simd_cast(v);
+    simd_masked_store!(SimdAlign::Unaligned, mask, mem_addr, truncated);
 }
 
 #[allow(improper_ctypes)]
@@ -12632,17 +12659,9 @@ unsafe extern "C" {
 
     #[link_name = "llvm.x86.avx512.mask.pmovs.wb.mem.512"]
     fn vpmovswbmem(mem_addr: *mut i8, a: i16x32, mask: u32);
-    #[link_name = "llvm.x86.avx512.mask.pmovs.wb.mem.256"]
-    fn vpmovswbmem256(mem_addr: *mut i8, a: i16x16, mask: u16);
-    #[link_name = "llvm.x86.avx512.mask.pmovs.wb.mem.128"]
-    fn vpmovswbmem128(mem_addr: *mut i8, a: i16x8, mask: u8);
 
     #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.512"]
     fn vpmovuswbmem(mem_addr: *mut i8, a: i16x32, mask: u32);
-    #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.256"]
-    fn vpmovuswbmem256(mem_addr: *mut i8, a: i16x16, mask: u16);
-    #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.128"]
-    fn vpmovuswbmem128(mem_addr: *mut i8, a: i16x8, mask: u8);
 }
 
 #[cfg(test)]

From 2d65a33c612cf85e9a0b80e73c870520eb289008 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sat, 7 Feb 2026 00:01:35 +0100
Subject: [PATCH 1473/1843] Implement indent check

---
 .../src/diagnostics/diagnostic_builder.rs     |  7 +++-
 .../rustc_macros/src/diagnostics/message.rs   | 39 +++++++++++++++++--
 .../rustc_macros/src/diagnostics/utils.rs     |  2 +-
 3 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
index b386408a1918..de8ee42caf45 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
@@ -203,8 +203,11 @@ impl DiagnosticDeriveVariantBuilder {
                         )
                         .emit();
                     }
-                    self.message =
-                        Some(Message { message_span: message.span(), value: message.value() });
+                    self.message = Some(Message {
+                        attr_span: attr.span(),
+                        message_span: message.span(),
+                        value: message.value(),
+                    });
                 }
 
                 // Parse arguments
diff --git a/compiler/rustc_macros/src/diagnostics/message.rs b/compiler/rustc_macros/src/diagnostics/message.rs
index 18d4d60dde3e..3276abfce413 100644
--- a/compiler/rustc_macros/src/diagnostics/message.rs
+++ b/compiler/rustc_macros/src/diagnostics/message.rs
@@ -9,6 +9,7 @@ use crate::diagnostics::error::span_err;
 
 #[derive(Clone)]
 pub(crate) struct Message {
+    pub attr_span: Span,
     pub message_span: Span,
     pub value: String,
 }
@@ -19,12 +20,18 @@ impl Message {
     /// For subdiagnostics, we cannot check this.
     pub(crate) fn diag_message(&self, variant: Option<&VariantInfo<'_>>) -> TokenStream {
         let message = &self.value;
-        verify_fluent_message(self.message_span, &message, variant);
+        self.verify(variant);
         quote! { rustc_errors::DiagMessage::Inline(std::borrow::Cow::Borrowed(#message)) }
     }
+
+    fn verify(&self, variant: Option<&VariantInfo<'_>>) {
+        verify_variables_used(self.message_span, &self.value, variant);
+        verify_message_style(self.message_span, &self.value);
+        verify_message_formatting(self.attr_span, self.message_span, &self.value);
+    }
 }
 
-fn verify_fluent_message(msg_span: Span, message_str: &str, variant: Option<&VariantInfo<'_>>) {
+fn verify_variables_used(msg_span: Span, message_str: &str, variant: Option<&VariantInfo<'_>>) {
     // Parse the fluent message
     const GENERATED_MSG_ID: &str = "generated_msg";
     let resource =
@@ -53,8 +60,6 @@ fn verify_fluent_message(msg_span: Span, message_str: &str, variant: Option<&Var
             }
         }
     }
-
-    verify_message_style(msg_span, message_str);
 }
 
 fn variable_references<'a>(msg: &fluent_syntax::ast::Message<&'a str>) -> Vec<&'a str> {
@@ -120,3 +125,29 @@ fn verify_message_style(msg_span: Span, message: &str) {
         return;
     }
 }
+
+/// Verifies that the message is properly indented into the code
+fn verify_message_formatting(attr_span: Span, msg_span: Span, message: &str) {
+    // Find the indent at the start of the message (`column()` is one-indexed)
+    let start = attr_span.unwrap().column() - 1;
+
+    for line in message.lines().skip(1) {
+        if line.is_empty() {
+            continue;
+        }
+        let indent = line.chars().take_while(|c| *c == ' ').count();
+        if indent < start {
+            span_err(
+                msg_span.unwrap(),
+                format!("message is not properly indented. {indent} < {start}"),
+            )
+            .emit();
+            return;
+        }
+        if indent % 4 != 0 {
+            span_err(msg_span.unwrap(), "message is not indented with a multiple of 4 spaces")
+                .emit();
+            return;
+        }
+    }
+}
diff --git a/compiler/rustc_macros/src/diagnostics/utils.rs b/compiler/rustc_macros/src/diagnostics/utils.rs
index c308f6126325..55a8445744cb 100644
--- a/compiler/rustc_macros/src/diagnostics/utils.rs
+++ b/compiler/rustc_macros/src/diagnostics/utils.rs
@@ -708,7 +708,7 @@ impl SubdiagnosticVariant {
                     }
                     if !input.is_empty() { input.parse::()?; }
                     if is_first {
-                        message = Some(Message { message_span: inline_message.span(), value: inline_message.value() });
+                        message = Some(Message { attr_span: attr.span(), message_span: inline_message.span(), value: inline_message.value() });
                         is_first = false;
                     } else {
                         span_err(inline_message.span().unwrap(), "a diagnostic message must be the first argument to the attribute").emit();

From 99c6009c45fd2c1b7f4740b96d9c634d83ad4a5a Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Sat, 7 Feb 2026 10:25:00 +0100
Subject: [PATCH 1474/1843] Reformat existing messages

---
 compiler/rustc_const_eval/src/errors.rs      |  46 +++---
 compiler/rustc_mir_build/src/errors.rs       |  34 ++---
 compiler/rustc_monomorphize/src/errors.rs    |   6 +-
 compiler/rustc_parse/src/errors.rs           |   2 +-
 compiler/rustc_passes/src/errors.rs          |   4 +-
 compiler/rustc_trait_selection/src/errors.rs | 150 ++++++++++---------
 6 files changed, 122 insertions(+), 120 deletions(-)

diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index 3b49dbd907c8..a673e0cb1efb 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -106,9 +106,9 @@ pub(crate) struct UnstableInStableExposed {
     pub is_function_call2: bool,
     #[suggestion(
         "if the {$is_function_call2 ->
-    [true] caller
-    *[false] function
-} is not (yet) meant to be exposed to stable const contexts, add `#[rustc_const_unstable]`",
+            [true] caller
+            *[false] function
+        } is not (yet) meant to be exposed to stable const contexts, add `#[rustc_const_unstable]`",
         code = "#[rustc_const_unstable(feature = \"...\", issue = \"...\")]\n",
         applicability = "has-placeholders"
     )]
@@ -300,11 +300,11 @@ pub(crate) struct UnallowedHeapAllocations {
     #[primary_span]
     #[label(
         r#"allocation not allowed in {$kind ->
-    [const] constant
-    [static] static
-    [const_fn] constant function
-    *[other] {""}
-}s"#
+            [const] constant
+            [static] static
+            [const_fn] constant function
+            *[other] {""}
+        }s"#
     )]
     pub span: Span,
     pub kind: ConstContext,
@@ -539,20 +539,20 @@ pub enum NonConstClosureNote {
     },
     #[note(
         r#"function pointers need an RFC before allowed to be called in {$kind ->
-    [const] constant
-    [static] static
-    [const_fn] constant function
-    *[other] {""}
-}s"#
+            [const] constant
+            [static] static
+            [const_fn] constant function
+            *[other] {""}
+        }s"#
     )]
     FnPtr,
     #[note(
         r#"closures need an RFC before allowed to be called in {$kind ->
-    [const] constant
-    [static] static
-    [const_fn] constant function
-    *[other] {""}
-}s"#
+            [const] constant
+            [static] static
+            [const_fn] constant function
+            *[other] {""}
+        }s"#
     )]
     Closure,
 }
@@ -608,11 +608,11 @@ pub struct LiveDrop<'tcx> {
     #[primary_span]
     #[label(
         r#"the destructor for this type cannot be evaluated in {$kind ->
-    [const] constant
-    [static] static
-    [const_fn] constant function
-    *[other] {""}
-}s"#
+            [const] constant
+            [static] static
+            [const_fn] constant function
+            *[other] {""}
+        }s"#
     )]
     pub span: Span,
     pub kind: ConstContext,
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 2474b3c49a61..fed0435f59a0 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -198,12 +198,12 @@ pub(crate) struct UnsafeOpInUnsafeFnCallToFunctionWithRequiresUnsafe {
     pub(crate) missing_target_features: DiagArgValue,
     pub(crate) missing_target_features_count: usize,
     #[note("the {$build_target_features} target {$build_target_features_count ->
-    [1] feature
-    *[count] features
+        [1] feature
+        *[count] features
     } being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
-    [1] it
-*[count] them
-} in `#[target_feature]`")]
+        [1] it
+        *[count] them
+    } in `#[target_feature]`")]
     pub(crate) note: bool,
     pub(crate) build_target_features: DiagArgValue,
     pub(crate) build_target_features_count: usize,
@@ -532,12 +532,12 @@ pub(crate) struct CallToFunctionWithRequiresUnsafe {
     pub(crate) missing_target_features: DiagArgValue,
     pub(crate) missing_target_features_count: usize,
     #[note("the {$build_target_features} target {$build_target_features_count ->
-    [1] feature
-    *[count] features
-} being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
-    [1] it
-    *[count] them
-} in `#[target_feature]`")]
+        [1] feature
+        *[count] features
+    } being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
+        [1] it
+        *[count] them
+    } in `#[target_feature]`")]
     pub(crate) note: bool,
     pub(crate) build_target_features: DiagArgValue,
     pub(crate) build_target_features_count: usize,
@@ -1264,9 +1264,9 @@ pub(crate) struct InterpretedAsConstSugg {
 pub(crate) enum SuggestLet {
     #[multipart_suggestion(
         "you might want to use `if let` to ignore the {$count ->
-[one] variant that isn't
-*[other] variants that aren't
-} matched",
+            [one] variant that isn't
+            *[other] variants that aren't
+        } matched",
         applicability = "has-placeholders"
     )]
     If {
@@ -1278,9 +1278,9 @@ pub(crate) enum SuggestLet {
     },
     #[suggestion(
         "you might want to use `let...else` to handle the {$count ->
-[one] variant that isn't
-*[other] variants that aren't
-} matched",
+            [one] variant that isn't
+            *[other] variants that aren't
+        } matched",
         code = " else {{ todo!() }}",
         applicability = "has-placeholders"
     )]
diff --git a/compiler/rustc_monomorphize/src/errors.rs b/compiler/rustc_monomorphize/src/errors.rs
index d045ae0b92cb..62a8743873bd 100644
--- a/compiler/rustc_monomorphize/src/errors.rs
+++ b/compiler/rustc_monomorphize/src/errors.rs
@@ -162,9 +162,9 @@ pub(crate) struct AbiRequiredTargetFeature<'a> {
     #[primary_span]
     #[label(
         "function {$is_call ->
-[true] called
-*[false] defined
-} here"
+            [true] called
+            *[false] defined
+        } here"
     )]
     pub span: Span,
     pub required_feature: &'a str,
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index b54782ec592c..fb77af1c1948 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -1151,7 +1151,7 @@ pub(crate) enum MatchArmBodyWithoutBracesSugg {
     #[multipart_suggestion(
         "surround the {$num_statements ->
             [one] statement
-           *[other] statements
+            *[other] statements
         } with a body",
         applicability = "machine-applicable"
     )]
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index 90b4d1b32bf7..f420bba9b4e9 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -1417,8 +1417,8 @@ pub(crate) struct DuplicateEiiImpls {
     pub second_crate: Symbol,
 
     #[note("in addition to these two, { $num_additional_crates ->
-         [one] another implementation was found in crate {$additional_crate_names}
-         *[other] more implementations were also found in the following crates: {$additional_crate_names}
+        [one] another implementation was found in crate {$additional_crate_names}
+        *[other] more implementations were also found in the following crates: {$additional_crate_names}
     }")]
     pub additional_crates: Option<()>,
 
diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs
index c904eb83896d..013e4b522fb5 100644
--- a/compiler/rustc_trait_selection/src/errors.rs
+++ b/compiler/rustc_trait_selection/src/errors.rs
@@ -327,18 +327,18 @@ pub struct InferenceBadError<'a> {
 pub enum SourceKindSubdiag<'a> {
     #[suggestion(
         "{$kind ->
-[with_pattern] consider giving `{$name}` an explicit type
-[closure] consider giving this closure parameter an explicit type
-*[other] consider giving this pattern a type
-}{$x_kind ->
-[has_name] , where the {$prefix_kind ->
-*[type] type for {$prefix}
-[const_with_param] value of const parameter
-[const] value of the constant
-} `{$arg_name}` is specified
-[underscore] , where the placeholders `_` are specified
-*[empty] {\"\"}
-}",
+            [with_pattern] consider giving `{$name}` an explicit type
+            [closure] consider giving this closure parameter an explicit type
+            *[other] consider giving this pattern a type
+        }{$x_kind ->
+            [has_name] , where the {$prefix_kind ->
+                *[type] type for {$prefix}
+                [const_with_param] value of const parameter
+                [const] value of the constant
+            } `{$arg_name}` is specified
+            [underscore] , where the placeholders `_` are specified
+            *[empty] {\"\"}
+        }",
         style = "verbose",
         code = ": {type_name}",
         applicability = "has-placeholders"
@@ -356,15 +356,15 @@ pub enum SourceKindSubdiag<'a> {
     },
     #[label(
         "cannot infer {$is_type ->
-[true] type
-*[false] the value
-} of the {$is_type ->
-[true] type
-*[false] const
-} {$parent_exists ->
-[true] parameter `{$param_name}` declared on the {$parent_prefix} `{$parent_name}`
-*[false] parameter {$param_name}
-}"
+            [true] type
+            *[false] the value
+        } of the {$is_type ->
+            [true] type
+            *[false] const
+        } {$parent_exists ->
+            [true] parameter `{$param_name}` declared on the {$parent_prefix} `{$parent_name}`
+            *[false] parameter {$param_name}
+        }"
     )]
     GenericLabel {
         #[primary_span]
@@ -377,9 +377,9 @@ pub enum SourceKindSubdiag<'a> {
     },
     #[suggestion(
         "consider specifying the generic {$arg_count ->
-[one] argument
-*[other] arguments
-}",
+            [one] argument
+            *[other] arguments
+        }",
         style = "verbose",
         code = "::<{args}>",
         applicability = "has-placeholders"
@@ -945,9 +945,9 @@ impl IntoDiagArg for TyOrSig<'_> {
 #[derive(Subdiagnostic)]
 pub enum ActualImplExplNotes<'tcx> {
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}closure with signature `{$ty_or_sig}` must implement `{$trait_path}`, for any two lifetimes `'{$lifetime_1}` and `'{$lifetime_2}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }closure with signature `{$ty_or_sig}` must implement `{$trait_path}`, for any two lifetimes `'{$lifetime_1}` and `'{$lifetime_2}`...")]
     ExpectedSignatureTwo {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -956,9 +956,9 @@ pub enum ActualImplExplNotes<'tcx> {
         lifetime_2: usize,
     },
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}closure with signature `{$ty_or_sig}` must implement `{$trait_path}`, for any lifetime `'{$lifetime_1}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }closure with signature `{$ty_or_sig}` must implement `{$trait_path}`, for any lifetime `'{$lifetime_1}`...")]
     ExpectedSignatureAny {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -966,9 +966,9 @@ pub enum ActualImplExplNotes<'tcx> {
         lifetime_1: usize,
     },
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}closure with signature `{$ty_or_sig}` must implement `{$trait_path}`, for some specific lifetime `'{$lifetime_1}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }closure with signature `{$ty_or_sig}` must implement `{$trait_path}`, for some specific lifetime `'{$lifetime_1}`...")]
     ExpectedSignatureSome {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -977,9 +977,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}closure with signature `{$ty_or_sig}` must implement `{$trait_path}`"
+            [true] ...
+            *[false] {\"\"}
+        }closure with signature `{$ty_or_sig}` must implement `{$trait_path}`"
     )]
     ExpectedSignatureNothing {
         leading_ellipsis: bool,
@@ -987,9 +987,9 @@ pub enum ActualImplExplNotes<'tcx> {
         trait_path: Highlighted<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
     },
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`, for any two lifetimes `'{$lifetime_1}` and `'{$lifetime_2}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`, for any two lifetimes `'{$lifetime_1}` and `'{$lifetime_2}`...")]
     ExpectedPassiveTwo {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -998,9 +998,9 @@ pub enum ActualImplExplNotes<'tcx> {
         lifetime_2: usize,
     },
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`, for any lifetime `'{$lifetime_1}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`, for any lifetime `'{$lifetime_1}`...")]
     ExpectedPassiveAny {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -1008,9 +1008,9 @@ pub enum ActualImplExplNotes<'tcx> {
         lifetime_1: usize,
     },
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`, for some specific lifetime `'{$lifetime_1}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`, for some specific lifetime `'{$lifetime_1}`...")]
     ExpectedPassiveSome {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -1019,9 +1019,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`"
+            [true] ...
+            *[false] {\"\"}
+        }`{$trait_path}` would have to be implemented for the type `{$ty_or_sig}`"
     )]
     ExpectedPassiveNothing {
         leading_ellipsis: bool,
@@ -1029,9 +1029,9 @@ pub enum ActualImplExplNotes<'tcx> {
         trait_path: Highlighted<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
     },
     #[note("{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$ty_or_sig}` must implement `{$trait_path}`, for any two lifetimes `'{$lifetime_1}` and `'{$lifetime_2}`...")]
+        [true] ...
+        *[false] {\"\"}
+    }`{$ty_or_sig}` must implement `{$trait_path}`, for any two lifetimes `'{$lifetime_1}` and `'{$lifetime_2}`...")]
     ExpectedOtherTwo {
         leading_ellipsis: bool,
         ty_or_sig: TyOrSig<'tcx>,
@@ -1041,9 +1041,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$ty_or_sig}` must implement `{$trait_path}`, for any lifetime `'{$lifetime_1}`..."
+            [true] ...
+            *[false] {\"\"}
+        }`{$ty_or_sig}` must implement `{$trait_path}`, for any lifetime `'{$lifetime_1}`..."
     )]
     ExpectedOtherAny {
         leading_ellipsis: bool,
@@ -1053,9 +1053,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$ty_or_sig}` must implement `{$trait_path}`, for some specific lifetime `'{$lifetime_1}`..."
+            [true] ...
+            *[false] {\"\"}
+        }`{$ty_or_sig}` must implement `{$trait_path}`, for some specific lifetime `'{$lifetime_1}`..."
     )]
     ExpectedOtherSome {
         leading_ellipsis: bool,
@@ -1065,9 +1065,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "{$leading_ellipsis ->
-[true] ...
-*[false] {\"\"}
-}`{$ty_or_sig}` must implement `{$trait_path}`"
+            [true] ...
+            *[false] {\"\"}
+        }`{$ty_or_sig}` must implement `{$trait_path}`"
     )]
     ExpectedOtherNothing {
         leading_ellipsis: bool,
@@ -1076,9 +1076,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "...but it actually implements `{$trait_path}`{$has_lifetime ->
-[true] , for some specific lifetime `'{$lifetime}`
-*[false] {\"\"}
-}"
+            [true] , for some specific lifetime `'{$lifetime}`
+            *[false] {\"\"}
+        }"
     )]
     ButActuallyImplementsTrait {
         trait_path: Highlighted<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
@@ -1087,9 +1087,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "...but `{$trait_path}` is actually implemented for the type `{$ty}`{$has_lifetime ->
-[true] , for some specific lifetime `'{$lifetime}`
-*[false] {\"\"}
-}"
+            [true] , for some specific lifetime `'{$lifetime}`
+            *[false] {\"\"}
+        }"
     )]
     ButActuallyImplementedForTy {
         trait_path: Highlighted<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
@@ -1099,9 +1099,9 @@ pub enum ActualImplExplNotes<'tcx> {
     },
     #[note(
         "...but `{$ty}` actually implements `{$trait_path}`{$has_lifetime ->
-[true] , for some specific lifetime `'{$lifetime}`
-*[false] {\"\"}
-}"
+            [true] , for some specific lifetime `'{$lifetime}`
+            *[false] {\"\"}
+        }"
     )]
     ButActuallyTyImplements {
         trait_path: Highlighted<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
@@ -1246,7 +1246,7 @@ pub struct TraitImplDiff {
     pub trait_sp: Span,
     #[note(
         "expected signature `{$expected}`
-               {\"   \"}found signature `{$found}`"
+        {\"   \"}found signature `{$found}`"
     )]
     pub note: (),
     #[subdiagnostic]
@@ -1940,10 +1940,12 @@ pub enum ObligationCauseFailureCode {
         #[primary_span]
         span: Span,
     },
-    #[diag("{$lang_item_name ->
-[panic_impl] `#[panic_handler]`
-*[lang_item_name] lang item `{$lang_item_name}`
-} function has wrong type", code = E0308)]
+    #[diag(
+        "{$lang_item_name ->
+            [panic_impl] `#[panic_handler]`
+            *[lang_item_name] lang item `{$lang_item_name}`
+        } function has wrong type"
+    , code = E0308)]
     FnLangCorrectType {
         #[primary_span]
         span: Span,

From d3d4fd9312da82cbb0b17a597f458b282febfea7 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Mon, 9 Feb 2026 10:14:40 +1100
Subject: [PATCH 1475/1843] Tweak query key trait bounds.

Query keys must be stably hashable. Currently this requirement is
expressed as a where-clause on `impl QueryDispatcher for
SemiDynamicQueryDispatcher` and a where-clause on
`create_deferred_query_stack_frame`.

This commit removes those where-clause bounds and adds a single bound to
`QueryCache::Key`, which already has some other bounds. I.e. it
consolidates the bounds. It also gives them a name (`QueryCacheKey`) to
avoid repeating them. There is also a related `Key` trait in
`rustc_middle`; it should probably be merged with `QueryCacheKey` in the
future, but not today.

This cleanup helps with the next two commits, which do bigger
rearrangements, and where the where-clauses caused me some difficulties.
---
 compiler/rustc_query_impl/src/lib.rs            |  4 ----
 compiler/rustc_query_impl/src/plumbing.rs       |  3 +--
 compiler/rustc_query_system/src/lib.rs          |  1 +
 compiler/rustc_query_system/src/query/caches.rs | 11 ++++++++---
 compiler/rustc_query_system/src/query/mod.rs    |  4 +++-
 5 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 228d0070b55b..7e12365fc2ac 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -10,7 +10,6 @@
 
 use std::marker::ConstParamTy;
 
-use rustc_data_structures::stable_hasher::HashStable;
 use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::{self, DepKind, DepKindVTable, DepNodeIndex};
@@ -23,7 +22,6 @@ use rustc_middle::query::plumbing::{QuerySystem, QuerySystemFns, QueryVTable};
 use rustc_middle::query::values::Value;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
-use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{
     CycleError, CycleErrorHandling, HashResult, QueryCache, QueryDispatcher, QueryMap, QueryMode,
     QueryState,
@@ -78,8 +76,6 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> Clone
 // This is `impl QueryDispatcher for SemiDynamicQueryDispatcher`.
 impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> QueryDispatcher<'tcx>
     for SemiDynamicQueryDispatcher<'tcx, C, FLAGS>
-where
-    for<'a> C::Key: HashStable>,
 {
     type Qcx = QueryCtxt<'tcx>;
     type Key = C::Key;
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 676d01be3385..dd8ea35019a4 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -27,7 +27,6 @@ use rustc_middle::ty::print::with_reduced_queries;
 use rustc_middle::ty::tls::{self, ImplicitCtxt};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::{DepNodeKey, FingerprintStyle, HasDepContext};
-use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{
     QueryCache, QueryContext, QueryDispatcher, QueryJobId, QueryMap, QuerySideEffect,
     QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra,
@@ -360,7 +359,7 @@ pub(crate) fn create_deferred_query_stack_frame<'tcx, Cache>(
 ) -> QueryStackFrame>
 where
     Cache: QueryCache,
-    Cache::Key: Key + DynSend + DynSync + for<'a> HashStable> + 'tcx,
+    Cache::Key: Key + DynSend + DynSync,
 {
     let kind = vtable.dep_kind;
 
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index 7f6426e395f9..fc63df3595c4 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -2,6 +2,7 @@
 #![allow(internal_features)]
 #![feature(assert_matches)]
 #![feature(min_specialization)]
+#![feature(trait_alias)]
 // tidy-alphabetical-end
 
 pub mod dep_graph;
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index 67ad767d4d31..c1f5e5b67085 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -3,12 +3,17 @@ use std::hash::Hash;
 use std::sync::OnceLock;
 
 use rustc_data_structures::sharded::ShardedHashMap;
+use rustc_data_structures::stable_hasher::HashStable;
 pub use rustc_data_structures::vec_cache::VecCache;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_index::Idx;
 use rustc_span::def_id::{DefId, DefIndex};
 
 use crate::dep_graph::DepNodeIndex;
+use crate::ich::StableHashingContext;
+
+/// Traits that all query keys must satisfy.
+pub trait QueryCacheKey = Hash + Eq + Copy + Debug + for<'a> HashStable>;
 
 /// Trait for types that serve as an in-memory cache for query results,
 /// for a given key (argument) type and value (return) type.
@@ -16,7 +21,7 @@ use crate::dep_graph::DepNodeIndex;
 /// Types implementing this trait are associated with actual key/value types
 /// by the `Cache` associated type of the `rustc_middle::query::Key` trait.
 pub trait QueryCache: Sized {
-    type Key: Hash + Eq + Copy + Debug;
+    type Key: QueryCacheKey;
     type Value: Copy;
 
     /// Returns the cached value (and other information) associated with the
@@ -48,7 +53,7 @@ impl Default for DefaultCache {
 
 impl QueryCache for DefaultCache
 where
-    K: Eq + Hash + Copy + Debug,
+    K: QueryCacheKey,
     V: Copy,
 {
     type Key = K;
@@ -175,7 +180,7 @@ where
 
 impl QueryCache for VecCache
 where
-    K: Idx + Eq + Hash + Copy + Debug,
+    K: Idx + QueryCacheKey,
     V: Copy,
 {
     type Key = K;
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 29a572b726f0..01bb6ff09fa1 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -12,7 +12,9 @@ use rustc_macros::{Decodable, Encodable};
 use rustc_span::Span;
 use rustc_span::def_id::DefId;
 
-pub use self::caches::{DefIdCache, DefaultCache, QueryCache, SingleCache, VecCache};
+pub use self::caches::{
+    DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache,
+};
 pub use self::dispatcher::{HashResult, QueryDispatcher};
 pub use self::job::{
     QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, break_query_cycles,

From 52caa7ae6d06a43f9fb07d37ba39c233e9273455 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Mon, 9 Feb 2026 10:38:44 +1100
Subject: [PATCH 1476/1843] Remove `QueryDispatcherUnerased::Dispatcher`
 associated type.

It's always `SemiDynamicQueryDispatcher`, so we can just use that type
directly. (This requires adding some explicit generic params to
`QueryDispatcherUnerased`.) Less indirection makes the code clearer, and
this is a prerequisite for the next commit, which is a much bigger
simplification.
---
 compiler/rustc_query_impl/src/lib.rs      |  9 ++---
 compiler/rustc_query_impl/src/plumbing.rs | 44 ++++++++++++++---------
 2 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 7e12365fc2ac..327a8f4bc6c4 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -207,17 +207,14 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> QueryDispatcher<'tcx>
 ///
 /// There is one macro-generated implementation of this trait for each query,
 /// on the type `rustc_query_impl::query_impl::$name::QueryType`.
-trait QueryDispatcherUnerased<'tcx> {
+trait QueryDispatcherUnerased<'tcx, C: QueryCache, const FLAGS: QueryFlags> {
     type UnerasedValue;
-    type Dispatcher: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>;
 
     const NAME: &'static &'static str;
 
-    fn query_dispatcher(tcx: TyCtxt<'tcx>) -> Self::Dispatcher;
+    fn query_dispatcher(tcx: TyCtxt<'tcx>) -> SemiDynamicQueryDispatcher<'tcx, C, FLAGS>;
 
-    fn restore_val(
-        value: >::Value,
-    ) -> Self::UnerasedValue;
+    fn restore_val(value: C::Value) -> Self::UnerasedValue;
 }
 
 pub fn query_system<'tcx>(
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index dd8ea35019a4..bbda716eadb0 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -34,9 +34,9 @@ use rustc_query_system::query::{
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::def_id::LOCAL_CRATE;
 
-use crate::QueryDispatcherUnerased;
 use crate::error::{QueryOverflow, QueryOverflowNote};
 use crate::execution::{all_inactive, force_query};
+use crate::{QueryDispatcherUnerased, QueryFlags, SemiDynamicQueryDispatcher};
 
 /// Implements [`QueryContext`] for use by [`rustc_query_system`], since that
 /// crate does not have direct access to [`TyCtxt`].
@@ -377,13 +377,13 @@ where
     QueryStackFrame::new(info, kind, hash, def_id, def_id_for_ty_in_cycle)
 }
 
-pub(crate) fn encode_query_results<'a, 'tcx, Q>(
-    query: Q::Dispatcher,
+pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
 ) where
-    Q: QueryDispatcherUnerased<'tcx>,
+    Q: QueryDispatcherUnerased<'tcx, C, FLAGS>,
     Q::UnerasedValue: Encodable>,
 {
     let _timer = qcx.tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name());
@@ -506,17 +506,22 @@ where
     }
 }
 
-pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q>(
+pub(crate) fn make_dep_kind_vtable_for_query<
+    'tcx,
+    Q,
+    C: QueryCache + 'tcx,
+    const FLAGS: QueryFlags,
+>(
     is_anon: bool,
     is_eval_always: bool,
 ) -> DepKindVTable<'tcx>
 where
-    Q: QueryDispatcherUnerased<'tcx>,
+    Q: QueryDispatcherUnerased<'tcx, C, FLAGS>,
 {
     let fingerprint_style = if is_anon {
         FingerprintStyle::Opaque
     } else {
-        ::Key::fingerprint_style()
+        >>::fingerprint_style()
     };
 
     if is_anon || !fingerprint_style.reconstructible() {
@@ -712,25 +717,26 @@ macro_rules! define_queries {
                 is_feedable: feedable!([$($modifiers)*]),
             };
 
-            impl<'tcx> QueryDispatcherUnerased<'tcx> for QueryType<'tcx> {
+            impl<'tcx> QueryDispatcherUnerased<'tcx, queries::$name::Storage<'tcx>, FLAGS>
+                for QueryType<'tcx>
+            {
                 type UnerasedValue = queries::$name::Value<'tcx>;
-                type Dispatcher = SemiDynamicQueryDispatcher<
-                    'tcx,
-                    queries::$name::Storage<'tcx>,
-                    FLAGS,
-                >;
 
                 const NAME: &'static &'static str = &stringify!($name);
 
                 #[inline(always)]
-                fn query_dispatcher(tcx: TyCtxt<'tcx>) -> Self::Dispatcher {
+                fn query_dispatcher(tcx: TyCtxt<'tcx>)
+                    -> SemiDynamicQueryDispatcher<'tcx, queries::$name::Storage<'tcx>, FLAGS>
+                {
                     SemiDynamicQueryDispatcher {
                         vtable: &tcx.query_system.query_vtables.$name,
                     }
                 }
 
                 #[inline(always)]
-                fn restore_val(value: >::Value) -> Self::UnerasedValue {
+                fn restore_val(value:  as QueryCache>::Value)
+                    -> Self::UnerasedValue
+                {
                     erase::restore_val::>(value)
                 }
             }
@@ -786,7 +792,11 @@ macro_rules! define_queries {
                     encoder: &mut CacheEncoder<'_, 'tcx>,
                     query_result_index: &mut EncodedDepNodeIndex
                 ) {
-                    $crate::plumbing::encode_query_results::>(
+                    $crate::plumbing::encode_query_results::<
+                        query_impl::$name::QueryType<'tcx>,
+                        _,
+                        _
+                    > (
                         query_impl::$name::QueryType::query_dispatcher(tcx),
                         QueryCtxt::new(tcx),
                         encoder,
@@ -958,7 +968,7 @@ macro_rules! define_queries {
 
             $(pub(crate) fn $name<'tcx>() -> DepKindVTable<'tcx> {
                 use $crate::query_impl::$name::QueryType;
-                $crate::plumbing::make_dep_kind_vtable_for_query::>(
+                $crate::plumbing::make_dep_kind_vtable_for_query::, _, _>(
                     is_anon!([$($modifiers)*]),
                     is_eval_always!([$($modifiers)*]),
                 )

From 1fa165deeeff28e9b8b4a2de16b92c0844059f69 Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan 
Date: Sun, 8 Feb 2026 22:01:24 -0500
Subject: [PATCH 1477/1843] style: remove unneeded trailing commas

Make format-like macro calls look similar to what `cargo fmt` does automatically - remove trailing commas. When removing a comma, I also inlined some variables for consistency and clarity.
---
 .../infer/nice_region_error/find_anon_type.rs             | 4 ++--
 .../infer/nice_region_error/static_impl_trait.rs          | 6 +++---
 .../src/error_reporting/traits/ambiguity.rs               | 6 +++---
 .../src/error_reporting/traits/fulfillment_errors.rs      | 2 +-
 .../src/error_reporting/traits/on_unimplemented.rs        | 2 +-
 .../src/traits/select/candidate_assembly.rs               | 8 ++++----
 compiler/rustc_trait_selection/src/traits/select/mod.rs   | 2 +-
 7 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/find_anon_type.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/find_anon_type.rs
index 8fe4ffebd865..7d061e65df80 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/find_anon_type.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/find_anon_type.rs
@@ -190,8 +190,8 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
             }
 
             Some(rbv::ResolvedArg::LateBound(debruijn_index, _, id)) => {
-                debug!("FindNestedTypeVisitor::visit_ty: LateBound depth = {:?}", debruijn_index,);
-                debug!("id={:?}", id);
+                debug!("FindNestedTypeVisitor::visit_ty: LateBound depth = {debruijn_index:?}");
+                debug!("id={id:?}");
                 if debruijn_index == self.current_index && id.to_def_id() == self.region_def_id {
                     return ControlFlow::Break(());
                 }
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs
index 3ee6e6b739c0..3ed1f7c3481f 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/nice_region_error/static_impl_trait.rs
@@ -239,14 +239,14 @@ pub fn suggest_new_region_bound(
                         };
                         spans_suggs.push((fn_return.span.shrink_to_hi(), format!(" + {name} ")));
                         err.multipart_suggestion_verbose(
-                            format!("{declare} `{ty}` {captures}, {use_lt}",),
+                            format!("{declare} `{ty}` {captures}, {use_lt}"),
                             spans_suggs,
                             Applicability::MaybeIncorrect,
                         );
                     } else {
                         err.span_suggestion_verbose(
                             fn_return.span.shrink_to_hi(),
-                            format!("{declare} `{ty}` {captures}, {explicit}",),
+                            format!("{declare} `{ty}` {captures}, {explicit}"),
                             &plus_lt,
                             Applicability::MaybeIncorrect,
                         );
@@ -257,7 +257,7 @@ pub fn suggest_new_region_bound(
                 if let LifetimeKind::ImplicitObjectLifetimeDefault = lt.kind {
                     err.span_suggestion_verbose(
                         fn_return.span.shrink_to_hi(),
-                        format!("{declare} the trait object {captures}, {explicit}",),
+                        format!("{declare} the trait object {captures}, {explicit}"),
                         &plus_lt,
                         Applicability::MaybeIncorrect,
                     );
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs
index eb72f71382ef..16c4ac68cdf0 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs
@@ -710,7 +710,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             predicate
         );
         let post = if post.len() > 1 || (post.len() == 1 && post[0].contains('\n')) {
-            format!(":\n{}", post.iter().map(|p| format!("- {p}")).collect::>().join("\n"),)
+            format!(":\n{}", post.iter().map(|p| format!("- {p}")).collect::>().join("\n"))
         } else if post.len() == 1 {
             format!(": `{}`", post[0])
         } else {
@@ -722,7 +722,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 err.note(format!("cannot satisfy `{predicate}`"));
             }
             (0, _, 1) => {
-                err.note(format!("{} in the `{}` crate{}", msg, crates[0], post,));
+                err.note(format!("{msg} in the `{}` crate{post}", crates[0]));
             }
             (0, _, _) => {
                 err.note(format!(
@@ -739,7 +739,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             (_, 1, 1) => {
                 let span: MultiSpan = spans.into();
                 err.span_note(span, msg);
-                err.note(format!("and another `impl` found in the `{}` crate{}", crates[0], post,));
+                err.note(format!("and another `impl` found in the `{}` crate{post}", crates[0]));
             }
             _ => {
                 let span: MultiSpan = spans.into();
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
index 3cb3224302b6..18597e001c9f 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
@@ -2241,7 +2241,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 err.highlighted_span_help(
                     self.tcx.def_span(def_id),
                     vec![
-                        StringPart::normal(format!("the trait `{trait_}` ",)),
+                        StringPart::normal(format!("the trait `{trait_}` ")),
                         StringPart::highlighted("is"),
                         StringPart::normal(desc),
                         StringPart::highlighted(self_ty),
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs
index 5eff7bba7c63..a98f952d55a3 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs
@@ -888,7 +888,7 @@ impl<'tcx> OnUnimplementedFormatString {
                     }
                 } else {
                     let reported =
-                        struct_span_code_err!(tcx.dcx(), self.span, E0231, "{}", e.description,)
+                        struct_span_code_err!(tcx.dcx(), self.span, E0231, "{}", e.description)
                             .emit();
                     result = Err(reported);
                 }
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index f5bf74a79919..935834b832f2 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -312,7 +312,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             // `async`/`gen` constructs get lowered to a special kind of coroutine that
             // should *not* `impl Coroutine`.
             ty::Coroutine(did, ..) if self.tcx().is_general_coroutine(*did) => {
-                debug!(?self_ty, ?obligation, "assemble_coroutine_candidates",);
+                debug!(?self_ty, ?obligation, "assemble_coroutine_candidates");
 
                 candidates.vec.push(CoroutineCandidate);
             }
@@ -334,7 +334,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             // async constructs get lowered to a special kind of coroutine that
             // should directly `impl Future`.
             if self.tcx().coroutine_is_async(*did) {
-                debug!(?self_ty, ?obligation, "assemble_future_candidates",);
+                debug!(?self_ty, ?obligation, "assemble_future_candidates");
 
                 candidates.vec.push(FutureCandidate);
             }
@@ -352,7 +352,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         if let ty::Coroutine(did, ..) = self_ty.kind()
             && self.tcx().coroutine_is_gen(*did)
         {
-            debug!(?self_ty, ?obligation, "assemble_iterator_candidates",);
+            debug!(?self_ty, ?obligation, "assemble_iterator_candidates");
 
             candidates.vec.push(IteratorCandidate);
         }
@@ -378,7 +378,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             // gen constructs get lowered to a special kind of coroutine that
             // should directly `impl AsyncIterator`.
             if self.tcx().coroutine_is_async_gen(did) {
-                debug!(?self_ty, ?obligation, "assemble_iterator_candidates",);
+                debug!(?self_ty, ?obligation, "assemble_iterator_candidates");
 
                 // Can only confirm this candidate if we have constrained
                 // the `Yield` type to at least `Poll>`..
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 787dd4ea6254..2233fb4f90e4 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -1223,7 +1223,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     && self.match_fresh_trait_preds(stack.fresh_trait_pred, prev.fresh_trait_pred)
             })
         {
-            debug!("evaluate_stack --> unbound argument, recursive --> giving up",);
+            debug!("evaluate_stack --> unbound argument, recursive --> giving up");
             return Ok(EvaluatedToAmbigStackDependent);
         }
 

From 81fb703664280626c03428c384695d766be10031 Mon Sep 17 00:00:00 2001
From: Weixie Cui 
Date: Sun, 8 Feb 2026 11:44:50 +0800
Subject: [PATCH 1478/1843] Fix copy-paste bug: use sub_trace.cause instead of
 sup_trace.cause in report_sub_sup_conflict

In `report_sub_sup_conflict`, when calling `values_str` for
`sub_trace.values`, the code was incorrectly passing `sup_trace.cause`
instead of `sub_trace.cause`. This is a copy-paste error from the
preceding line which correctly uses `sup_trace.cause` for
`sup_trace.values`.

The `cause` parameter matters for `ValuePairs::PolySigs` comparisons,
where `values_str` inspects `cause.code()` to extract
`impl_item_def_id` and `trait_item_def_id` for richer function
signature diagnostics. Using the wrong trace's cause could produce
incorrect function def IDs, leading to misleading diagnostic output
when the sub and sup traces originate from different obligations.

Even for non-PolySigs variants where `cause` is currently unused by
`values_str`, passing the semantically correct cause is the right
thing to do for correctness and maintainability.
---
 .../rustc_trait_selection/src/error_reporting/infer/region.rs   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
index d15fb40dd1cb..81cca3dd67ac 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
@@ -1020,7 +1020,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             && let Some((sup_expected, sup_found)) =
                 self.values_str(sup_trace.values, &sup_trace.cause, err.long_ty_path())
             && let Some((sub_expected, sub_found)) =
-                self.values_str(sub_trace.values, &sup_trace.cause, err.long_ty_path())
+                self.values_str(sub_trace.values, &sub_trace.cause, err.long_ty_path())
             && sub_expected == sup_expected
             && sub_found == sup_found
         {

From f394d1e6c617439e027fe2fae2b9d581709639db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lauren=C8=9Biu=20Nicola?= 
Date: Mon, 9 Feb 2026 07:21:53 +0200
Subject: [PATCH 1479/1843] Fix lockfile

---
 src/tools/rust-analyzer/Cargo.lock | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/src/tools/rust-analyzer/Cargo.lock b/src/tools/rust-analyzer/Cargo.lock
index 18c53986a012..755ae55eea46 100644
--- a/src/tools/rust-analyzer/Cargo.lock
+++ b/src/tools/rust-analyzer/Cargo.lock
@@ -3754,15 +3754,3 @@ dependencies = [
  "memchr",
  "time",
 ]
-
-[[patch.unused]]
-name = "salsa"
-version = "0.25.2"
-
-[[patch.unused]]
-name = "salsa-macro-rules"
-version = "0.25.2"
-
-[[patch.unused]]
-name = "salsa-macros"
-version = "0.25.2"

From 3b705eae4fe9f16efd8193fc5dc822fc9e66e5da Mon Sep 17 00:00:00 2001
From: xizheyin 
Date: Mon, 9 Feb 2026 14:24:59 +0800
Subject: [PATCH 1480/1843] Add ui test insufficient-suggestion-issue-141679.rs

---
 .../insufficient-suggestion-issue-141679.rs           |  8 ++++++++
 .../insufficient-suggestion-issue-141679.stderr       | 11 +++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs
 create mode 100644 tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr

diff --git a/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs
new file mode 100644
index 000000000000..daf8693b5264
--- /dev/null
+++ b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs
@@ -0,0 +1,8 @@
+use std::rc::Rc;
+pub struct Foo;
+
+pub type Function = Rc;
+
+impl Function {}
+//~^ ERROR cannot define inherent `impl` for a type outside of the crate where the type is defined [E0116]
+fn main(){}
\ No newline at end of file
diff --git a/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr
new file mode 100644
index 000000000000..cbb94ea4d367
--- /dev/null
+++ b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr
@@ -0,0 +1,11 @@
+error[E0116]: cannot define inherent `impl` for a type outside of the crate where the type is defined
+  --> $DIR/insufficient-suggestion-issue-141679.rs:6:1
+   |
+LL | impl Function {}
+   | ^^^^^^^^^^^^^ impl for type defined outside of crate
+   |
+   = note: define and implement a trait or new type instead
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0116`.

From 91a612620822993435a49c288256c1028351a236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= 
Date: Mon, 9 Feb 2026 07:36:48 +0100
Subject: [PATCH 1481/1843] Use `scope` for `par_slice` instead of `join`

---
 .../src/sync/parallel.rs                      | 41 +++++++------------
 1 file changed, 14 insertions(+), 27 deletions(-)

diff --git a/compiler/rustc_data_structures/src/sync/parallel.rs b/compiler/rustc_data_structures/src/sync/parallel.rs
index b515c0bee8a6..552d02b8b91d 100644
--- a/compiler/rustc_data_structures/src/sync/parallel.rs
+++ b/compiler/rustc_data_structures/src/sync/parallel.rs
@@ -138,34 +138,21 @@ fn par_slice(
     guard: &ParallelGuard,
     for_each: impl Fn(&mut I) + DynSync + DynSend,
 ) {
-    struct State<'a, F> {
-        for_each: FromDyn,
-        guard: &'a ParallelGuard,
-        group: usize,
-    }
-
-    fn par_rec(
-        items: &mut [I],
-        state: &State<'_, F>,
-    ) {
-        if items.len() <= state.group {
-            for item in items {
-                state.guard.run(|| (state.for_each)(item));
-            }
-        } else {
-            let (left, right) = items.split_at_mut(items.len() / 2);
-            let mut left = state.for_each.derive(left);
-            let mut right = state.for_each.derive(right);
-            rustc_thread_pool::join(move || par_rec(*left, state), move || par_rec(*right, state));
+    let for_each = FromDyn::from(for_each);
+    let mut items = for_each.derive(items);
+    rustc_thread_pool::scope(|s| {
+        let proof = items.derive(());
+        let group_size = std::cmp::max(items.len() / 128, 1);
+        for group in items.chunks_exact_mut(group_size) {
+            let group = proof.derive(group);
+            s.spawn(|_| {
+                let mut group = group;
+                for i in group.iter_mut() {
+                    guard.run(|| for_each(i));
+                }
+            });
         }
-    }
-
-    let state = State {
-        for_each: FromDyn::from(for_each),
-        guard,
-        group: std::cmp::max(items.len() / 128, 1),
-    };
-    par_rec(items, &state)
+    });
 }
 
 pub fn par_for_each_in>(

From f24839539428bca14becbf813d0352e1aff17c18 Mon Sep 17 00:00:00 2001
From: Adwin White 
Date: Sun, 8 Feb 2026 17:00:47 +0800
Subject: [PATCH 1482/1843] bless tests

---
 tests/crashes/137916.rs                       | 13 ----
 tests/crashes/138274.rs                       | 18 ------
 tests/ui/async-await/issue-70818.rs           |  3 +-
 tests/ui/async-await/issue-70818.stderr       | 23 +------
 .../async-await/issue-70935-complex-spans.rs  |  1 -
 .../issue-70935-complex-spans.stderr          | 47 +-------------
 tests/ui/coroutine/issue-105084.rs            |  1 -
 tests/ui/coroutine/issue-105084.stderr        | 29 +--------
 .../stalled-coroutine-obligations.rs          | 32 ++++++++++
 .../stalled-coroutine-obligations.stderr      | 40 ++++++++++++
 .../stalled-coroutine-obligations.rs          | 50 +++++++++++++++
 .../stalled-coroutine-obligations.stderr      | 62 +++++++++++++++++++
 12 files changed, 190 insertions(+), 129 deletions(-)
 delete mode 100644 tests/crashes/137916.rs
 delete mode 100644 tests/crashes/138274.rs
 create mode 100644 tests/ui/coroutine/stalled-coroutine-obligations.rs
 create mode 100644 tests/ui/coroutine/stalled-coroutine-obligations.stderr
 create mode 100644 tests/ui/traits/next-solver/stalled-coroutine-obligations.rs
 create mode 100644 tests/ui/traits/next-solver/stalled-coroutine-obligations.stderr

diff --git a/tests/crashes/137916.rs b/tests/crashes/137916.rs
deleted file mode 100644
index b25e7b200d95..000000000000
--- a/tests/crashes/137916.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-//@ known-bug: #137916
-//@ edition: 2021
-use std::ptr::null;
-
-async fn a() -> Box {
-    Box::new(async {
-        let non_send = null::<()>();
-        &non_send;
-        async {}.await
-    })
-}
-
-fn main() {}
diff --git a/tests/crashes/138274.rs b/tests/crashes/138274.rs
deleted file mode 100644
index d657b27e46f3..000000000000
--- a/tests/crashes/138274.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-//@ known-bug: #138274
-//@ edition: 2021
-//@ compile-flags: --crate-type=lib
-trait Trait {}
-
-fn foo() -> Box {
-    todo!()
-}
-
-fn fetch() {
-    async {
-        let fut = async {
-            let _x = foo();
-            async {}.await;
-        };
-        let _: Box = Box::new(fut);
-    };
-}
diff --git a/tests/ui/async-await/issue-70818.rs b/tests/ui/async-await/issue-70818.rs
index c11332fe7d84..af9f33d9050b 100644
--- a/tests/ui/async-await/issue-70818.rs
+++ b/tests/ui/async-await/issue-70818.rs
@@ -2,9 +2,8 @@
 
 use std::future::Future;
 fn foo(ty: T, ty1: U) -> impl Future + Send {
-    //~^ ERROR future cannot be sent between threads safely
+    //~^ ERROR: future cannot be sent between threads safely
     async { (ty, ty1) }
-    //~^ ERROR future cannot be sent between threads safely
 }
 
 fn main() {}
diff --git a/tests/ui/async-await/issue-70818.stderr b/tests/ui/async-await/issue-70818.stderr
index 07fd20cdd77e..8de6a825042b 100644
--- a/tests/ui/async-await/issue-70818.stderr
+++ b/tests/ui/async-await/issue-70818.stderr
@@ -1,24 +1,3 @@
-error: future cannot be sent between threads safely
-  --> $DIR/issue-70818.rs:6:5
-   |
-LL |     async { (ty, ty1) }
-   |     ^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send`
-   |
-note: captured value is not `Send`
-  --> $DIR/issue-70818.rs:6:18
-   |
-LL |     async { (ty, ty1) }
-   |                  ^^^ has type `U` which is not `Send`
-note: required by a bound in an opaque type
-  --> $DIR/issue-70818.rs:4:69
-   |
-LL | fn foo(ty: T, ty1: U) -> impl Future + Send {
-   |                                                                     ^^^^
-help: consider restricting type parameter `U` with trait `Send`
-   |
-LL | fn foo(ty: T, ty1: U) -> impl Future + Send {
-   |                  +++++++++++++++++++
-
 error: future cannot be sent between threads safely
   --> $DIR/issue-70818.rs:4:38
    |
@@ -35,5 +14,5 @@ help: consider restricting type parameter `U` with trait `Send`
 LL | fn foo(ty: T, ty1: U) -> impl Future + Send {
    |                  +++++++++++++++++++
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
diff --git a/tests/ui/async-await/issue-70935-complex-spans.rs b/tests/ui/async-await/issue-70935-complex-spans.rs
index 2851637ae78f..27826f79dcd9 100644
--- a/tests/ui/async-await/issue-70935-complex-spans.rs
+++ b/tests/ui/async-await/issue-70935-complex-spans.rs
@@ -15,7 +15,6 @@ async fn baz(_c: impl FnMut() -> T) where T: Future {
 fn foo(x: NotSync) -> impl Future + Send {
     //~^ ERROR `*mut ()` cannot be shared between threads safely
     async move {
-    //~^ ERROR `*mut ()` cannot be shared between threads safely
         baz(|| async {
             foo(x.clone());
         }).await;
diff --git a/tests/ui/async-await/issue-70935-complex-spans.stderr b/tests/ui/async-await/issue-70935-complex-spans.stderr
index 31d15c459217..b9180c620b0d 100644
--- a/tests/ui/async-await/issue-70935-complex-spans.stderr
+++ b/tests/ui/async-await/issue-70935-complex-spans.stderr
@@ -1,46 +1,3 @@
-error[E0277]: `*mut ()` cannot be shared between threads safely
-  --> $DIR/issue-70935-complex-spans.rs:17:5
-   |
-LL | /     async move {
-LL | |
-LL | |         baz(|| async {
-LL | |             foo(x.clone());
-LL | |         }).await;
-LL | |     }
-   | |_____^ `*mut ()` cannot be shared between threads safely
-   |
-   = help: within `NotSync`, the trait `Sync` is not implemented for `*mut ()`
-note: required because it appears within the type `PhantomData<*mut ()>`
-  --> $SRC_DIR/core/src/marker.rs:LL:COL
-note: required because it appears within the type `NotSync`
-  --> $DIR/issue-70935-complex-spans.rs:9:8
-   |
-LL | struct NotSync(PhantomData<*mut ()>);
-   |        ^^^^^^^
-   = note: required for `&NotSync` to implement `Send`
-note: required because it's used within this closure
-  --> $DIR/issue-70935-complex-spans.rs:19:13
-   |
-LL |         baz(|| async {
-   |             ^^
-note: required because it's used within this `async` fn body
-  --> $DIR/issue-70935-complex-spans.rs:12:67
-   |
-LL |   async fn baz(_c: impl FnMut() -> T) where T: Future {
-   |  ___________________________________________________________________^
-LL | | }
-   | |_^
-note: required because it's used within this `async` block
-  --> $DIR/issue-70935-complex-spans.rs:17:5
-   |
-LL |     async move {
-   |     ^^^^^^^^^^
-note: required by a bound in an opaque type
-  --> $DIR/issue-70935-complex-spans.rs:15:37
-   |
-LL | fn foo(x: NotSync) -> impl Future + Send {
-   |                                     ^^^^
-
 error[E0277]: `*mut ()` cannot be shared between threads safely
   --> $DIR/issue-70935-complex-spans.rs:15:23
    |
@@ -57,7 +14,7 @@ LL | struct NotSync(PhantomData<*mut ()>);
    |        ^^^^^^^
    = note: required for `&NotSync` to implement `Send`
 note: required because it's used within this closure
-  --> $DIR/issue-70935-complex-spans.rs:19:13
+  --> $DIR/issue-70935-complex-spans.rs:18:13
    |
 LL |         baz(|| async {
    |             ^^
@@ -74,6 +31,6 @@ note: required because it's used within this `async` block
 LL |     async move {
    |     ^^^^^^^^^^
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/coroutine/issue-105084.rs b/tests/ui/coroutine/issue-105084.rs
index cddee4990175..4e9f6ee3a595 100644
--- a/tests/ui/coroutine/issue-105084.rs
+++ b/tests/ui/coroutine/issue-105084.rs
@@ -36,7 +36,6 @@ fn main() {
     // one inside `g` and one inside `h`.
     // Proceed and drop `t` in `g`.
     Pin::new(&mut g).resume(());
-    //~^ ERROR borrow of moved value: `g`
 
     // Proceed and drop `t` in `h` -> double free!
     Pin::new(&mut h).resume(());
diff --git a/tests/ui/coroutine/issue-105084.stderr b/tests/ui/coroutine/issue-105084.stderr
index 23c1fdc54592..a69d20e607a4 100644
--- a/tests/ui/coroutine/issue-105084.stderr
+++ b/tests/ui/coroutine/issue-105084.stderr
@@ -1,27 +1,3 @@
-error[E0382]: borrow of moved value: `g`
-  --> $DIR/issue-105084.rs:38:14
-   |
-LL |     let mut g = #[coroutine]
-   |         ----- move occurs because `g` has type `{coroutine@$DIR/issue-105084.rs:16:5: 16:7}`, which does not implement the `Copy` trait
-...
-LL |     let mut h = copy(g);
-   |                      - value moved here
-...
-LL |     Pin::new(&mut g).resume(());
-   |              ^^^^^^ value borrowed here after move
-   |
-note: consider changing this parameter type in function `copy` to borrow instead if owning the value isn't necessary
-  --> $DIR/issue-105084.rs:10:21
-   |
-LL | fn copy(x: T) -> T {
-   |    ----             ^ this parameter takes ownership of the value
-   |    |
-   |    in this function
-help: consider cloning the value if the performance cost is acceptable
-   |
-LL |     let mut h = copy(g.clone());
-   |                       ++++++++
-
 error[E0277]: the trait bound `Box<(i32, ())>: Copy` is not satisfied in `{coroutine@$DIR/issue-105084.rs:16:5: 16:7}`
   --> $DIR/issue-105084.rs:32:17
    |
@@ -45,7 +21,6 @@ note: required by a bound in `copy`
 LL | fn copy(x: T) -> T {
    |            ^^^^ required by this bound in `copy`
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
-Some errors have detailed explanations: E0277, E0382.
-For more information about an error, try `rustc --explain E0277`.
+For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/coroutine/stalled-coroutine-obligations.rs b/tests/ui/coroutine/stalled-coroutine-obligations.rs
new file mode 100644
index 000000000000..89af3c9a583c
--- /dev/null
+++ b/tests/ui/coroutine/stalled-coroutine-obligations.rs
@@ -0,0 +1,32 @@
+//@ edition: 2021
+
+// Regression tests for #137916 and #138274
+// We now check stalled coroutine obligations eagerly at the start of `mir_borrowck`.
+// So these unsatisfied bounds are caught before causing ICEs.
+use std::ptr::null;
+
+async fn a() -> Box {
+    Box::new(async {
+        //~^ ERROR: future cannot be sent between threads safely
+        let non_send = null::<()>();
+        &non_send;
+        async {}.await
+    })
+}
+
+
+trait Trait {}
+fn foo() -> Box { todo!() }
+
+fn fetch() {
+    async {
+        let fut = async {
+            let _x = foo();
+            async {}.await;
+        };
+        let _: Box = Box::new(fut);
+        //~^ ERROR: future cannot be sent between threads safely
+    };
+}
+
+fn main() {}
diff --git a/tests/ui/coroutine/stalled-coroutine-obligations.stderr b/tests/ui/coroutine/stalled-coroutine-obligations.stderr
new file mode 100644
index 000000000000..cbf395dd6cfb
--- /dev/null
+++ b/tests/ui/coroutine/stalled-coroutine-obligations.stderr
@@ -0,0 +1,40 @@
+error: future cannot be sent between threads safely
+  --> $DIR/stalled-coroutine-obligations.rs:9:5
+   |
+LL | /     Box::new(async {
+LL | |
+LL | |         let non_send = null::<()>();
+LL | |         &non_send;
+LL | |         async {}.await
+LL | |     })
+   | |______^ future created by async block is not `Send`
+   |
+   = help: within `{async block@$DIR/stalled-coroutine-obligations.rs:9:14: 9:19}`, the trait `Send` is not implemented for `*const ()`
+note: future is not `Send` as this value is used across an await
+  --> $DIR/stalled-coroutine-obligations.rs:13:18
+   |
+LL |         let non_send = null::<()>();
+   |             -------- has type `*const ()` which is not `Send`
+LL |         &non_send;
+LL |         async {}.await
+   |                  ^^^^^ await occurs here, with `non_send` maybe used later
+   = note: required for the cast from `Box<{async block@$DIR/stalled-coroutine-obligations.rs:9:14: 9:19}>` to `Box`
+
+error: future cannot be sent between threads safely
+  --> $DIR/stalled-coroutine-obligations.rs:27:32
+   |
+LL |         let _: Box = Box::new(fut);
+   |                                ^^^^^^^^^^^^^ future created by async block is not `Send`
+   |
+   = help: the trait `Send` is not implemented for `dyn Trait`
+note: future is not `Send` as this value is used across an await
+  --> $DIR/stalled-coroutine-obligations.rs:25:22
+   |
+LL |             let _x = foo();
+   |                 -- has type `Box` which is not `Send`
+LL |             async {}.await;
+   |                      ^^^^^ await occurs here, with `_x` maybe used later
+   = note: required for the cast from `Box<{async block@$DIR/stalled-coroutine-obligations.rs:23:19: 23:24}>` to `Box`
+
+error: aborting due to 2 previous errors
+
diff --git a/tests/ui/traits/next-solver/stalled-coroutine-obligations.rs b/tests/ui/traits/next-solver/stalled-coroutine-obligations.rs
new file mode 100644
index 000000000000..7c789af9e4cc
--- /dev/null
+++ b/tests/ui/traits/next-solver/stalled-coroutine-obligations.rs
@@ -0,0 +1,50 @@
+//@ edition: 2024
+//@ compile-flags: -Znext-solver --diagnostic-width=300
+
+// Previously we check stalled coroutine obligations after borrowck pass.
+// And we wrongly assume that these obligations hold in borrowck which leads to
+// silent normalization failures.
+// In the next solver, we register opaques types via `NormalizesTo` goals.
+// So these failures also cause those opaques types not registered in storage.
+//
+// Regression test for #151322 and #151323.
+
+#![feature(type_alias_impl_trait)]
+#![feature(negative_impls)]
+#![feature(auto_traits)]
+
+fn stalled_copy_clone() {
+    type T = impl Copy;
+    let foo: T = async {};
+    //~^ ERROR: the trait bound
+
+    type U = impl Clone;
+    let bar: U = async {};
+    //~^ ERROR: the trait bound
+}
+
+auto trait Valid {}
+struct False;
+impl !Valid for False {}
+
+fn stalled_auto_traits() {
+    type T = impl Valid;
+    let a = False;
+    let foo: T = async { a };
+    //~^ ERROR: the trait bound `False: Valid` is not satisfied
+}
+
+
+trait Trait {
+    fn stalled_send(&self, b: *mut ()) -> impl Future + Send {
+    //~^ ERROR: type mismatch resolving
+    //~| ERROR: type mismatch resolving
+        async move {
+            //~^ ERROR: type mismatch resolving
+            b
+        }
+    }
+}
+
+
+fn main() {}
diff --git a/tests/ui/traits/next-solver/stalled-coroutine-obligations.stderr b/tests/ui/traits/next-solver/stalled-coroutine-obligations.stderr
new file mode 100644
index 000000000000..6afa406bf367
--- /dev/null
+++ b/tests/ui/traits/next-solver/stalled-coroutine-obligations.stderr
@@ -0,0 +1,62 @@
+error[E0277]: the trait bound `{async block@$DIR/stalled-coroutine-obligations.rs:18:18: 18:23}: Copy` is not satisfied
+  --> $DIR/stalled-coroutine-obligations.rs:18:14
+   |
+LL |     let foo: T = async {};
+   |              ^ the trait `Copy` is not implemented for `{async block@$DIR/stalled-coroutine-obligations.rs:18:18: 18:23}`
+
+error[E0277]: the trait bound `{async block@$DIR/stalled-coroutine-obligations.rs:22:18: 22:23}: Clone` is not satisfied
+  --> $DIR/stalled-coroutine-obligations.rs:22:14
+   |
+LL |     let bar: U = async {};
+   |              ^ the trait `Clone` is not implemented for `{async block@$DIR/stalled-coroutine-obligations.rs:22:18: 22:23}`
+
+error[E0277]: the trait bound `False: Valid` is not satisfied in `{async block@$DIR/stalled-coroutine-obligations.rs:33:18: 33:23}`
+  --> $DIR/stalled-coroutine-obligations.rs:33:14
+   |
+LL |     let foo: T = async { a };
+   |              ^   ----- within this `{async block@$DIR/stalled-coroutine-obligations.rs:33:18: 33:23}`
+   |              |
+   |              unsatisfied trait bound
+   |
+help: within `{async block@$DIR/stalled-coroutine-obligations.rs:33:18: 33:23}`, the trait `Valid` is not implemented for `False`
+  --> $DIR/stalled-coroutine-obligations.rs:27:1
+   |
+LL | struct False;
+   | ^^^^^^^^^^^^
+note: captured value does not implement `Valid`
+  --> $DIR/stalled-coroutine-obligations.rs:33:26
+   |
+LL |     let foo: T = async { a };
+   |                          ^ has type `False` which does not implement `Valid`
+
+error[E0271]: type mismatch resolving `impl Future + Send == {async block@$DIR/stalled-coroutine-obligations.rs:42:9: 42:19}`
+  --> $DIR/stalled-coroutine-obligations.rs:39:5
+   |
+LL |     fn stalled_send(&self, b: *mut ()) -> impl Future + Send {
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ types differ
+
+error[E0271]: type mismatch resolving `impl Future + Send == {async block@$DIR/stalled-coroutine-obligations.rs:42:9: 42:19}`
+  --> $DIR/stalled-coroutine-obligations.rs:42:9
+   |
+LL | /         async move {
+LL | |
+LL | |             b
+LL | |         }
+   | |_________^ types differ
+
+error[E0271]: type mismatch resolving `{async block@$DIR/stalled-coroutine-obligations.rs:42:9: 42:19} <: impl Future + Send`
+  --> $DIR/stalled-coroutine-obligations.rs:39:62
+   |
+LL |       fn stalled_send(&self, b: *mut ()) -> impl Future + Send {
+   |  ______________________________________________________________^
+LL | |
+LL | |
+LL | |         async move {
+...  |
+LL | |     }
+   | |_____^ types differ
+
+error: aborting due to 6 previous errors
+
+Some errors have detailed explanations: E0271, E0277.
+For more information about an error, try `rustc --explain E0271`.

From ed90b3566960797ca7afbeca6c64ff042bf52158 Mon Sep 17 00:00:00 2001
From: xizheyin 
Date: Mon, 9 Feb 2026 15:06:55 +0800
Subject: [PATCH 1483/1843] Add note when inherent impl for a alias type
 defined outside of the crate

---
 .../src/coherence/inherent_impls.rs            | 17 ++++++++++++++++-
 compiler/rustc_hir_analysis/src/errors.rs      | 18 +++++++++++++++++-
 tests/ui/error-codes/E0116.stderr              |  3 ++-
 .../insufficient-suggestion-issue-141679.rs    |  2 +-
 ...insufficient-suggestion-issue-141679.stderr |  8 +++++++-
 .../no-attr-empty-impl.stderr                  | 12 ++++++++----
 .../no-other-unrelated-errors.stderr           |  3 ++-
 .../ui/traits/trait-or-new-type-instead.stderr |  3 ++-
 8 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
index edaf33e493c0..588747f46d17 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
@@ -110,7 +110,22 @@ impl<'tcx> InherentCollect<'tcx> {
             Ok(())
         } else {
             let impl_span = self.tcx.def_span(impl_def_id);
-            Err(self.tcx.dcx().emit_err(errors::InherentTyOutsideNew { span: impl_span }))
+            let mut err = errors::InherentTyOutsideNew { span: impl_span, note: None };
+
+            if let hir::TyKind::Path(rustc_hir::QPath::Resolved(_, path)) =
+                self.tcx.hir_node_by_def_id(impl_def_id).expect_item().expect_impl().self_ty.kind
+                && let rustc_hir::def::Res::Def(DefKind::TyAlias, def_id) = path.res
+            {
+                let ty_name = self.tcx.def_path_str(def_id);
+                let alias_ty_name = self.tcx.type_of(def_id).skip_binder().to_string();
+                err.note = Some(errors::InherentTyOutsideNewAliasNote {
+                    span: self.tcx.def_span(def_id),
+                    ty_name,
+                    alias_ty_name,
+                });
+            }
+
+            Err(self.tcx.dcx().emit_err(err))
         }
     }
 
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index 6a23b42ae098..fb79789df76e 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -1223,11 +1223,27 @@ pub(crate) struct InherentTyOutsideRelevant {
 
 #[derive(Diagnostic)]
 #[diag("cannot define inherent `impl` for a type outside of the crate where the type is defined", code = E0116)]
-#[note("define and implement a trait or new type instead")]
+#[help(
+    "consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it"
+)]
+#[note(
+    "for more details about the orphan rules, see "
+)]
 pub(crate) struct InherentTyOutsideNew {
     #[primary_span]
     #[label("impl for type defined outside of crate")]
     pub span: Span,
+    #[subdiagnostic]
+    pub note: Option,
+}
+
+#[derive(Subdiagnostic)]
+#[note("`{$ty_name}` does not define a new type, only an alias of `{$alias_ty_name}` defined here")]
+pub(crate) struct InherentTyOutsideNewAliasNote {
+    #[primary_span]
+    pub span: Span,
+    pub ty_name: String,
+    pub alias_ty_name: String,
 }
 
 #[derive(Diagnostic)]
diff --git a/tests/ui/error-codes/E0116.stderr b/tests/ui/error-codes/E0116.stderr
index 1ea5a57f46db..20e3b196226a 100644
--- a/tests/ui/error-codes/E0116.stderr
+++ b/tests/ui/error-codes/E0116.stderr
@@ -4,7 +4,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl Vec {}
    | ^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs
index daf8693b5264..b13b6f418d42 100644
--- a/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs
+++ b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.rs
@@ -5,4 +5,4 @@ pub type Function = Rc;
 
 impl Function {}
 //~^ ERROR cannot define inherent `impl` for a type outside of the crate where the type is defined [E0116]
-fn main(){}
\ No newline at end of file
+fn main(){}
diff --git a/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr
index cbb94ea4d367..a62f7f82ba9d 100644
--- a/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr
+++ b/tests/ui/incoherent-inherent-impls/insufficient-suggestion-issue-141679.stderr
@@ -4,7 +4,13 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl Function {}
    | ^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
+note: `Function` does not define a new type, only an alias of `Rc` defined here
+  --> $DIR/insufficient-suggestion-issue-141679.rs:4:1
+   |
+LL | pub type Function = Rc;
+   | ^^^^^^^^^^^^^^^^^
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/incoherent-inherent-impls/no-attr-empty-impl.stderr b/tests/ui/incoherent-inherent-impls/no-attr-empty-impl.stderr
index f8491697910c..de61c3900d46 100644
--- a/tests/ui/incoherent-inherent-impls/no-attr-empty-impl.stderr
+++ b/tests/ui/incoherent-inherent-impls/no-attr-empty-impl.stderr
@@ -4,7 +4,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl extern_crate::StructWithAttr {}
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error[E0116]: cannot define inherent `impl` for a type outside of the crate where the type is defined
   --> $DIR/no-attr-empty-impl.rs:7:1
@@ -12,7 +13,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl extern_crate::StructNoAttr {}
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error[E0116]: cannot define inherent `impl` for a type outside of the crate where the type is defined
   --> $DIR/no-attr-empty-impl.rs:10:1
@@ -20,7 +22,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl extern_crate::EnumWithAttr {}
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error[E0116]: cannot define inherent `impl` for a type outside of the crate where the type is defined
   --> $DIR/no-attr-empty-impl.rs:13:1
@@ -28,7 +31,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl extern_crate::EnumNoAttr {}
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error[E0390]: cannot define inherent `impl` for primitive types
   --> $DIR/no-attr-empty-impl.rs:16:1
diff --git a/tests/ui/incoherent-inherent-impls/no-other-unrelated-errors.stderr b/tests/ui/incoherent-inherent-impls/no-other-unrelated-errors.stderr
index 2a33262f8389..f01817e29443 100644
--- a/tests/ui/incoherent-inherent-impls/no-other-unrelated-errors.stderr
+++ b/tests/ui/incoherent-inherent-impls/no-other-unrelated-errors.stderr
@@ -4,7 +4,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl Vec {}
    | ^^^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/traits/trait-or-new-type-instead.stderr b/tests/ui/traits/trait-or-new-type-instead.stderr
index 5f5aa3ac5698..ad12a84a4b80 100644
--- a/tests/ui/traits/trait-or-new-type-instead.stderr
+++ b/tests/ui/traits/trait-or-new-type-instead.stderr
@@ -4,7 +4,8 @@ error[E0116]: cannot define inherent `impl` for a type outside of the crate wher
 LL | impl Option {
    | ^^^^^^^^^^^^^^^^^ impl for type defined outside of crate
    |
-   = note: define and implement a trait or new type instead
+   = help: consider defining a trait and implementing it for the type or using a newtype wrapper like `struct MyType(ExternalType);` and implement it
+   = note: for more details about the orphan rules, see 
 
 error: aborting due to 1 previous error
 

From 2b1dc3144bd38e3f655bcc991b685d824f4176f6 Mon Sep 17 00:00:00 2001
From: "Eddy (Eduard) Stefes" 
Date: Wed, 14 Jan 2026 13:08:10 +0100
Subject: [PATCH 1484/1843] add a new s390x-unknown-none-softfloat target

This target is intended to be used for kernel development. Becasue on s390x
float and vector registers overlap we have to disable the vector extension.

The default s390x-unknown-gnu-linux target will not allow use of
softfloat.

Co-authored-by: Jubilee 
---
 compiler/rustc_target/src/spec/mod.rs         |  8 ++
 .../targets/s390x_unknown_none_softfloat.rs   | 39 ++++++++++
 compiler/rustc_target/src/target_features.rs  | 29 +++++--
 src/bootstrap/src/core/sanity.rs              |  1 +
 src/doc/rustc/src/SUMMARY.md                  |  1 +
 src/doc/rustc/src/platform-support.md         |  1 +
 .../s390x-unknown-none-softfloat.md           | 77 +++++++++++++++++++
 tests/assembly-llvm/targets/targets-elf.rs    |  3 +
 8 files changed, 153 insertions(+), 6 deletions(-)
 create mode 100644 compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs
 create mode 100644 src/doc/rustc/src/platform-support/s390x-unknown-none-softfloat.md

diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 9799f3e50ae2..eabfda35028d 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1007,6 +1007,8 @@ crate::target_spec_enum! {
         X86Sse2 = "x86-sse2",
         /// On x86-32/64 only: do not use any FPU or SIMD registers for the ABI.
         X86Softfloat = "x86-softfloat",
+        // On S390x only: do not use any FPU or Vector registers for the ABI.
+        S390xSoftFloat = "s390x-softfloat",
     }
 
     parse_error_type = "rustc abi";
@@ -1460,6 +1462,7 @@ supported_targets! {
     ("powerpc64le-unknown-linux-gnu", powerpc64le_unknown_linux_gnu),
     ("powerpc64le-unknown-linux-musl", powerpc64le_unknown_linux_musl),
     ("s390x-unknown-linux-gnu", s390x_unknown_linux_gnu),
+    ("s390x-unknown-none-softfloat", s390x_unknown_none_softfloat),
     ("s390x-unknown-linux-musl", s390x_unknown_linux_musl),
     ("sparc-unknown-linux-gnu", sparc_unknown_linux_gnu),
     ("sparc64-unknown-linux-gnu", sparc64_unknown_linux_gnu),
@@ -3209,6 +3212,11 @@ impl Target {
                     Arch::X86 | Arch::X86_64,
                     "`x86-softfloat` ABI is only valid for x86 targets"
                 ),
+                RustcAbi::S390xSoftFloat => check_matches!(
+                    self.arch,
+                    Arch::S390x,
+                    "`s390x-softfloat` ABI is only valid for s390x targets"
+                ),
             }
         }
 
diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs
new file mode 100644
index 000000000000..1453a3d6c9b3
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs
@@ -0,0 +1,39 @@
+use rustc_abi::{Align, Endian};
+
+use crate::spec::{
+    Abi, Arch, Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, RustcAbi, SanitizerSet,
+    StackProbeType, Target, TargetMetadata, TargetOptions,
+};
+
+pub(crate) fn target() -> Target {
+    let opts = TargetOptions {
+        abi: Abi::SoftFloat,
+        cpu: "z10".into(),
+        endian: Endian::Big,
+        features: "+soft-float,-vector".into(),
+        linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
+        linker: Some("rust-lld".into()),
+        max_atomic_width: Some(128),
+        min_global_align: Some(Align::from_bits(16).unwrap()),
+        panic_strategy: PanicStrategy::Abort,
+        relocation_model: RelocModel::Static,
+        rustc_abi: Some(RustcAbi::S390xSoftFloat),
+        stack_probes: StackProbeType::Inline,
+        supported_sanitizers: SanitizerSet::KERNELADDRESS,
+        ..Default::default()
+    };
+
+    Target {
+        llvm_target: "s390x-unknown-linux-gnu".into(),
+        metadata: TargetMetadata {
+            description: Some("S390x Linux".into()),
+            host_tools: Some(false),
+            std: Some(false),
+            tier: Some(2),
+        },
+        arch: Arch::S390x,
+        data_layout: "E-S64-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64".into(),
+        options: opts,
+        pointer_width: 64,
+    }
+}
diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs
index bc12e1eb9737..671761a4edd4 100644
--- a/compiler/rustc_target/src/target_features.rs
+++ b/compiler/rustc_target/src/target_features.rs
@@ -863,7 +863,7 @@ const IBMZ_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("miscellaneous-extensions-3", Stable, &[]),
     ("miscellaneous-extensions-4", Stable, &[]),
     ("nnp-assist", Stable, &["vector"]),
-    ("soft-float", Forbidden { reason: "currently unsupported ABI-configuration feature" }, &[]),
+    ("soft-float", Forbidden { reason: "unsupported ABI-configuration feature" }, &[]),
     ("transactional-execution", Unstable(sym::s390x_target_feature), &[]),
     ("vector", Stable, &[]),
     ("vector-enhancements-1", Stable, &["vector"]),
@@ -1105,6 +1105,7 @@ impl Target {
                         // LLVM handles the rest.
                         FeatureConstraints { required: &["soft-float"], incompatible: &[] }
                     }
+                    Some(r) => panic!("invalid Rust ABI for x86: {r:?}"),
                 }
             }
             Arch::X86_64 => {
@@ -1218,11 +1219,27 @@ impl Target {
                 }
             }
             Arch::S390x => {
-                // We don't currently support a softfloat target on this architecture.
-                // As usual, we have to reject swapping the `soft-float` target feature.
-                // The "vector" target feature does not affect the ABI for floats
-                // because the vector and float registers overlap.
-                FeatureConstraints { required: &[], incompatible: &["soft-float"] }
+                // Same as x86, We use our own ABI indicator here;
+                // LLVM does not have anything native and will switch ABI based
+                // on the soft-float target feature.
+                // Every case should require or forbid `soft-float`!
+                // The "vector" target feature may only be used without soft-float
+                // because the float and vector registers overlap and the
+                // standard s390x C ABI may pass vectors via these registers.
+                match self.rustc_abi {
+                    None => {
+                        // Default hardfloat ABI.
+                        FeatureConstraints { required: &[], incompatible: &["soft-float"] }
+                    }
+                    Some(RustcAbi::S390xSoftFloat) => {
+                        // Softfloat ABI, requires corresponding target feature.
+                        // llvm will switch to soft-float ABI just based on this feature.
+                        FeatureConstraints { required: &["soft-float"], incompatible: &["vector"] }
+                    }
+                    Some(r) => {
+                        panic!("invalid Rust ABI for s390x: {r:?}");
+                    }
+                }
             }
             _ => NOTHING,
         }
diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs
index 8b6405afa892..e5327ab79ee2 100644
--- a/src/bootstrap/src/core/sanity.rs
+++ b/src/bootstrap/src/core/sanity.rs
@@ -48,6 +48,7 @@ const STAGE0_MISSING_TARGETS: &[&str] = &[
     "thumbv6-none-eabi",
     "aarch64v8r-unknown-none",
     "aarch64v8r-unknown-none-softfloat",
+    "s390x-unknown-none-softfloat",
 ];
 
 /// Minimum version threshold for libstdc++ required when using prebuilt LLVM
diff --git a/src/doc/rustc/src/SUMMARY.md b/src/doc/rustc/src/SUMMARY.md
index 2ec0c3648bdf..9049828c8b12 100644
--- a/src/doc/rustc/src/SUMMARY.md
+++ b/src/doc/rustc/src/SUMMARY.md
@@ -120,6 +120,7 @@
     - [riscv64a23-unknown-linux-gnu](platform-support/riscv64a23-unknown-linux-gnu.md)
     - [s390x-unknown-linux-gnu](platform-support/s390x-unknown-linux-gnu.md)
     - [s390x-unknown-linux-musl](platform-support/s390x-unknown-linux-musl.md)
+    - [s390x-unknown-none-softfloat](platform-support/s390x-unknown-none-softfloat.md)
     - [sparc-unknown-none-elf](./platform-support/sparc-unknown-none-elf.md)
     - [solaris](platform-support/solaris.md)
     - [\*-nto-qnx-\*](platform-support/nto-qnx.md)
diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index 114f66282afd..29553f9c2a00 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -188,6 +188,7 @@ target | std | notes
 [`riscv64im-unknown-none-elf`](platform-support/riscv64im-unknown-none-elf.md) | * | Bare RISC-V (RV64IM ISA)
 `riscv64imac-unknown-none-elf` | * | Bare RISC-V (RV64IMAC ISA)
 `sparc64-unknown-linux-gnu` | ✓ | SPARC Linux (kernel 4.4+, glibc 2.23)
+[`s390x-unknown-none-softfloat`](platform-support/s390x-unknown-none-softfloat.md) | * | Bare S390x (softfloat ABI)
 [`thumbv6m-none-eabi`](platform-support/thumbv6m-none-eabi.md) | * | Bare Armv6-M
 [`thumbv7em-none-eabi`](platform-support/thumbv7em-none-eabi.md) | * | Bare Armv7E-M
 [`thumbv7em-none-eabihf`](platform-support/thumbv7em-none-eabi.md) | * | Bare Armv7E-M, hardfloat
diff --git a/src/doc/rustc/src/platform-support/s390x-unknown-none-softfloat.md b/src/doc/rustc/src/platform-support/s390x-unknown-none-softfloat.md
new file mode 100644
index 000000000000..39a1a7c52c39
--- /dev/null
+++ b/src/doc/rustc/src/platform-support/s390x-unknown-none-softfloat.md
@@ -0,0 +1,77 @@
+# `s390x-unknown-none-softfloat`
+
+**Tier: 2**
+
+IBM z/Architecture (s390x) code in ELF format for kernels, etc.
+
+## Target maintainers
+
+[@uweigand](https://github.com/uweigand)
+[@cuviper](https://github.com/cuviper)
+
+## Requirements
+
+This target is intended for kernel development on s390x only. This target is
+cross-compiled. There is no support for `std`.There is no default allocator,
+but it's possible to use `alloc` by supplying an allocator.
+
+The target does not assume existence of a FPU and does not make use of any
+non-GPR register. This allows the generated code to run in environments, such
+as kernels, which may need to avoid the use of such registers or which may have
+special considerations about the use of such registers (e.g. saving and
+restoring them to avoid breaking userspace code using the same registers). You
+can change code generation to use additional CPU features via the
+`-C target-feature=` codegen options to rustc, or via the `#[target_feature]`
+mechanism within Rust code.
+
+By default, code generated with the soft-float target should run on any Z System
+starting at [Z10][s390x-isa]. Enabling additional target features or changing the
+`-Ctarget-cpu` may raise the ISA required from the `z10` baseline.
+
+`extern "C"` does not use a stable ABI and is subject to change between compiler
+or codegen backend versions.
+
+The target only generates object files in the ELF format. Any alternate formats
+or special considerations for binary layout will require linker options or linker
+scripts.
+
+* [z/Architecture Principles of Operation][s390x-isa]
+
+[s390x-isa]: https://publibfp.dhe.ibm.com/epubs/pdf/a227832d.pdf
+[s390x-abi]: https://github.com/IBM/s390x-abi
+
+## Building the target
+
+You can build Rust with support for the target by adding it to the `target`
+list in `bootstrap.toml`:
+
+```toml
+[build]
+target = ["s390x-unknown-none-softfloat"]
+```
+
+## Building Rust programs
+
+This target is not intended to build stand-alone binaries. You should only use
+it in conjunction with the kernel build toolchain.
+
+## Testing
+
+As code generated by this target is intended to always be part of the kernel,
+there are no additional requirements for testing.
+
+If you want to do native testing but do not have your own s390x
+machine, there are several options how to get access to one:
+
+* The [IBM LinuxONE Community Cloud][cloud-community] provides a
+  self-service portal where you can create s390x virtual machine
+  instances.  These are intended for temporary use (limited to 120 days).
+
+* The [IBM LinuxONE Open Source Cloud][cloud-opensource] provides
+  permanent access to s390x machines.  This requires approval by IBM,
+  which will normally be granted if you're planning to use the machine
+  to work on an open-source project that is relevant to the IBM Z
+  ecosystem - the Rust compiler would certainly qualify.
+
+[cloud-community]: https://linuxone.cloud.marist.edu/
+[cloud-opensource]: https://community.ibm.com/zsystems/form/l1cc-oss-vm-request/
diff --git a/tests/assembly-llvm/targets/targets-elf.rs b/tests/assembly-llvm/targets/targets-elf.rs
index c38e86315b27..b7deb6686cfe 100644
--- a/tests/assembly-llvm/targets/targets-elf.rs
+++ b/tests/assembly-llvm/targets/targets-elf.rs
@@ -544,6 +544,9 @@
 //@ revisions: s390x_unknown_linux_musl
 //@ [s390x_unknown_linux_musl] compile-flags: --target s390x-unknown-linux-musl
 //@ [s390x_unknown_linux_musl] needs-llvm-components: systemz
+//@ revisions: s390x_unknown_none_softfloat
+//@ [s390x_unknown_none_softfloat] compile-flags: --target s390x-unknown-none-softfloat
+//@ [s390x_unknown_none_softfloat] needs-llvm-components: systemz
 //@ revisions: sparc64_unknown_helenos
 //@ [sparc64_unknown_helenos] compile-flags: --target sparc64-unknown-helenos
 //@ [sparc64_unknown_helenos] needs-llvm-components: sparc

From 51affa03942074a84ccce448ea6b7b9e02a4f4a6 Mon Sep 17 00:00:00 2001
From: "Eddy (Eduard) Stefes" 
Date: Thu, 22 Jan 2026 13:50:54 +0100
Subject: [PATCH 1485/1843] add tests for s390x-unknown-none-softfloat

tests will check:
- correct emit of assembly for softfloat target
- incompatible set features will emit warnings/errors
- incompatible target tripples in crates will not link
---
 tests/assembly-llvm/s390x-softfloat-abi.rs    | 64 +++++++++++++++++++
 ...0x-softfloat-gate.disable-softfloat.stderr | 12 ++++
 ...90x-softfloat-gate.enable-softfloat.stderr |  7 ++
 tests/ui/abi/s390x-softfloat-gate.rs          | 38 +++++++++++
 ...imd-abi-checks-s390x.z13_soft_float.stderr |  2 +-
 .../auxiliary/disabled_softfloat.rs           |  8 +++
 .../auxiliary/enabled_softfloat.rs            |  8 +++
 ...softfloat_targets.disable-softfloat.stderr | 12 ++++
 ..._softfloat_targets.enable-softfloat.stderr | 12 ++++
 .../incompatible_softfloat_targets.rs         | 20 ++++++
 10 files changed, 182 insertions(+), 1 deletion(-)
 create mode 100644 tests/assembly-llvm/s390x-softfloat-abi.rs
 create mode 100644 tests/ui/abi/s390x-softfloat-gate.disable-softfloat.stderr
 create mode 100644 tests/ui/abi/s390x-softfloat-gate.enable-softfloat.stderr
 create mode 100644 tests/ui/abi/s390x-softfloat-gate.rs
 create mode 100644 tests/ui/target_modifiers/auxiliary/disabled_softfloat.rs
 create mode 100644 tests/ui/target_modifiers/auxiliary/enabled_softfloat.rs
 create mode 100644 tests/ui/target_modifiers/incompatible_softfloat_targets.disable-softfloat.stderr
 create mode 100644 tests/ui/target_modifiers/incompatible_softfloat_targets.enable-softfloat.stderr
 create mode 100644 tests/ui/target_modifiers/incompatible_softfloat_targets.rs

diff --git a/tests/assembly-llvm/s390x-softfloat-abi.rs b/tests/assembly-llvm/s390x-softfloat-abi.rs
new file mode 100644
index 000000000000..e2d9d013d201
--- /dev/null
+++ b/tests/assembly-llvm/s390x-softfloat-abi.rs
@@ -0,0 +1,64 @@
+//@ add-minicore
+//@ revisions: enable-softfloat disable-softfloat
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3 --crate-type=lib
+//@[enable-softfloat] compile-flags: --target=s390x-unknown-none-softfloat
+//@[enable-softfloat] needs-llvm-components: systemz
+//@[disable-softfloat] compile-flags: --target=s390x-unknown-linux-gnu
+//@[disable-softfloat] needs-llvm-components: systemz
+//@ ignore-backends: gcc
+
+#![feature(no_core, lang_items)]
+#![no_std]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+extern "C" {
+    fn extern_func(value: f64) -> f64;
+}
+
+// CHECK-LABEL: test_softfloat
+#[no_mangle]
+extern "C" fn test_softfloat() -> f64 {
+    let value = 3.141_f64;
+
+    // without softfloat we load the value direct to the first float register
+    // we do NOT construct a softfloat in r2 (first non-float arg register)
+    // disable-softfloat: ld %f{{.*}}, 0(%r{{.*}})
+    // disable-softfloat-NOT: llihf %r{{.*}}, 1074340036
+    // disable-softfloat-NOT: oilf %r{{.*}}, 2611340116
+
+    // with softfloat we construct the softfloat arg in r2
+    // we do NOT pass anything by f0 (first float arg register)
+    // float registers can not be accessed
+    // enable-softfloat: llihf %r{{.*}}, 1074340036
+    // enable-softfloat-NEXT: oilf %r{{.*}}, 2611340116
+    // enable-softfloat-NOT: ld %f{{.*}}, 0(%r{{.*}})
+
+    unsafe { extern_func(value) };
+    // disable-softfloat-NEXT: brasl %r{{.*}}, extern_func@PLT
+    // enable-softfloat-NEXT: brasl %r{{.*}}, extern_func@PLT
+
+    // for return we check that without softfloat we write to float register
+    // disable-softfloat: ld %f{{.*}}, 0(%r{{.*}})
+    // disable-softfloat-NOT: llihf %r{{.*}}, 1072841097
+    // disable-softfloat-NOT: oilf %r{{.*}}, 927712936
+
+    #[cfg(not(target_feature = "soft-float"))]
+    {
+        1.141_f64
+    }
+
+    // for return we check that WITH softfloat we write to genral purpose register
+    // enable-softfloat: llihf %r{{.*}}, 1072841097
+    // enable-softfloat-NEXT: oilf %r{{.*}}, 927712936
+    // enable-softfloat-NOT: ld %f{{.*}}, 0(%r{{.*}})
+    #[cfg(target_feature = "soft-float")]
+    {
+        2.718_f64
+    }
+    // enable-softfloat: br %r{{.*}}
+    // disable-softfloat: br %r{{.*}}
+}
diff --git a/tests/ui/abi/s390x-softfloat-gate.disable-softfloat.stderr b/tests/ui/abi/s390x-softfloat-gate.disable-softfloat.stderr
new file mode 100644
index 000000000000..e82d5b744a26
--- /dev/null
+++ b/tests/ui/abi/s390x-softfloat-gate.disable-softfloat.stderr
@@ -0,0 +1,12 @@
+warning: target feature `soft-float` cannot be enabled with `-Ctarget-feature`: unsupported ABI-configuration feature
+   |
+   = note: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #116344 
+
+warning: target feature `soft-float` must be disabled to ensure that the ABI of the current target can be implemented correctly
+   |
+   = note: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #116344 
+
+warning: 2 warnings emitted
+
diff --git a/tests/ui/abi/s390x-softfloat-gate.enable-softfloat.stderr b/tests/ui/abi/s390x-softfloat-gate.enable-softfloat.stderr
new file mode 100644
index 000000000000..ecc96e448dcf
--- /dev/null
+++ b/tests/ui/abi/s390x-softfloat-gate.enable-softfloat.stderr
@@ -0,0 +1,7 @@
+warning: target feature `vector` must be disabled to ensure that the ABI of the current target can be implemented correctly
+   |
+   = note: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #116344 
+
+warning: 1 warning emitted
+
diff --git a/tests/ui/abi/s390x-softfloat-gate.rs b/tests/ui/abi/s390x-softfloat-gate.rs
new file mode 100644
index 000000000000..496929eb0955
--- /dev/null
+++ b/tests/ui/abi/s390x-softfloat-gate.rs
@@ -0,0 +1,38 @@
+//@ add-minicore
+//@ revisions: disable-softfloat enable-softfloat
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3 --crate-type=lib
+
+// we expect the build to fail in the feature
+//@ build-pass
+//@ [enable-softfloat] compile-flags: --target=s390x-unknown-none-softfloat
+//@ [enable-softfloat] compile-flags: -C target-feature=+vector
+//@ [enable-softfloat] needs-llvm-components: systemz
+//@ [disable-softfloat] compile-flags: --target=s390x-unknown-linux-gnu
+//@ [disable-softfloat] compile-flags: -C target-feature=+soft-float
+//@ [disable-softfloat] needs-llvm-components: systemz
+//@ ignore-backends: gcc
+
+//[disable-softfloat]~? WARN target feature `soft-float` must be disabled to ensure that the ABI of the current target can be implemented correctly
+//[disable-softfloat]~? WARN target feature `soft-float` cannot be enabled with `-Ctarget-feature`
+//[enable-softfloat]~? WARN target feature `vector` must be disabled to ensure that the ABI of the current target can be implemented correctly
+
+#![feature(no_core, lang_items)]
+#![no_std]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+extern "C" {
+    fn extern_func(value: f64) -> f64;
+}
+
+#[no_mangle]
+extern "C" fn test_softfloat() -> f64 {
+    let value = 3.141_f64;
+
+    unsafe { extern_func(value) } ;
+
+    2.718_f64
+}
diff --git a/tests/ui/abi/simd-abi-checks-s390x.z13_soft_float.stderr b/tests/ui/abi/simd-abi-checks-s390x.z13_soft_float.stderr
index 0e8e6637507d..cda51a211324 100644
--- a/tests/ui/abi/simd-abi-checks-s390x.z13_soft_float.stderr
+++ b/tests/ui/abi/simd-abi-checks-s390x.z13_soft_float.stderr
@@ -1,4 +1,4 @@
-warning: target feature `soft-float` cannot be enabled with `-Ctarget-feature`: currently unsupported ABI-configuration feature
+warning: target feature `soft-float` cannot be enabled with `-Ctarget-feature`: unsupported ABI-configuration feature
    |
    = note: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
    = note: for more information, see issue #116344 
diff --git a/tests/ui/target_modifiers/auxiliary/disabled_softfloat.rs b/tests/ui/target_modifiers/auxiliary/disabled_softfloat.rs
new file mode 100644
index 000000000000..0063d925deca
--- /dev/null
+++ b/tests/ui/target_modifiers/auxiliary/disabled_softfloat.rs
@@ -0,0 +1,8 @@
+//@ add-minicore
+//@ no-prefer-dynamic
+//@ compile-flags: --target=s390x-unknown-linux-gnu
+//@ needs-llvm-components: systemz
+
+#![feature(no_core)]
+#![crate_type = "rlib"]
+#![no_core]
diff --git a/tests/ui/target_modifiers/auxiliary/enabled_softfloat.rs b/tests/ui/target_modifiers/auxiliary/enabled_softfloat.rs
new file mode 100644
index 000000000000..7439a1eb69b5
--- /dev/null
+++ b/tests/ui/target_modifiers/auxiliary/enabled_softfloat.rs
@@ -0,0 +1,8 @@
+//@ add-minicore
+//@ no-prefer-dynamic
+//@ compile-flags: --target=s390x-unknown-none-softfloat
+//@ needs-llvm-components: systemz
+
+#![feature(no_core)]
+#![crate_type = "rlib"]
+#![no_core]
diff --git a/tests/ui/target_modifiers/incompatible_softfloat_targets.disable-softfloat.stderr b/tests/ui/target_modifiers/incompatible_softfloat_targets.disable-softfloat.stderr
new file mode 100644
index 000000000000..090ebe6ca14c
--- /dev/null
+++ b/tests/ui/target_modifiers/incompatible_softfloat_targets.disable-softfloat.stderr
@@ -0,0 +1,12 @@
+error[E0461]: couldn't find crate `enabled_softfloat` with expected target triple s390x-unknown-linux-gnu
+  --> $DIR/incompatible_softfloat_targets.rs:17:1
+   |
+LL | extern crate enabled_softfloat;
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: the following crate versions were found:
+           crate `enabled_softfloat`, target triple s390x-unknown-none-softfloat: $TEST_BUILD_DIR/auxiliary/libenabled_softfloat.rlib
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0461`.
diff --git a/tests/ui/target_modifiers/incompatible_softfloat_targets.enable-softfloat.stderr b/tests/ui/target_modifiers/incompatible_softfloat_targets.enable-softfloat.stderr
new file mode 100644
index 000000000000..9adb6d265651
--- /dev/null
+++ b/tests/ui/target_modifiers/incompatible_softfloat_targets.enable-softfloat.stderr
@@ -0,0 +1,12 @@
+error[E0461]: couldn't find crate `disabled_softfloat` with expected target triple s390x-unknown-none-softfloat
+  --> $DIR/incompatible_softfloat_targets.rs:19:1
+   |
+LL | extern crate disabled_softfloat;
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: the following crate versions were found:
+           crate `disabled_softfloat`, target triple s390x-unknown-linux-gnu: $TEST_BUILD_DIR/auxiliary/libdisabled_softfloat.rlib
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0461`.
diff --git a/tests/ui/target_modifiers/incompatible_softfloat_targets.rs b/tests/ui/target_modifiers/incompatible_softfloat_targets.rs
new file mode 100644
index 000000000000..3d253c5f0c78
--- /dev/null
+++ b/tests/ui/target_modifiers/incompatible_softfloat_targets.rs
@@ -0,0 +1,20 @@
+//@ add-minicore
+//@ aux-build: disabled_softfloat.rs
+//@ aux-build: enabled_softfloat.rs
+//@ revisions: disable-softfloat enable-softfloat
+//@ check-fail
+//@ [enable-softfloat] compile-flags: --target=s390x-unknown-none-softfloat
+//@ [enable-softfloat] needs-llvm-components: systemz
+//@ [disable-softfloat] compile-flags: --target=s390x-unknown-linux-gnu
+//@ [disable-softfloat] needs-llvm-components: systemz
+//@ ignore-backends: gcc
+
+
+#![feature(no_core)]
+#![crate_type = "rlib"]
+#![no_core]
+
+extern crate enabled_softfloat;
+//[disable-softfloat]~^ ERROR couldn't find crate `enabled_softfloat` with expected target triple s390x-unknown-linux-gnu
+extern crate disabled_softfloat;
+//[enable-softfloat]~^ ERROR couldn't find crate `disabled_softfloat` with expected target triple s390x-unknown-none-softfloat

From 7c3d096b0dcb39ad23b42394d889ec8177f7ce26 Mon Sep 17 00:00:00 2001
From: "Eddy (Eduard) Stefes" 
Date: Fri, 6 Feb 2026 10:53:59 +0100
Subject: [PATCH 1486/1843] allow for variant aliases in  target_spec_enum!

---
 compiler/rustc_target/src/lib.rs | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs
index dfa1b2320718..1dc62cb3659c 100644
--- a/compiler/rustc_target/src/lib.rs
+++ b/compiler/rustc_target/src/lib.rs
@@ -76,7 +76,7 @@ macro_rules! target_spec_enum {
         pub enum $Name:ident {
             $(
                 $( #[$variant_attr:meta] )*
-                $Variant:ident = $string:literal,
+                $Variant:ident = $string:literal $(,$alias:literal)* ,
             )*
         }
         parse_error_type = $parse_error_type:literal;
@@ -88,6 +88,7 @@ macro_rules! target_spec_enum {
             $(
                 $( #[$variant_attr] )*
                 #[serde(rename = $string)] // for JSON schema generation only
+                $( #[serde(alias = $alias)] )*
                 $Variant,
             )*
         }
@@ -97,7 +98,10 @@ macro_rules! target_spec_enum {
 
             fn from_str(s: &str) -> Result {
                 Ok(match s {
-                    $( $string => Self::$Variant, )*
+                    $(
+                        $string => Self::$Variant,
+                        $($alias => Self::$Variant,)*
+                    )*
                     _ => {
                         let all = [$( concat!("'", $string, "'") ),*].join(", ");
                         return Err(format!("invalid {}: '{s}'. allowed values: {all}", $parse_error_type));
@@ -123,7 +127,7 @@ macro_rules! target_spec_enum {
         pub enum $Name:ident {
             $(
                 $( #[$variant_attr:meta] )*
-                $Variant:ident = $string:literal,
+                $Variant:ident = $string:literal $(,$alias:literal)* ,
             )*
         }
         $( #[$other_variant_attr:meta] )*
@@ -134,6 +138,7 @@ macro_rules! target_spec_enum {
         pub enum $Name {
             $(
                 $( #[$variant_attr:meta] )*
+                 $( #[serde(alias = $alias)] )*
                 $Variant,
             )*
             /// The vast majority of the time, the compiler deals with a fixed
@@ -165,7 +170,10 @@ macro_rules! target_spec_enum {
 
             fn from_str(s: &str) -> Result {
                 Ok(match s {
-                    $( $string => Self::$Variant, )*
+                    $(
+                        $string => Self::$Variant,
+                        $($alias => Self::$Variant,)*
+                    )*
                     _ => Self::$OtherVariant(s.to_owned().into()),
                 })
             }

From 83dba5b430160bee94e5cc9514993fea4bfa0d96 Mon Sep 17 00:00:00 2001
From: "Eddy (Eduard) Stefes" 
Date: Thu, 22 Jan 2026 13:20:52 +0100
Subject: [PATCH 1487/1843] renamed RustcAbi::X86Softfloat to Softfloat an made
 an alias to the old string

---
 compiler/rustc_target/src/callconv/x86_win64.rs |  2 +-
 compiler/rustc_target/src/spec/mod.rs           | 17 +++++------------
 .../src/spec/targets/i686_unknown_uefi.rs       |  2 +-
 .../targets/s390x_unknown_none_softfloat.rs     |  2 +-
 .../src/spec/targets/x86_64_unknown_none.rs     |  2 +-
 .../src/spec/targets/x86_64_unknown_uefi.rs     |  2 +-
 compiler/rustc_target/src/target_features.rs    |  7 +++----
 7 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_target/src/callconv/x86_win64.rs b/compiler/rustc_target/src/callconv/x86_win64.rs
index 48f7700a86d5..4026c4f471a7 100644
--- a/compiler/rustc_target/src/callconv/x86_win64.rs
+++ b/compiler/rustc_target/src/callconv/x86_win64.rs
@@ -28,7 +28,7 @@ where
             BackendRepr::ScalableVector { .. } => panic!("scalable vectors are unsupported"),
             BackendRepr::Scalar(scalar) => {
                 if is_ret && matches!(scalar.primitive(), Primitive::Int(Integer::I128, _)) {
-                    if cx.target_spec().rustc_abi == Some(RustcAbi::X86Softfloat) {
+                    if cx.target_spec().rustc_abi == Some(RustcAbi::Softfloat) {
                         // Use the native `i128` LLVM type for the softfloat ABI -- in other words, adjust nothing.
                     } else {
                         // `i128` is returned in xmm0 by Clang and GCC
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index eabfda35028d..537185f536ab 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1005,10 +1005,8 @@ crate::target_spec_enum! {
     pub enum RustcAbi {
         /// On x86-32 only: make use of SSE and SSE2 for ABI purposes.
         X86Sse2 = "x86-sse2",
-        /// On x86-32/64 only: do not use any FPU or SIMD registers for the ABI.
-        X86Softfloat = "x86-softfloat",
-        // On S390x only: do not use any FPU or Vector registers for the ABI.
-        S390xSoftFloat = "s390x-softfloat",
+        /// On x86-32/64 and S390x: do not use any FPU or SIMD registers for the ABI.
+        Softfloat = "softfloat", "x86-softfloat",
     }
 
     parse_error_type = "rustc abi";
@@ -3207,15 +3205,10 @@ impl Target {
                     Arch::X86,
                     "`x86-sse2` ABI is only valid for x86-32 targets"
                 ),
-                RustcAbi::X86Softfloat => check_matches!(
+                RustcAbi::Softfloat => check_matches!(
                     self.arch,
-                    Arch::X86 | Arch::X86_64,
-                    "`x86-softfloat` ABI is only valid for x86 targets"
-                ),
-                RustcAbi::S390xSoftFloat => check_matches!(
-                    self.arch,
-                    Arch::S390x,
-                    "`s390x-softfloat` ABI is only valid for s390x targets"
+                    Arch::X86 | Arch::X86_64 | Arch::S390x,
+                    "`softfloat` ABI is only valid for x86 and s390x targets"
                 ),
             }
         }
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_uefi.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_uefi.rs
index 37b202097b46..7696edd5ba71 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_uefi.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_uefi.rs
@@ -22,7 +22,7 @@ pub(crate) fn target() -> Target {
     // If you initialize FP units yourself, you can override these flags with custom linker
     // arguments, thus giving you access to full MMX/SSE acceleration.
     base.features = "-mmx,-sse,+soft-float".into();
-    base.rustc_abi = Some(RustcAbi::X86Softfloat);
+    base.rustc_abi = Some(RustcAbi::Softfloat);
 
     // Turn off DWARF. This fixes an lld warning, "section name .debug_frame is longer than 8
     // characters and will use a non-standard string table". That section will not be created if
diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs
index 1453a3d6c9b3..7d42c1fd9244 100644
--- a/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs
+++ b/compiler/rustc_target/src/spec/targets/s390x_unknown_none_softfloat.rs
@@ -17,7 +17,7 @@ pub(crate) fn target() -> Target {
         min_global_align: Some(Align::from_bits(16).unwrap()),
         panic_strategy: PanicStrategy::Abort,
         relocation_model: RelocModel::Static,
-        rustc_abi: Some(RustcAbi::S390xSoftFloat),
+        rustc_abi: Some(RustcAbi::Softfloat),
         stack_probes: StackProbeType::Inline,
         supported_sanitizers: SanitizerSet::KERNELADDRESS,
         ..Default::default()
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs
index 520a59d6a6f6..0afe7a0b68b0 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs
@@ -20,7 +20,7 @@ pub(crate) fn target() -> Target {
         relro_level: RelroLevel::Full,
         linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
         linker: Some("rust-lld".into()),
-        rustc_abi: Some(RustcAbi::X86Softfloat),
+        rustc_abi: Some(RustcAbi::Softfloat),
         features: "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2,+soft-float".into(),
         supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
         disable_redzone: true,
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs
index 8a494d0e56dd..333e20bd0ac1 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs
@@ -27,7 +27,7 @@ pub(crate) fn target() -> Target {
     // If you initialize FP units yourself, you can override these flags with custom linker
     // arguments, thus giving you access to full MMX/SSE acceleration.
     base.features = "-mmx,-sse,+soft-float".into();
-    base.rustc_abi = Some(RustcAbi::X86Softfloat);
+    base.rustc_abi = Some(RustcAbi::Softfloat);
 
     Target {
         llvm_target: "x86_64-unknown-windows".into(),
diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs
index 671761a4edd4..dfa2b839a4c0 100644
--- a/compiler/rustc_target/src/target_features.rs
+++ b/compiler/rustc_target/src/target_features.rs
@@ -1098,14 +1098,13 @@ impl Target {
                             incompatible: &["soft-float"],
                         }
                     }
-                    Some(RustcAbi::X86Softfloat) => {
+                    Some(RustcAbi::Softfloat) => {
                         // Softfloat ABI, requires corresponding target feature. That feature trumps
                         // `x87` and all other FPU features so those do not matter.
                         // Note that this one requirement is the entire implementation of the ABI!
                         // LLVM handles the rest.
                         FeatureConstraints { required: &["soft-float"], incompatible: &[] }
                     }
-                    Some(r) => panic!("invalid Rust ABI for x86: {r:?}"),
                 }
             }
             Arch::X86_64 => {
@@ -1119,7 +1118,7 @@ impl Target {
                             incompatible: &["soft-float"],
                         }
                     }
-                    Some(RustcAbi::X86Softfloat) => {
+                    Some(RustcAbi::Softfloat) => {
                         // Softfloat ABI, requires corresponding target feature. That feature trumps
                         // `x87` and all other FPU features so those do not matter.
                         // Note that this one requirement is the entire implementation of the ABI!
@@ -1231,7 +1230,7 @@ impl Target {
                         // Default hardfloat ABI.
                         FeatureConstraints { required: &[], incompatible: &["soft-float"] }
                     }
-                    Some(RustcAbi::S390xSoftFloat) => {
+                    Some(RustcAbi::Softfloat) => {
                         // Softfloat ABI, requires corresponding target feature.
                         // llvm will switch to soft-float ABI just based on this feature.
                         FeatureConstraints { required: &["soft-float"], incompatible: &["vector"] }

From f91c8083d8a90d6a275584389e853cccb0b45b08 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Mon, 9 Feb 2026 11:01:03 +1100
Subject: [PATCH 1488/1843] Adjust `make_dep_kind_vtable_for_query`.

It takes an `is_anon` argument, but it's now generic over `QueryFlags`,
which contains the same `is_anon` field. So the argument is no longer
necessary.
---
 compiler/rustc_query_impl/src/plumbing.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index bbda716eadb0..aefd4418e192 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -512,12 +512,12 @@ pub(crate) fn make_dep_kind_vtable_for_query<
     C: QueryCache + 'tcx,
     const FLAGS: QueryFlags,
 >(
-    is_anon: bool,
     is_eval_always: bool,
 ) -> DepKindVTable<'tcx>
 where
     Q: QueryDispatcherUnerased<'tcx, C, FLAGS>,
 {
+    let is_anon = FLAGS.is_anon;
     let fingerprint_style = if is_anon {
         FingerprintStyle::Opaque
     } else {
@@ -969,7 +969,6 @@ macro_rules! define_queries {
             $(pub(crate) fn $name<'tcx>() -> DepKindVTable<'tcx> {
                 use $crate::query_impl::$name::QueryType;
                 $crate::plumbing::make_dep_kind_vtable_for_query::, _, _>(
-                    is_anon!([$($modifiers)*]),
                     is_eval_always!([$($modifiers)*]),
                 )
             })*

From 541d7fc19d14c64a1490c065a73986f9e018b3d6 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Mon, 9 Feb 2026 13:02:06 +1100
Subject: [PATCH 1489/1843] Remove `trait QueryDispatcher`.

It existed only to bridge the divide between `rustc_query_system` and
`rustc_query_impl`. But enough pieces have been moved from the former to
the latter that the trait is no longer needed. Less indirection and
abstraction makes the code easier to understand.
---
 compiler/rustc_query_impl/src/execution.rs    | 157 +++++++-----------
 compiler/rustc_query_impl/src/lib.rs          |  55 +++---
 compiler/rustc_query_impl/src/plumbing.rs     |  30 ++--
 .../rustc_query_system/src/query/caches.rs    |   3 +
 .../src/query/dispatcher.rs                   |  87 ----------
 compiler/rustc_query_system/src/query/mod.rs  |   2 +-
 6 files changed, 111 insertions(+), 223 deletions(-)

diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
index e58e626a7dfa..ba442d8a0081 100644
--- a/compiler/rustc_query_impl/src/execution.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -9,14 +9,14 @@ use rustc_middle::dep_graph::DepsType;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext};
 use rustc_query_system::query::{
-    ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryContext, QueryDispatcher,
-    QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, QueryMode, QueryStackDeferred,
-    QueryStackFrame, QueryState, incremental_verify_ich, report_cycle,
+    ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryContext, QueryJob,
+    QueryJobId, QueryJobInfo, QueryLatch, QueryMap, QueryMode, QueryStackDeferred, QueryStackFrame,
+    QueryState, incremental_verify_ich, report_cycle,
 };
 use rustc_span::{DUMMY_SP, Span};
 
 use crate::dep_graph::{DepContext, DepNode, DepNodeIndex};
-use crate::plumbing::QueryCtxt;
+use crate::{QueryCtxt, QueryFlags, SemiDynamicQueryDispatcher};
 
 #[inline]
 fn equivalent_key(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
@@ -95,23 +95,21 @@ where
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<'tcx, Q>(query: Q, qcx: QueryCtxt<'tcx>, cycle_error: CycleError) -> Q::Value
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+    qcx: QueryCtxt<'tcx>,
+    cycle_error: CycleError,
+) -> C::Value {
     let error = report_cycle(qcx.tcx.sess, &cycle_error);
     handle_cycle_error(query, qcx, &cycle_error, error)
 }
 
-fn handle_cycle_error<'tcx, Q>(
-    query: Q,
+fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     cycle_error: &CycleError,
     error: Diag<'_>,
-) -> Q::Value
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> C::Value {
     match query.cycle_error_handling() {
         CycleErrorHandling::Error => {
             let guar = error.emit();
@@ -205,15 +203,12 @@ where
 
 #[cold]
 #[inline(never)]
-fn cycle_error<'tcx, Q>(
-    query: Q,
+fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     try_execute: QueryJobId,
     span: Span,
-) -> (Q::Value, Option)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (C::Value, Option) {
     // Ensure there was no errors collecting all active jobs.
     // We need the complete map to ensure we find a cycle to break.
     let query_map = qcx
@@ -226,17 +221,14 @@ where
 }
 
 #[inline(always)]
-fn wait_for_query<'tcx, Q>(
-    query: Q,
+fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     span: Span,
-    key: Q::Key,
+    key: C::Key,
     latch: QueryLatch<'tcx>,
     current: Option,
-) -> (Q::Value, Option)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (C::Value, Option) {
     // For parallel queries, we'll block and wait until the query running
     // in another thread has completed. Record how long we wait in the
     // self-profiler.
@@ -275,16 +267,13 @@ where
 }
 
 #[inline(never)]
-fn try_execute_query<'tcx, Q, const INCR: bool>(
-    query: Q,
+fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     span: Span,
-    key: Q::Key,
+    key: C::Key,
     dep_node: Option,
-) -> (Q::Value, Option)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (C::Value, Option) {
     let state = query.query_state(qcx);
     let key_hash = sharded::make_hash(&key);
     let mut state_lock = state.active.lock_shard_by_hash(key_hash);
@@ -315,7 +304,7 @@ where
             // Drop the lock before we start executing the query
             drop(state_lock);
 
-            execute_job::(query, qcx, state, key, key_hash, id, dep_node)
+            execute_job::(query, qcx, state, key, key_hash, id, dep_node)
         }
         Entry::Occupied(mut entry) => {
             match &mut entry.get_mut().1 {
@@ -344,18 +333,15 @@ where
 }
 
 #[inline(always)]
-fn execute_job<'tcx, Q, const INCR: bool>(
-    query: Q,
+fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
-    state: &'tcx QueryState<'tcx, Q::Key>,
-    key: Q::Key,
+    state: &'tcx QueryState<'tcx, C::Key>,
+    key: C::Key,
     key_hash: u64,
     id: QueryJobId,
     dep_node: Option,
-) -> (Q::Value, Option)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (C::Value, Option) {
     // Use `JobOwner` so the query will be poisoned if executing it panics.
     let job_owner = JobOwner { state, key };
 
@@ -409,15 +395,12 @@ where
 
 // Fast path for when incr. comp. is off.
 #[inline(always)]
-fn execute_job_non_incr<'tcx, Q>(
-    query: Q,
+fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
-    key: Q::Key,
+    key: C::Key,
     job_id: QueryJobId,
-) -> (Q::Value, DepNodeIndex)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (C::Value, DepNodeIndex) {
     debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled());
 
     // Fingerprint the key, just to assert that it doesn't
@@ -445,17 +428,14 @@ where
 }
 
 #[inline(always)]
-fn execute_job_incr<'tcx, Q>(
-    query: Q,
+fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     dep_graph_data: &DepGraphData,
-    key: Q::Key,
+    key: C::Key,
     mut dep_node_opt: Option,
     job_id: QueryJobId,
-) -> (Q::Value, DepNodeIndex)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (C::Value, DepNodeIndex) {
     if !query.anon() && !query.eval_always() {
         // `to_dep_node` is expensive for some `DepKind`s.
         let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(qcx.tcx, &key));
@@ -495,16 +475,13 @@ where
 }
 
 #[inline(always)]
-fn try_load_from_disk_and_cache_in_memory<'tcx, Q>(
-    query: Q,
+fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     dep_graph_data: &DepGraphData,
     qcx: QueryCtxt<'tcx>,
-    key: &Q::Key,
+    key: &C::Key,
     dep_node: &DepNode,
-) -> Option<(Q::Value, DepNodeIndex)>
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> Option<(C::Value, DepNodeIndex)> {
     // Note this function can be called concurrently from the same query
     // We must ensure that this is handled correctly.
 
@@ -598,15 +575,12 @@ where
 ///
 /// Note: The optimization is only available during incr. comp.
 #[inline(never)]
-fn ensure_must_run<'tcx, Q>(
-    query: Q,
+fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
-    key: &Q::Key,
+    key: &C::Key,
     check_cache: bool,
-) -> (bool, Option)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> (bool, Option) {
     if query.eval_always() {
         return (true, None);
     }
@@ -644,31 +618,25 @@ where
 }
 
 #[inline(always)]
-pub(super) fn get_query_non_incr<'tcx, Q>(
-    query: Q,
+pub(super) fn get_query_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     span: Span,
-    key: Q::Key,
-) -> Q::Value
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+    key: C::Key,
+) -> C::Value {
     debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled());
 
-    ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, None).0)
+    ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, None).0)
 }
 
 #[inline(always)]
-pub(super) fn get_query_incr<'tcx, Q>(
-    query: Q,
+pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
     span: Span,
-    key: Q::Key,
+    key: C::Key,
     mode: QueryMode,
-) -> Option
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+) -> Option {
     debug_assert!(qcx.tcx.dep_graph.is_fully_enabled());
 
     let dep_node = if let QueryMode::Ensure { check_cache } = mode {
@@ -681,18 +649,21 @@ where
         None
     };
 
-    let (result, dep_node_index) =
-        ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, dep_node));
+    let (result, dep_node_index) = ensure_sufficient_stack(|| {
+        try_execute_query::(query, qcx, span, key, dep_node)
+    });
     if let Some(dep_node_index) = dep_node_index {
         qcx.tcx.dep_graph.read_index(dep_node_index)
     }
     Some(result)
 }
 
-pub(super) fn force_query<'tcx, Q>(query: Q, qcx: QueryCtxt<'tcx>, key: Q::Key, dep_node: DepNode)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+    qcx: QueryCtxt<'tcx>,
+    key: C::Key,
+    dep_node: DepNode,
+) {
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
     if let Some((_, index)) = query.query_cache(qcx).lookup(&key) {
@@ -703,6 +674,6 @@ where
     debug_assert!(!query.anon());
 
     ensure_sufficient_stack(|| {
-        try_execute_query::(query, qcx, DUMMY_SP, key, Some(dep_node))
+        try_execute_query::(query, qcx, DUMMY_SP, key, Some(dep_node))
     });
 }
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 327a8f4bc6c4..7ca4a1300fe6 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -12,7 +12,7 @@ use std::marker::ConstParamTy;
 
 use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
-use rustc_middle::dep_graph::{self, DepKind, DepKindVTable, DepNodeIndex};
+use rustc_middle::dep_graph::{self, DepKind, DepKindVTable, DepNode, DepNodeIndex};
 use rustc_middle::queries::{
     self, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates,
 };
@@ -23,8 +23,7 @@ use rustc_middle::query::values::Value;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::query::{
-    CycleError, CycleErrorHandling, HashResult, QueryCache, QueryDispatcher, QueryMap, QueryMode,
-    QueryState,
+    CycleError, CycleErrorHandling, HashResult, QueryCache, QueryMap, QueryMode, QueryState,
 };
 use rustc_span::{ErrorGuaranteed, Span};
 
@@ -50,8 +49,8 @@ struct QueryFlags {
     is_feedable: bool,
 }
 
-/// Combines a [`QueryVTable`] with some additional compile-time booleans
-/// to implement [`QueryDispatcher`], for use by code in [`rustc_query_system`].
+/// Combines a [`QueryVTable`] with some additional compile-time booleans.
+/// "Dispatcher" should be understood as a near-synonym of "vtable".
 ///
 /// Baking these boolean flags into the type gives a modest but measurable
 /// improvement to compiler perf and compiler code size; see
@@ -73,54 +72,49 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> Clone
     }
 }
 
-// This is `impl QueryDispatcher for SemiDynamicQueryDispatcher`.
-impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> QueryDispatcher<'tcx>
-    for SemiDynamicQueryDispatcher<'tcx, C, FLAGS>
-{
-    type Qcx = QueryCtxt<'tcx>;
-    type Key = C::Key;
-    type Value = C::Value;
-    type Cache = C;
-
+impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'tcx, C, FLAGS> {
     #[inline(always)]
     fn name(self) -> &'static str {
         self.vtable.name
     }
 
     #[inline(always)]
-    fn will_cache_on_disk_for_key(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
+    fn will_cache_on_disk_for_key(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> bool {
         self.vtable.will_cache_on_disk_for_key_fn.map_or(false, |f| f(tcx, key))
     }
 
+    // Don't use this method to access query results, instead use the methods on TyCtxt.
     #[inline(always)]
-    fn query_state(self, qcx: QueryCtxt<'tcx>) -> &'tcx QueryState<'tcx, Self::Key> {
+    fn query_state(self, qcx: QueryCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> {
         // Safety:
         // This is just manually doing the subfield referencing through pointer math.
         unsafe {
             &*(&qcx.tcx.query_system.states as *const QueryStates<'tcx>)
                 .byte_add(self.vtable.query_state)
-                .cast::>()
+                .cast::>()
         }
     }
 
+    // Don't use this method to access query results, instead use the methods on TyCtxt.
     #[inline(always)]
-    fn query_cache(self, qcx: QueryCtxt<'tcx>) -> &'tcx Self::Cache {
+    fn query_cache(self, qcx: QueryCtxt<'tcx>) -> &'tcx C {
         // Safety:
         // This is just manually doing the subfield referencing through pointer math.
         unsafe {
             &*(&qcx.tcx.query_system.caches as *const QueryCaches<'tcx>)
                 .byte_add(self.vtable.query_cache)
-                .cast::()
+                .cast::()
         }
     }
 
+    // Don't use this method to compute query results, instead use the methods on TyCtxt.
     #[inline(always)]
-    fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+    fn execute_query(self, tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value {
         (self.vtable.execute_query)(tcx, key)
     }
 
     #[inline(always)]
-    fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
+    fn compute(self, qcx: QueryCtxt<'tcx>, key: C::Key) -> C::Value {
         (self.vtable.compute_fn)(qcx.tcx, key)
     }
 
@@ -128,10 +122,10 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> QueryDispatcher<'tcx>
     fn try_load_from_disk(
         self,
         qcx: QueryCtxt<'tcx>,
-        key: &Self::Key,
+        key: &C::Key,
         prev_index: SerializedDepNodeIndex,
         index: DepNodeIndex,
-    ) -> Option {
+    ) -> Option {
         // `?` will return None immediately for queries that never cache to disk.
         self.vtable.try_load_from_disk_fn?(qcx.tcx, key, prev_index, index)
     }
@@ -140,23 +134,24 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> QueryDispatcher<'tcx>
     fn is_loadable_from_disk(
         self,
         qcx: QueryCtxt<'tcx>,
-        key: &Self::Key,
+        key: &C::Key,
         index: SerializedDepNodeIndex,
     ) -> bool {
         self.vtable.is_loadable_from_disk_fn.map_or(false, |f| f(qcx.tcx, key, index))
     }
 
+    /// Synthesize an error value to let compilation continue after a cycle.
     fn value_from_cycle_error(
         self,
         tcx: TyCtxt<'tcx>,
         cycle_error: &CycleError,
         guar: ErrorGuaranteed,
-    ) -> Self::Value {
+    ) -> C::Value {
         (self.vtable.value_from_cycle_error)(tcx, cycle_error, guar)
     }
 
     #[inline(always)]
-    fn format_value(self) -> fn(&Self::Value) -> String {
+    fn format_value(self) -> fn(&C::Value) -> String {
         self.vtable.format_value
     }
 
@@ -191,13 +186,17 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> QueryDispatcher<'tcx>
     }
 
     #[inline(always)]
-    fn hash_result(self) -> HashResult {
+    fn hash_result(self) -> HashResult {
         self.vtable.hash_result
     }
+
+    fn construct_dep_node(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> DepNode {
+        DepNode::construct(tcx, self.dep_kind(), key)
+    }
 }
 
 /// Provides access to vtable-like operations for a query
-/// (by creating a [`QueryDispatcher`]),
+/// (by creating a [`SemiDynamicQueryDispatcher`]),
 /// but also keeps track of the "unerased" value type of the query
 /// (i.e. the actual result type in the query declaration).
 ///
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index aefd4418e192..776d720f500b 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -28,8 +28,8 @@ use rustc_middle::ty::tls::{self, ImplicitCtxt};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::{DepNodeKey, FingerprintStyle, HasDepContext};
 use rustc_query_system::query::{
-    QueryCache, QueryContext, QueryDispatcher, QueryJobId, QueryMap, QuerySideEffect,
-    QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra,
+    QueryCache, QueryContext, QueryJobId, QueryMap, QuerySideEffect, QueryStackDeferred,
+    QueryStackFrame, QueryStackFrameExtra,
 };
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::def_id::LOCAL_CRATE;
@@ -404,8 +404,8 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: Quer
     });
 }
 
-pub(crate) fn query_key_hash_verify<'tcx>(
-    query: impl QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
+pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
     qcx: QueryCtxt<'tcx>,
 ) {
     let _timer = qcx.tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name());
@@ -430,13 +430,14 @@ pub(crate) fn query_key_hash_verify<'tcx>(
     });
 }
 
-fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+fn try_load_from_on_disk_cache<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+    tcx: TyCtxt<'tcx>,
+    dep_node: DepNode,
+) {
     debug_assert!(tcx.dep_graph.is_green(&dep_node));
 
-    let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
+    let key = C::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
         panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
     });
     if query.will_cache_on_disk_for_key(tcx, &key) {
@@ -476,10 +477,11 @@ where
     value
 }
 
-fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
-where
-    Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
-{
+fn force_from_dep_node<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
+    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+    tcx: TyCtxt<'tcx>,
+    dep_node: DepNode,
+) -> bool {
     // We must avoid ever having to call `force_from_dep_node()` for a
     // `DepNode::codegen_unit`:
     // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
@@ -498,7 +500,7 @@ where
         "calling force_from_dep_node() on dep_kinds::codegen_unit"
     );
 
-    if let Some(key) = Q::Key::recover(tcx, &dep_node) {
+    if let Some(key) = C::Key::recover(tcx, &dep_node) {
         force_query(query, QueryCtxt::new(tcx), key, dep_node);
         true
     } else {
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index c1f5e5b67085..eea1c5872290 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -21,6 +21,9 @@ pub trait QueryCacheKey = Hash + Eq + Copy + Debug + for<'a> HashStable = Option, &V) -> Fingerprint>;
-
-/// Unambiguous shorthand for `::DepContext`.
-#[expect(type_alias_bounds)]
-type DepContextOf<'tcx, This: QueryDispatcher<'tcx>> =
-    <>::Qcx as HasDepContext>::DepContext;
-
-/// Trait that can be used as a vtable for a single query, providing operations
-/// and metadata for that query.
-///
-/// Implemented by `rustc_query_impl::SemiDynamicQueryDispatcher`, which
-/// mostly delegates to `rustc_middle::query::plumbing::QueryVTable`.
-/// Those types are not visible from this `rustc_query_system` crate.
-///
-/// "Dispatcher" should be understood as a near-synonym of "vtable".
-pub trait QueryDispatcher<'tcx>: Copy + 'tcx {
-    fn name(self) -> &'static str;
-
-    /// Query context used by this dispatcher, i.e. `rustc_query_impl::QueryCtxt`.
-    type Qcx: QueryContext<'tcx>;
-
-    // `Key` and `Value` are `Copy` instead of `Clone` to ensure copying them stays cheap,
-    // but it isn't necessary.
-    type Key: DepNodeKey> + Eq + Hash + Copy + Debug;
-    type Value: Copy;
-
-    type Cache: QueryCache;
-
-    fn format_value(self) -> fn(&Self::Value) -> String;
-
-    // Don't use this method to access query results, instead use the methods on TyCtxt
-    fn query_state(self, tcx: Self::Qcx) -> &'tcx QueryState<'tcx, Self::Key>;
-
-    // Don't use this method to access query results, instead use the methods on TyCtxt
-    fn query_cache(self, tcx: Self::Qcx) -> &'tcx Self::Cache;
-
-    fn will_cache_on_disk_for_key(self, tcx: DepContextOf<'tcx, Self>, key: &Self::Key) -> bool;
-
-    // Don't use this method to compute query results, instead use the methods on TyCtxt
-    fn execute_query(self, tcx: DepContextOf<'tcx, Self>, k: Self::Key) -> Self::Value;
-
-    fn compute(self, tcx: Self::Qcx, key: Self::Key) -> Self::Value;
-
-    fn try_load_from_disk(
-        self,
-        tcx: Self::Qcx,
-        key: &Self::Key,
-        prev_index: SerializedDepNodeIndex,
-        index: DepNodeIndex,
-    ) -> Option;
-
-    fn is_loadable_from_disk(
-        self,
-        qcx: Self::Qcx,
-        key: &Self::Key,
-        idx: SerializedDepNodeIndex,
-    ) -> bool;
-
-    /// Synthesize an error value to let compilation continue after a cycle.
-    fn value_from_cycle_error(
-        self,
-        tcx: DepContextOf<'tcx, Self>,
-        cycle_error: &CycleError,
-        guar: ErrorGuaranteed,
-    ) -> Self::Value;
-
-    fn anon(self) -> bool;
-    fn eval_always(self) -> bool;
-    fn depth_limit(self) -> bool;
-    fn feedable(self) -> bool;
-
-    fn dep_kind(self) -> DepKind;
-    fn cycle_error_handling(self) -> CycleErrorHandling;
-    fn hash_result(self) -> HashResult;
-
-    // Just here for convenience and checking that the key matches the kind, don't override this.
-    fn construct_dep_node(self, tcx: DepContextOf<'tcx, Self>, key: &Self::Key) -> DepNode {
-        DepNode::construct(tcx, self.dep_kind(), key)
-    }
-}
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 01bb6ff09fa1..828502b47d4c 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -15,7 +15,7 @@ use rustc_span::def_id::DefId;
 pub use self::caches::{
     DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache,
 };
-pub use self::dispatcher::{HashResult, QueryDispatcher};
+pub use self::dispatcher::HashResult;
 pub use self::job::{
     QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, break_query_cycles,
     print_query_stack, report_cycle,

From f9958048e0e9a969848dfd24a9432a40de054429 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Mon, 9 Feb 2026 13:28:44 +1100
Subject: [PATCH 1490/1843] Move `HashResult`.

It's the only thing left in `rustc_query_system::query::dispatcher`.
---
 compiler/rustc_middle/src/query/plumbing.rs         | 6 +++++-
 compiler/rustc_query_impl/src/lib.rs                | 4 ++--
 compiler/rustc_query_system/src/query/dispatcher.rs | 5 -----
 compiler/rustc_query_system/src/query/mod.rs        | 2 --
 4 files changed, 7 insertions(+), 10 deletions(-)
 delete mode 100644 compiler/rustc_query_system/src/query/dispatcher.rs

diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs
index abb02caef25d..04a3ceb0918a 100644
--- a/compiler/rustc_middle/src/query/plumbing.rs
+++ b/compiler/rustc_middle/src/query/plumbing.rs
@@ -1,12 +1,14 @@
 use std::ops::Deref;
 
+use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::sync::{AtomicU64, WorkerLocal};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::hir_id::OwnerId;
 use rustc_macros::HashStable;
 use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use rustc_query_system::ich::StableHashingContext;
 pub(crate) use rustc_query_system::query::QueryJobId;
-use rustc_query_system::query::{CycleError, CycleErrorHandling, HashResult, QueryCache};
+use rustc_query_system::query::{CycleError, CycleErrorHandling, QueryCache};
 use rustc_span::{ErrorGuaranteed, Span};
 pub use sealed::IntoQueryParam;
 
@@ -30,6 +32,8 @@ pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn(
 pub type IsLoadableFromDiskFn<'tcx, Key> =
     fn(tcx: TyCtxt<'tcx>, key: &Key, index: SerializedDepNodeIndex) -> bool;
 
+pub type HashResult = Option, &V) -> Fingerprint>;
+
 /// Stores function pointers and other metadata for a particular query.
 ///
 /// Used indirectly by query plumbing in `rustc_query_system` via a trait,
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 7ca4a1300fe6..a33bdd22a797 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -18,12 +18,12 @@ use rustc_middle::queries::{
 };
 use rustc_middle::query::AsLocalKey;
 use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache};
-use rustc_middle::query::plumbing::{QuerySystem, QuerySystemFns, QueryVTable};
+use rustc_middle::query::plumbing::{HashResult, QuerySystem, QuerySystemFns, QueryVTable};
 use rustc_middle::query::values::Value;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::query::{
-    CycleError, CycleErrorHandling, HashResult, QueryCache, QueryMap, QueryMode, QueryState,
+    CycleError, CycleErrorHandling, QueryCache, QueryMap, QueryMode, QueryState,
 };
 use rustc_span::{ErrorGuaranteed, Span};
 
diff --git a/compiler/rustc_query_system/src/query/dispatcher.rs b/compiler/rustc_query_system/src/query/dispatcher.rs
deleted file mode 100644
index 3ef8295a07c3..000000000000
--- a/compiler/rustc_query_system/src/query/dispatcher.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-use rustc_data_structures::fingerprint::Fingerprint;
-
-use crate::ich::StableHashingContext;
-
-pub type HashResult = Option, &V) -> Fingerprint>;
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 828502b47d4c..f4a3fda7e372 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -15,7 +15,6 @@ use rustc_span::def_id::DefId;
 pub use self::caches::{
     DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache,
 };
-pub use self::dispatcher::HashResult;
 pub use self::job::{
     QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, break_query_cycles,
     print_query_stack, report_cycle,
@@ -24,7 +23,6 @@ pub use self::plumbing::*;
 use crate::dep_graph::{DepKind, DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
 
 mod caches;
-mod dispatcher;
 mod job;
 mod plumbing;
 

From eb2ba7d6b9429297c582fed42fd5da5e5a51cd8f Mon Sep 17 00:00:00 2001
From: Takayuki Maeda 
Date: Mon, 9 Feb 2026 18:18:24 +0900
Subject: [PATCH 1491/1843] fix ICE in supertrait_vtable_slot when supertrait
 has missing generics

---
 .../src/traits/vtable.rs                      | 11 +++-
 .../vtable/missing-generics-issue-151330.rs   | 23 ++++++++
 .../missing-generics-issue-151330.stderr      | 53 +++++++++++++++++++
 3 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/traits/vtable/missing-generics-issue-151330.rs
 create mode 100644 tests/ui/traits/vtable/missing-generics-issue-151330.stderr

diff --git a/compiler/rustc_trait_selection/src/traits/vtable.rs b/compiler/rustc_trait_selection/src/traits/vtable.rs
index 584c8e2a27c8..539c1d3a35ba 100644
--- a/compiler/rustc_trait_selection/src/traits/vtable.rs
+++ b/compiler/rustc_trait_selection/src/traits/vtable.rs
@@ -434,7 +434,16 @@ pub(crate) fn supertrait_vtable_slot<'tcx>(
         }
     };
 
-    prepare_vtable_segments(tcx, source_principal, vtable_segment_callback).unwrap()
+    prepare_vtable_segments(tcx, source_principal, vtable_segment_callback).unwrap_or_else(|| {
+        // This can happen if the trait hierarchy is malformed (e.g., due to
+        // missing generics on a supertrait bound). There should already be an error
+        // emitted for this, so we just delay the ICE.
+        tcx.dcx().delayed_bug(format!(
+            "could not find the supertrait vtable slot for `{}` -> `{}`",
+            source, target
+        ));
+        None
+    })
 }
 
 pub(super) fn provide(providers: &mut Providers) {
diff --git a/tests/ui/traits/vtable/missing-generics-issue-151330.rs b/tests/ui/traits/vtable/missing-generics-issue-151330.rs
new file mode 100644
index 000000000000..352ad7be3d3a
--- /dev/null
+++ b/tests/ui/traits/vtable/missing-generics-issue-151330.rs
@@ -0,0 +1,23 @@
+//@ compile-flags: -Znext-solver=globally
+// Regression test for issue https://github.com/rust-lang/rust/issues/151330
+
+trait Supertrait {}
+
+trait Trait
: Supertrait {}
+//~^ ERROR missing generics for trait `Supertrait`
+//~| ERROR missing generics for trait `Supertrait`
+//~| ERROR missing generics for trait `Supertrait`
+
+impl
 Trait
 for () {}
+
+const fn upcast
(x: &dyn Trait
) -> &dyn Trait
 {
+    x
+}
+
+const fn foo() -> &'static dyn Supertrait<()> {
+    upcast::<()>(&())
+}
+
+const _: &'static dyn Supertrait<()> = foo();
+
+fn main() {}
diff --git a/tests/ui/traits/vtable/missing-generics-issue-151330.stderr b/tests/ui/traits/vtable/missing-generics-issue-151330.stderr
new file mode 100644
index 000000000000..65ef08cad32a
--- /dev/null
+++ b/tests/ui/traits/vtable/missing-generics-issue-151330.stderr
@@ -0,0 +1,53 @@
+error[E0107]: missing generics for trait `Supertrait`
+  --> $DIR/missing-generics-issue-151330.rs:6:17
+   |
+LL | trait Trait
: Supertrait {}
+   |                 ^^^^^^^^^^ expected 1 generic argument
+   |
+note: trait defined here, with 1 generic parameter: `T`
+  --> $DIR/missing-generics-issue-151330.rs:4:7
+   |
+LL | trait Supertrait {}
+   |       ^^^^^^^^^^ -
+help: add missing generic argument
+   |
+LL | trait Trait
: Supertrait {}
+   |                           +++
+
+error[E0107]: missing generics for trait `Supertrait`
+  --> $DIR/missing-generics-issue-151330.rs:6:17
+   |
+LL | trait Trait
: Supertrait {}
+   |                 ^^^^^^^^^^ expected 1 generic argument
+   |
+note: trait defined here, with 1 generic parameter: `T`
+  --> $DIR/missing-generics-issue-151330.rs:4:7
+   |
+LL | trait Supertrait {}
+   |       ^^^^^^^^^^ -
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing generic argument
+   |
+LL | trait Trait
: Supertrait {}
+   |                           +++
+
+error[E0107]: missing generics for trait `Supertrait`
+  --> $DIR/missing-generics-issue-151330.rs:6:17
+   |
+LL | trait Trait
: Supertrait {}
+   |                 ^^^^^^^^^^ expected 1 generic argument
+   |
+note: trait defined here, with 1 generic parameter: `T`
+  --> $DIR/missing-generics-issue-151330.rs:4:7
+   |
+LL | trait Supertrait {}
+   |       ^^^^^^^^^^ -
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing generic argument
+   |
+LL | trait Trait
: Supertrait {}
+   |                           +++
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0107`.

From 3339b061af6a9c37ae326f60895048ddbbf3cd6d Mon Sep 17 00:00:00 2001
From: Oli Scherer 
Date: Sat, 7 Feb 2026 19:24:59 +0000
Subject: [PATCH 1492/1843] Do not require `'static` for obtaining reflection
 information.

---
 library/core/src/intrinsics/mod.rs |  2 +-
 library/core/src/mem/type_info.rs  | 14 ++++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs
index 051dda731881..094def9796db 100644
--- a/library/core/src/intrinsics/mod.rs
+++ b/library/core/src/intrinsics/mod.rs
@@ -2887,7 +2887,7 @@ pub const fn type_name() -> &'static str;
 #[rustc_nounwind]
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[rustc_intrinsic]
-pub const fn type_id() -> crate::any::TypeId;
+pub const fn type_id() -> crate::any::TypeId;
 
 /// Tests (at compile-time) if two [`crate::any::TypeId`] instances identify the
 /// same type. This is necessary because at const-eval time the actual discriminating
diff --git a/library/core/src/mem/type_info.rs b/library/core/src/mem/type_info.rs
index 8b30803c97c9..d3a7421ff2ee 100644
--- a/library/core/src/mem/type_info.rs
+++ b/library/core/src/mem/type_info.rs
@@ -2,7 +2,7 @@
 //! runtime or const-eval processable way.
 
 use crate::any::TypeId;
-use crate::intrinsics::type_of;
+use crate::intrinsics::{type_id, type_of};
 
 /// Compile-time type information.
 #[derive(Debug)]
@@ -28,11 +28,17 @@ impl TypeId {
 
 impl Type {
     /// Returns the type information of the generic type parameter.
+    ///
+    /// Note: Unlike `TypeId`s obtained via `TypeId::of`, the `Type`
+    /// struct and its fields contain `TypeId`s that are not necessarily
+    /// derived from types that outlive `'static`. This means that using
+    /// the `TypeId`s (transitively) obtained from this function will
+    /// be able to break invariants that other `TypeId` consuming crates
+    /// may have assumed to hold.
     #[unstable(feature = "type_info", issue = "146922")]
     #[rustc_const_unstable(feature = "type_info", issue = "146922")]
-    // FIXME(reflection): don't require the 'static bound
-    pub const fn of() -> Self {
-        const { TypeId::of::().info() }
+    pub const fn of() -> Self {
+        const { type_id::().info() }
     }
 }
 

From 6babe687a50440ae6a1d60a748f6a1f809b56d7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Thu, 5 Feb 2026 15:57:14 +0100
Subject: [PATCH 1493/1843] Port rustc_reservation_impl to the new attribute
 parser

---
 .../src/attributes/rustc_internal.rs          | 26 +++++++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs    |  1 +
 .../rustc_hir/src/attrs/data_structures.rs    |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 compiler/rustc_hir_analysis/src/collect.rs    |  3 ++-
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 .../src/traits/coherence.rs                   |  9 +++----
 .../src/traits/select/mod.rs                  |  9 +++----
 8 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index cbf4af8ce4f4..0fd8c8a04a57 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -873,3 +873,29 @@ impl NoArgsAttributeParser for RustcStrictCoherenceParser {
     ]);
     const CREATE: fn(Span) -> AttributeKind = AttributeKind::RustcStrictCoherence;
 }
+
+pub(crate) struct RustcReservationImplParser;
+
+impl SingleAttributeParser for RustcReservationImplParser {
+    const PATH: &[Symbol] = &[sym::rustc_reservation_impl];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Impl { of_trait: true })]);
+
+    const TEMPLATE: AttributeTemplate = template!(NameValueStr: "reservation message");
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option {
+        let Some(nv) = args.name_value() else {
+            cx.expected_name_value(args.span().unwrap_or(cx.attr_span), None);
+            return None;
+        };
+
+        let Some(value_str) = nv.value_as_str() else {
+            cx.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
+            return None;
+        };
+
+        Some(AttributeKind::RustcReservationImpl(cx.attr_span, value_str))
+    }
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index afbb70654f4d..ddb5ae33bc98 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -209,6 +209,7 @@ attribute_parsers!(
         Single,
         Single,
         Single,
+        Single,
         Single,
         Single,
         Single,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index ef790fe76b74..d9c4179cca83 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1219,6 +1219,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_regions]`
     RustcRegions,
 
+    /// Represents `#[rustc_reservation_impl]`
+    RustcReservationImpl(Span, Symbol),
+
     /// Represents `#[rustc_scalable_vector(N)]`
     RustcScalableVector {
         /// The base multiple of lanes that are in a scalable vector, if provided. `element_count`
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index cba7679d2df3..d55cb1d6f66a 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -151,6 +151,7 @@ impl AttributeKind {
             RustcPubTransparent(..) => Yes,
             RustcReallocator => No,
             RustcRegions => No,
+            RustcReservationImpl(..) => Yes,
             RustcScalableVector { .. } => Yes,
             RustcShouldNotBeCalledOnConstItems(..) => Yes,
             RustcSimdMonomorphizeLaneLimit(..) => Yes, // Affects layout computation, which needs to work cross-crate
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index f3086266eec0..219e44cd2e27 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -1275,7 +1275,8 @@ fn impl_trait_header(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::ImplTraitHeader
         .of_trait
         .unwrap_or_else(|| panic!("expected impl trait, found inherent impl on {def_id:?}"));
     let selfty = tcx.type_of(def_id).instantiate_identity();
-    let is_rustc_reservation = tcx.has_attr(def_id, sym::rustc_reservation_impl);
+    let is_rustc_reservation =
+        find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcReservationImpl(..));
 
     check_impl_constness(tcx, impl_.constness, &of_trait.trait_ref);
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index dd0de90d575b..3f89ff577d59 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -342,6 +342,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcPreserveUbChecks
                     | AttributeKind::RustcReallocator
                     | AttributeKind::RustcRegions
+                    | AttributeKind::RustcReservationImpl(..)
                     | AttributeKind::RustcScalableVector { .. }
                     | AttributeKind::RustcShouldNotBeCalledOnConstItems(..)
                     | AttributeKind::RustcSimdMonomorphizeLaneLimit(..)
@@ -396,7 +397,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_trivial_field_reads
                             | sym::rustc_on_unimplemented
                             | sym::rustc_do_not_const_check
-                            | sym::rustc_reservation_impl
                             | sym::rustc_doc_primitive
                             | sym::rustc_conversion_suggestion
                             | sym::rustc_deprecated_safe_2024
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index 9f59f6c59250..fc628e78a3e2 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -8,8 +8,10 @@ use std::fmt::Debug;
 
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_errors::{Diag, EmissionGuarantee};
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId};
+use rustc_hir::find_attr;
 use rustc_infer::infer::{DefineOpaqueTypes, InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::PredicateObligations;
 use rustc_macros::{TypeFoldable, TypeVisitable};
@@ -23,7 +25,7 @@ use rustc_middle::ty::{
 };
 pub use rustc_next_trait_solver::coherence::*;
 use rustc_next_trait_solver::solve::SolverDelegateEvalExt;
-use rustc_span::{DUMMY_SP, Span, sym};
+use rustc_span::{DUMMY_SP, Span};
 use tracing::{debug, instrument, warn};
 
 use super::ObligationCtxt;
@@ -758,10 +760,7 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a, 'tcx> {
             } = cand.kind()
                 && let ty::ImplPolarity::Reservation = infcx.tcx.impl_polarity(def_id)
             {
-                let message = infcx
-                    .tcx
-                    .get_attr(def_id, sym::rustc_reservation_impl)
-                    .and_then(|a| a.value_str());
+                let message = find_attr!(infcx.tcx.get_all_attrs(def_id), AttributeKind::RustcReservationImpl(_, message) => *message);
                 if let Some(message) = message {
                     self.causes.insert(IntercrateAmbiguityCause::ReservationImpl { message });
                 }
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 787dd4ea6254..4469afc3a4eb 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -12,9 +12,9 @@ use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{Diag, EmissionGuarantee};
-use rustc_hir as hir;
-use rustc_hir::LangItem;
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def_id::DefId;
+use rustc_hir::{self as hir, LangItem, find_attr};
 use rustc_infer::infer::BoundRegionConversionTime::{self, HigherRankedType};
 use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::infer::at::ToTrace;
@@ -33,7 +33,7 @@ use rustc_middle::ty::{
     may_use_unstable_feature,
 };
 use rustc_next_trait_solver::solve::AliasBoundKind;
-use rustc_span::{Symbol, sym};
+use rustc_span::Symbol;
 use tracing::{debug, instrument, trace};
 
 use self::EvaluationResult::*;
@@ -1445,8 +1445,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             && let ty::ImplPolarity::Reservation = tcx.impl_polarity(def_id)
         {
             if let Some(intercrate_ambiguity_clauses) = &mut self.intercrate_ambiguity_causes {
-                let message =
-                    tcx.get_attr(def_id, sym::rustc_reservation_impl).and_then(|a| a.value_str());
+                let message = find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcReservationImpl(_, message) => *message);
                 if let Some(message) = message {
                     debug!(
                         "filter_reservation_impls: \

From 88c296a19fb205e08edd9636a7ccf94b0346720c Mon Sep 17 00:00:00 2001
From: xizheyin 
Date: Mon, 9 Feb 2026 18:24:11 +0800
Subject: [PATCH 1494/1843] std: introduce path normalize methods at top of
 std::path

---
 library/std/src/path.rs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/library/std/src/path.rs b/library/std/src/path.rs
index 25bd7005b994..14b41a427f1e 100644
--- a/library/std/src/path.rs
+++ b/library/std/src/path.rs
@@ -19,6 +19,20 @@
 //! matter the platform or filesystem. An exception to this is made for Windows
 //! drive letters.
 //!
+//! ## Path normalization
+//!
+//! Several methods in this module perform basic path normalization by disregarding
+//! repeated separators, non-leading `.` components, and trailing separators. These include:
+//! - Methods for iteration, such as [`Path::components`] and [`Path::iter`]
+//! - Methods for inspection, such as [`Path::has_root`]
+//! - Comparisons using [`PartialEq`], [`PartialOrd`], and [`Ord`]
+//!
+//! [`Path::join`] and [`PathBuf::push`] also disregard trailing slashes.
+//!
+// FIXME(normalize_lexically): mention normalize_lexically once stable
+//! These methods **do not** resolve `..` components or symlinks. For full normalization
+//! including `..` resolution, use [`Path::canonicalize`] (which does access the filesystem).
+//!
 //! ## Simple usage
 //!
 //! Path manipulation includes both parsing components from slices and building

From 8cd47adab253c6ec0eea949563215fa7fdbc7241 Mon Sep 17 00:00:00 2001
From: Juho Kahala <57393910+quaternic@users.noreply.github.com>
Date: Mon, 9 Feb 2026 12:32:04 +0200
Subject: [PATCH 1495/1843] libm: Fix tests for lgamma

The tests were using `rug::ln_gamma` as a reference for `libm::lgamma`,
which actually computes the natural logarithm *of the absolute value* of
the Gamma function.

This changes the range of inputs used for the icount-benchmarks of these
functions, which causes false regressions in [1].

[1]: https://github.com/rust-lang/compiler-builtins/actions/runs/21788698368/job/62864230903?pr=1075#step:7:2710.

Fixes: a1a066611dc2 ("Create interfaces for testing against MPFR")
---
 .../compiler-builtins/libm-test/src/domain.rs |  2 +-
 .../libm-test/src/mpfloat.rs                  | 27 +++++++++++++-
 .../libm-test/src/precision.rs                | 36 +++++++++----------
 3 files changed, 45 insertions(+), 20 deletions(-)

diff --git a/library/compiler-builtins/libm-test/src/domain.rs b/library/compiler-builtins/libm-test/src/domain.rs
index 94641be9b548..eb009bfa093f 100644
--- a/library/compiler-builtins/libm-test/src/domain.rs
+++ b/library/compiler-builtins/libm-test/src/domain.rs
@@ -207,7 +207,7 @@ impl EitherPrim, Domain> {
     .into_prim_float()];
 
     /// Domain for `loggamma`
-    const LGAMMA: [Self; 1] = Self::STRICTLY_POSITIVE;
+    const LGAMMA: [Self; 1] = Self::UNBOUNDED1;
 
     /// Domain for `jn` and `yn`.
     // FIXME: the domain should provide some sort of "reasonable range" so we don't actually test
diff --git a/library/compiler-builtins/libm-test/src/mpfloat.rs b/library/compiler-builtins/libm-test/src/mpfloat.rs
index 9b51dc6051d0..85f0a4da4a6e 100644
--- a/library/compiler-builtins/libm-test/src/mpfloat.rs
+++ b/library/compiler-builtins/libm-test/src/mpfloat.rs
@@ -170,7 +170,9 @@ libm_macros::for_each_function! {
         ldexpf,
         ldexpf128,
         ldexpf16,
+        lgamma,
         lgamma_r,
+        lgammaf,
         lgammaf_r,
         modf,
         modff,
@@ -213,7 +215,6 @@ libm_macros::for_each_function! {
         fmaximum_num | fmaximum_numf | fmaximum_numf16 | fmaximum_numf128 => max,
         fmin | fminf | fminf16 | fminf128 |
         fminimum_num | fminimum_numf | fminimum_numf16 | fminimum_numf128 => min,
-        lgamma | lgammaf => ln_gamma,
         log | logf => ln,
         log1p | log1pf => ln_1p,
         tgamma | tgammaf => gamma,
@@ -576,6 +577,30 @@ impl MpOp for crate::op::lgammaf_r::Routine {
     }
 }
 
+impl MpOp for crate::op::lgamma::Routine {
+    type MpTy = MpFloat;
+
+    fn new_mp() -> Self::MpTy {
+        new_mpfloat::()
+    }
+
+    fn run(this: &mut Self::MpTy, input: Self::RustArgs) -> Self::RustRet {
+        ::run(this, input).0
+    }
+}
+
+impl MpOp for crate::op::lgammaf::Routine {
+    type MpTy = MpFloat;
+
+    fn new_mp() -> Self::MpTy {
+        new_mpfloat::()
+    }
+
+    fn run(this: &mut Self::MpTy, input: Self::RustArgs) -> Self::RustRet {
+        ::run(this, input).0
+    }
+}
+
 /* stub implementations so we don't need to special case them */
 
 impl MpOp for crate::op::nextafter::Routine {
diff --git a/library/compiler-builtins/libm-test/src/precision.rs b/library/compiler-builtins/libm-test/src/precision.rs
index a94fe429f5f3..5d52da168fe7 100644
--- a/library/compiler-builtins/libm-test/src/precision.rs
+++ b/library/compiler-builtins/libm-test/src/precision.rs
@@ -67,7 +67,7 @@ pub fn default_ulp(ctx: &CheckCtx) -> u32 {
         Bn::Exp2 => 1,
         Bn::Expm1 => 1,
         Bn::Hypot => 1,
-        Bn::Lgamma | Bn::LgammaR => 16,
+        Bn::Lgamma | Bn::LgammaR => 4,
         Bn::Log => 1,
         Bn::Log10 => 1,
         Bn::Log1p => 1,
@@ -102,7 +102,6 @@ pub fn default_ulp(ctx: &CheckCtx) -> u32 {
         match ctx.base_name {
             Bn::Cosh => ulp = 2,
             Bn::Exp10 if usize::BITS < 64 => ulp = 4,
-            Bn::Lgamma | Bn::LgammaR => ulp = 400,
             Bn::Tanh => ulp = 4,
             _ => (),
         }
@@ -218,17 +217,17 @@ impl MaybeOverride<(f16,)> for SpecialCase {}
 
 impl MaybeOverride<(f32,)> for SpecialCase {
     fn check_float(input: (f32,), actual: F, expected: F, ctx: &CheckCtx) -> CheckAction {
-        if (ctx.base_name == BaseName::Lgamma || ctx.base_name == BaseName::LgammaR)
-            && input.0 > 4e36
-            && expected.is_infinite()
-            && !actual.is_infinite()
-        {
-            // This result should saturate but we return a finite value.
+        if ctx.base_name == BaseName::J0 && input.0 < -1e34 {
+            // Errors get huge close to -inf
             return XFAIL_NOCHECK;
         }
 
-        if ctx.base_name == BaseName::J0 && input.0 < -1e34 {
-            // Errors get huge close to -inf
+        // FIXME(correctness): lgammaf has high relative inaccuracy near its zeroes
+        if matches!(ctx.base_name, BaseName::Lgamma | BaseName::LgammaR)
+            && input.0 > -13.0625
+            && input.0 < -2.0
+            && (expected.abs() < F::ONE || (input.0 - input.0.round()).abs() < 0.02)
+        {
             return XFAIL_NOCHECK;
         }
 
@@ -275,6 +274,15 @@ impl MaybeOverride<(f64,)> for SpecialCase {
             return XFAIL_NOCHECK;
         }
 
+        // FIXME(correctness): lgamma has high relative inaccuracy near its zeroes
+        if matches!(ctx.base_name, BaseName::Lgamma | BaseName::LgammaR)
+            && input.0 > -32.0
+            && input.0 < -2.0
+            && (expected.abs() < F::ONE || (input.0 - input.0.round()).abs() < 0.02)
+        {
+            return XFAIL_NOCHECK;
+        }
+
         // maybe_check_nan_bits(actual, expected, ctx)
         unop_common(input, actual, expected, ctx)
     }
@@ -304,14 +312,6 @@ fn unop_common(
     expected: F2,
     ctx: &CheckCtx,
 ) -> CheckAction {
-    if (ctx.base_name == BaseName::Lgamma || ctx.base_name == BaseName::LgammaR)
-        && input.0 < F1::ZERO
-        && !input.0.is_infinite()
-    {
-        // loggamma should not be defined for x < 0, yet we both return results
-        return XFAIL_NOCHECK;
-    }
-
     // fabs and copysign must leave NaNs untouched.
     if ctx.base_name == BaseName::Fabs && input.0.is_nan() {
         // LLVM currently uses x87 instructions which quieten signalling NaNs to handle the i686

From 8c0a493e358a27f1dea9888f85680c613e41903f Mon Sep 17 00:00:00 2001
From: Ralf Jung 
Date: Mon, 9 Feb 2026 12:03:29 +0100
Subject: [PATCH 1496/1843] BikeshedGuaranteedNoDrop trait: add comments
 indicating that it can be observed on stable

---
 compiler/rustc_next_trait_solver/src/solve/trait_goals.rs      | 3 +++
 .../rustc_trait_selection/src/traits/select/confirmation.rs    | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
index 003841f3af3f..dd012a5146e6 100644
--- a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
@@ -687,6 +687,9 @@ where
     ///
     /// because these impls overlap, and I'd rather not build a coherence hack for
     /// this harmless overlap.
+    ///
+    /// This trait is indirectly exposed on stable, so do *not* extend the set of types that
+    /// implement the trait without FCP!
     fn consider_builtin_bikeshed_guaranteed_no_drop_candidate(
         ecx: &mut EvalCtxt<'_, D>,
         goal: Goal,
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 4f65b30775ed..b0e045274d0d 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -1246,6 +1246,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         })
     }
 
+    /// This trait is indirectly exposed on stable, so do *not* extend the set of types that
+    /// implement the trait without FCP!
     fn confirm_bikeshed_guaranteed_no_drop_candidate(
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,

From ae24fdaccc71600ec540ff018208c5d592ad016a Mon Sep 17 00:00:00 2001
From: Lukas Wirth 
Date: Mon, 9 Feb 2026 12:45:01 +0100
Subject: [PATCH 1497/1843] fix: Fix `set_top_subtree_delimiter_span` using
 wrong index for close span

---
 src/tools/rust-analyzer/crates/tt/src/storage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rust-analyzer/crates/tt/src/storage.rs b/src/tools/rust-analyzer/crates/tt/src/storage.rs
index 4dd02d875a29..50a1106175ab 100644
--- a/src/tools/rust-analyzer/crates/tt/src/storage.rs
+++ b/src/tools/rust-analyzer/crates/tt/src/storage.rs
@@ -488,7 +488,7 @@ impl TopSubtree {
                 unreachable!()
             };
             *open_span = S::new(span.open.range, 0);
-            *close_span = S::new(span.close.range, 0);
+            *close_span = S::new(span.close.range, 1);
         }
         dispatch! {
             match &mut self.repr => tt => do_it(tt, span)

From 0cf7b94bf52f45996776d8b27708516928b12e0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Thu, 13 Nov 2025 15:44:20 +0100
Subject: [PATCH 1498/1843] fix Nix setup description

---
 src/doc/rustc-dev-guide/src/building/suggested.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/doc/rustc-dev-guide/src/building/suggested.md b/src/doc/rustc-dev-guide/src/building/suggested.md
index 3ed4f129d144..ac8f15db7751 100644
--- a/src/doc/rustc-dev-guide/src/building/suggested.md
+++ b/src/doc/rustc-dev-guide/src/building/suggested.md
@@ -426,7 +426,7 @@ You can then use that rust2 folder as a separate workspace for modifying and bui
 
 Several nix configurations are defined in `src/tools/nix-dev-shell`.
 
-If you're using direnv, you can create a symbol link to `src/tools/nix-dev-shell/envrc-flake` or `src/tools/nix-dev-shell/envrc-shell`
+If you're using direnv, you can create a symbolic link to `src/tools/nix-dev-shell/envrc-flake` or `src/tools/nix-dev-shell/envrc-shell`
 
 ```bash
 ln -s ./src/tools/nix-dev-shell/envrc-flake ./.envrc # Use flake
@@ -436,6 +436,12 @@ or
 ln -s ./src/tools/nix-dev-shell/envrc-shell ./.envrc # Use nix-shell
 ```
 
+If you're using the flake, make sure to also update it with the following command:
+
+```
+nix flake update --flake ./src/tools/nix-dev-shell
+```
+
 ### Note
 
 Note that when using nix on a not-NixOS distribution, it may be necessary to set

From 389294dcde4f50e9c910e5da6aca0582ced13796 Mon Sep 17 00:00:00 2001
From: Nora Breitmoser-Widdecke <23097564+puzzlewolf@users.noreply.github.com>
Date: Mon, 9 Feb 2026 13:35:09 +0100
Subject: [PATCH 1499/1843] Fix documentation for `indexing_slicing`

Replace typo'd name with link to the lint.
---
 clippy_lints/src/indexing_slicing.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clippy_lints/src/indexing_slicing.rs b/clippy_lints/src/indexing_slicing.rs
index a2fcdb4a54b4..e9ddd3ca8ede 100644
--- a/clippy_lints/src/indexing_slicing.rs
+++ b/clippy_lints/src/indexing_slicing.rs
@@ -44,7 +44,7 @@ declare_clippy_lint! {
     /// Checks for usage of indexing or slicing that may panic at runtime.
     ///
     /// This lint does not report on indexing or slicing operations
-    /// that always panic, clippy's `out_of_bound_indexing` already
+    /// that always panic, [out_of_bounds_indexing](#out_of_bounds_indexing) already
     /// handles those cases.
     ///
     /// ### Why restrict this?

From e04ae80c5ac7fab7d01aee9ad97ef109fe7f024d Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Mon, 9 Feb 2026 17:32:48 +1100
Subject: [PATCH 1500/1843] Rename `JobOwner` to `ActiveJobGuard`

This commit also adds and updates some relevant comments.
---
 compiler/rustc_middle/src/ty/context/tls.rs |  3 +--
 compiler/rustc_query_impl/src/execution.rs  | 25 +++++++++++++--------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs
index fa9995898ac2..a06f92792820 100644
--- a/compiler/rustc_middle/src/ty/context/tls.rs
+++ b/compiler/rustc_middle/src/ty/context/tls.rs
@@ -16,8 +16,7 @@ pub struct ImplicitCtxt<'a, 'tcx> {
     /// The current `TyCtxt`.
     pub tcx: TyCtxt<'tcx>,
 
-    /// The current query job, if any. This is updated by `JobOwner::start` in
-    /// `ty::query::plumbing` when executing a query.
+    /// The current query job, if any.
     pub query: Option,
 
     /// Used to prevent queries from calling too deeply.
diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
index ba442d8a0081..7bed6ee937db 100644
--- a/compiler/rustc_query_impl/src/execution.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -83,9 +83,12 @@ pub(crate) fn gather_active_jobs_inner<'tcx, K: Copy>(
     Some(())
 }
 
-/// A type representing the responsibility to execute the job in the `job` field.
-/// This will poison the relevant query if dropped.
-struct JobOwner<'tcx, K>
+/// Guard object representing the responsibility to execute a query job and
+/// mark it as completed.
+///
+/// This will poison the relevant query key if it is dropped without calling
+/// [`Self::complete`].
+struct ActiveJobGuard<'tcx, K>
 where
     K: Eq + Hash + Copy,
 {
@@ -137,12 +140,12 @@ fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
     }
 }
 
-impl<'tcx, K> JobOwner<'tcx, K>
+impl<'tcx, K> ActiveJobGuard<'tcx, K>
 where
     K: Eq + Hash + Copy,
 {
     /// Completes the query by updating the query cache with the `result`,
-    /// signals the waiter and forgets the JobOwner, so it won't poison the query
+    /// signals the waiter, and forgets the guard so it won't poison the query.
     fn complete(self, cache: &C, key_hash: u64, result: C::Value, dep_node_index: DepNodeIndex)
     where
         C: QueryCache,
@@ -174,7 +177,7 @@ where
     }
 }
 
-impl<'tcx, K> Drop for JobOwner<'tcx, K>
+impl<'tcx, K> Drop for ActiveJobGuard<'tcx, K>
 where
     K: Eq + Hash + Copy,
 {
@@ -342,11 +345,13 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
     id: QueryJobId,
     dep_node: Option,
 ) -> (C::Value, Option) {
-    // Use `JobOwner` so the query will be poisoned if executing it panics.
-    let job_owner = JobOwner { state, key };
+    // Set up a guard object that will automatically poison the query if a
+    // panic occurs while executing the query (or any intermediate plumbing).
+    let job_guard = ActiveJobGuard { state, key };
 
     debug_assert_eq!(qcx.tcx.dep_graph.is_fully_enabled(), INCR);
 
+    // Delegate to another function to actually execute the query job.
     let (result, dep_node_index) = if INCR {
         execute_job_incr(query, qcx, qcx.tcx.dep_graph.data().unwrap(), key, dep_node, id)
     } else {
@@ -388,7 +393,9 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
             }
         }
     }
-    job_owner.complete(cache, key_hash, result, dep_node_index);
+
+    // Tell the guard to perform completion bookkeeping, and also to not poison the query.
+    job_guard.complete(cache, key_hash, result, dep_node_index);
 
     (result, Some(dep_node_index))
 }

From c475bdaa53441f52977ce79ac2e16b807dcb006d Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Mon, 9 Feb 2026 17:36:29 +1100
Subject: [PATCH 1501/1843] Include `key_hash` in `ActiveJobGuard`

This value is a previously-computed hash of the key, so it makes sense to
bundle it with the key inside the guard, since the guard will need it on
completion anyway.
---
 compiler/rustc_query_impl/src/execution.rs | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
index 7bed6ee937db..88604c91d025 100644
--- a/compiler/rustc_query_impl/src/execution.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -94,6 +94,7 @@ where
 {
     state: &'tcx QueryState<'tcx, K>,
     key: K,
+    key_hash: u64,
 }
 
 #[cold]
@@ -146,14 +147,13 @@ where
 {
     /// Completes the query by updating the query cache with the `result`,
     /// signals the waiter, and forgets the guard so it won't poison the query.
-    fn complete(self, cache: &C, key_hash: u64, result: C::Value, dep_node_index: DepNodeIndex)
+    fn complete(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
     where
         C: QueryCache,
     {
-        let key = self.key;
-        let state = self.state;
-
-        // Forget ourself so our destructor won't poison the query
+        // Forget ourself so our destructor won't poison the query.
+        // (Extract fields by value first to make sure we don't leak anything.)
+        let Self { state, key, key_hash }: Self = self;
         mem::forget(self);
 
         // Mark as complete before we remove the job from the active state
@@ -185,11 +185,10 @@ where
     #[cold]
     fn drop(&mut self) {
         // Poison the query so jobs waiting on it panic.
-        let state = self.state;
+        let Self { state, key, key_hash } = *self;
         let job = {
-            let key_hash = sharded::make_hash(&self.key);
             let mut shard = state.active.lock_shard_by_hash(key_hash);
-            match shard.find_entry(key_hash, equivalent_key(&self.key)) {
+            match shard.find_entry(key_hash, equivalent_key(&key)) {
                 Err(_) => panic!(),
                 Ok(occupied) => {
                     let ((key, value), vacant) = occupied.remove();
@@ -347,7 +346,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
 ) -> (C::Value, Option) {
     // Set up a guard object that will automatically poison the query if a
     // panic occurs while executing the query (or any intermediate plumbing).
-    let job_guard = ActiveJobGuard { state, key };
+    let job_guard = ActiveJobGuard { state, key, key_hash };
 
     debug_assert_eq!(qcx.tcx.dep_graph.is_fully_enabled(), INCR);
 
@@ -395,7 +394,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
     }
 
     // Tell the guard to perform completion bookkeeping, and also to not poison the query.
-    job_guard.complete(cache, key_hash, result, dep_node_index);
+    job_guard.complete(cache, result, dep_node_index);
 
     (result, Some(dep_node_index))
 }

From 2b22150c1802bd5833492e263745a9037f1a190a Mon Sep 17 00:00:00 2001
From: Oscar Bray 
Date: Wed, 4 Feb 2026 18:37:10 +0000
Subject: [PATCH 1502/1843] Port rustc_no_implicit_bounds attribute to parser.

---
 .../rustc_attr_parsing/src/attributes/crate_level.rs   |  9 +++++++++
 compiler/rustc_attr_parsing/src/context.rs             |  1 +
 compiler/rustc_hir/src/attrs/data_structures.rs        |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 .../rustc_hir_analysis/src/hir_ty_lowering/bounds.rs   | 10 ++++++----
 compiler/rustc_passes/src/check_attr.rs                |  2 +-
 6 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/crate_level.rs b/compiler/rustc_attr_parsing/src/attributes/crate_level.rs
index 557dfe09853b..bdfe7bfb8f1f 100644
--- a/compiler/rustc_attr_parsing/src/attributes/crate_level.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/crate_level.rs
@@ -283,3 +283,12 @@ impl NoArgsAttributeParser for RustcPreserveUbChecksParser {
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Crate)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcPreserveUbChecks;
 }
+
+pub(crate) struct RustcNoImplicitBoundsParser;
+
+impl NoArgsAttributeParser for RustcNoImplicitBoundsParser {
+    const PATH: &[Symbol] = &[sym::rustc_no_implicit_bounds];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Crate)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNoImplicitBounds;
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index afbb70654f4d..573e14786abf 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -281,6 +281,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index ef790fe76b74..869e3f43ea71 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1177,6 +1177,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_no_implicit_autorefs]`
     RustcNoImplicitAutorefs,
 
+    /// Represents `#[rustc_no_implicit_bounds]`
+    RustcNoImplicitBounds,
+
     /// Represents `#[rustc_non_const_trait_method]`.
     RustcNonConstTraitMethod,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index cba7679d2df3..001448b3e72e 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -137,6 +137,7 @@ impl AttributeKind {
             RustcMustImplementOneOf { .. } => No,
             RustcNeverReturnsNullPointer => Yes,
             RustcNoImplicitAutorefs => Yes,
+            RustcNoImplicitBounds => No,
             RustcNonConstTraitMethod => No, // should be reported via other queries like `constness`
             RustcNounwind => No,
             RustcObjcClass { .. } => No,
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
index a1c3af5f999d..85e96e200c2e 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
@@ -4,15 +4,16 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_errors::codes::*;
 use rustc_errors::struct_span_code_err;
 use rustc_hir as hir;
-use rustc_hir::PolyTraitRef;
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId};
+use rustc_hir::{PolyTraitRef, find_attr};
 use rustc_middle::bug;
 use rustc_middle::ty::{
     self as ty, IsSuggestable, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
     TypeVisitor, Upcast,
 };
-use rustc_span::{ErrorGuaranteed, Ident, Span, kw, sym};
+use rustc_span::{ErrorGuaranteed, Ident, Span, kw};
 use rustc_trait_selection::traits;
 use smallvec::SmallVec;
 use tracing::{debug, instrument};
@@ -170,7 +171,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         let tcx = self.tcx();
 
         // Skip adding any default bounds if `#![rustc_no_implicit_bounds]`
-        if tcx.has_attr(CRATE_DEF_ID, sym::rustc_no_implicit_bounds) {
+        if find_attr!(tcx.get_all_attrs(CRATE_DEF_ID), AttributeKind::RustcNoImplicitBounds) {
             return;
         }
 
@@ -284,7 +285,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         context: ImpliedBoundsContext<'tcx>,
     ) -> bool {
         let collected = collect_bounds(hir_bounds, context, trait_def_id);
-        !self.tcx().has_attr(CRATE_DEF_ID, sym::rustc_no_implicit_bounds) && !collected.any()
+        !find_attr!(self.tcx().get_all_attrs(CRATE_DEF_ID), AttributeKind::RustcNoImplicitBounds)
+            && !collected.any()
     }
 
     fn reject_duplicate_relaxed_bounds(&self, relaxed_bounds: SmallVec<[&PolyTraitRef<'_>; 1]>) {
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index dd0de90d575b..2dcfbaaef579 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -330,6 +330,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcMir(_)
                     | AttributeKind::RustcNeverReturnsNullPointer
                     | AttributeKind::RustcNoImplicitAutorefs
+                    | AttributeKind::RustcNoImplicitBounds
                     | AttributeKind::RustcNonConstTraitMethod
                     | AttributeKind::RustcNounwind
                     | AttributeKind::RustcObjcClass { .. }
@@ -413,7 +414,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             // crate-level attrs, are checked below
                             | sym::feature
                             | sym::register_tool
-                            | sym::rustc_no_implicit_bounds
                             | sym::test_runner,
                             ..
                         ] => {}

From 43e5203a8cc1d10f12895475069c216db4d2dd95 Mon Sep 17 00:00:00 2001
From: David Wood 
Date: Fri, 6 Feb 2026 16:16:17 +0000
Subject: [PATCH 1503/1843] compiletest: `-Zunstable-options` for json targets

---
 src/tools/compiletest/src/common.rs  | 4 +++-
 src/tools/compiletest/src/runtest.rs | 5 +++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index c71a350ffd30..54c0f2ec7430 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -949,7 +949,9 @@ impl TargetCfgs {
         // actually be changed with `-C` flags.
         for config in query_rustc_output(
             config,
-            &["--print=cfg", "--target", &config.target],
+            // `-Zunstable-options` is necessary when compiletest is running with custom targets
+            // (such as synthetic targets used to bless mir-opt tests).
+            &["-Zunstable-options", "--print=cfg", "--target", &config.target],
             Default::default(),
         )
         .trim()
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 1804490a5f12..742790a50fce 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -1670,6 +1670,11 @@ impl<'test> TestCx<'test> {
                 if self.props.force_host { &*self.config.host } else { &*self.config.target };
 
             compiler.arg(&format!("--target={}", target));
+            if target.ends_with(".json") {
+                // `-Zunstable-options` is necessary when compiletest is running with custom targets
+                // (such as synthetic targets used to bless mir-opt tests).
+                compiler.arg("-Zunstable-options");
+            }
         }
         self.set_revision_flags(&mut compiler);
 

From 40a264cd454e791f478dae1488bb67ed7d7d058d Mon Sep 17 00:00:00 2001
From: Weixie Cui 
Date: Mon, 9 Feb 2026 22:16:36 +0800
Subject: [PATCH 1504/1843] fix: rhs_span to rhs_span_new

---
 compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index 4fb739403ced..853ecd3c91f7 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -1695,7 +1695,7 @@ fn suggest_ampmut<'tcx>(
                 && let Either::Left(rhs_stmt_new) = body.stmt_at(*assign)
                 && let StatementKind::Assign(box (_, rvalue_new)) = &rhs_stmt_new.kind
                 && let rhs_span_new = rhs_stmt_new.source_info.span
-                && let Ok(rhs_str_new) = tcx.sess.source_map().span_to_snippet(rhs_span)
+                && let Ok(rhs_str_new) = tcx.sess.source_map().span_to_snippet(rhs_span_new)
             {
                 (rvalue, rhs_span, rhs_str) = (rvalue_new, rhs_span_new, rhs_str_new);
             }

From e7855f5bfaf5d0d568f1fe4d9c07aec1144f405d Mon Sep 17 00:00:00 2001
From: Wilfred Hughes 
Date: Tue, 3 Feb 2026 15:59:46 +0000
Subject: [PATCH 1505/1843] internal: Run clippy as a separate CI step

Currently clippy is run in CI as part of the macOS build. This is a
little confusing, because clippy failures just show as
"Rust (macos-latest)" which make it look like a macOS build failure.

Instead, treat clippy as a separate build step, like miri and rustfmt.

This should also make CI a little faster, because it reduces macOS
runner usage (which tend to be slower than Linux on GitHub actions),
and it reduces the number of steps where we need to install clippy.
---
 .../rust-analyzer/.github/workflows/ci.yaml   | 30 +++++++++++++++----
 .../crates/hir-ty/src/mir/eval.rs             |  2 +-
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/tools/rust-analyzer/.github/workflows/ci.yaml b/src/tools/rust-analyzer/.github/workflows/ci.yaml
index 1a0deee564ae..c3a5017d72d7 100644
--- a/src/tools/rust-analyzer/.github/workflows/ci.yaml
+++ b/src/tools/rust-analyzer/.github/workflows/ci.yaml
@@ -96,7 +96,7 @@ jobs:
         run: |
           rustup update --no-self-update stable
           rustup default stable
-          rustup component add --toolchain stable rust-src clippy rustfmt
+          rustup component add --toolchain stable rust-src rustfmt
           # We also install a nightly rustfmt, because we use `--file-lines` in
           # a test.
           rustup toolchain install nightly --profile minimal --component rustfmt
@@ -128,10 +128,6 @@ jobs:
       - name: Run cargo-machete
         run: cargo machete
 
-      - name: Run Clippy
-        if: matrix.os == 'macos-latest'
-        run: cargo clippy --all-targets -- -D clippy::disallowed_macros -D clippy::dbg_macro -D clippy::todo -D clippy::print_stdout -D clippy::print_stderr
-
   analysis-stats:
     if: github.repository == 'rust-lang/rust-analyzer'
     runs-on: ubuntu-latest
@@ -178,6 +174,28 @@ jobs:
 
       - run: cargo fmt -- --check
 
+  clippy:
+    if: github.repository == 'rust-lang/rust-analyzer'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Note that clippy output is currently dependent on whether rust-src is installed,
+      # https://github.com/rust-lang/rust-clippy/issues/14625
+      - name: Install Rust toolchain
+        run: |
+          rustup update --no-self-update stable
+          rustup default stable
+          rustup component add --toolchain stable rust-src clippy
+
+      # https://github.com/actions-rust-lang/setup-rust-toolchain/blob/main/rust.json
+      - name: Install Rust Problem Matcher
+        run: echo "::add-matcher::.github/rust.json"
+
+      - run: cargo clippy --all-targets -- -D clippy::disallowed_macros -D clippy::dbg_macro -D clippy::todo -D clippy::print_stdout -D clippy::print_stderr
+
   miri:
     if: github.repository == 'rust-lang/rust-analyzer'
     runs-on: ubuntu-latest
@@ -309,7 +327,7 @@ jobs:
         run: typos
 
   conclusion:
-    needs: [rust, rust-cross, typescript, typo-check, proc-macro-srv, miri, rustfmt, analysis-stats]
+    needs: [rust, rust-cross, typescript, typo-check, proc-macro-srv, miri, rustfmt, clippy, analysis-stats]
     # We need to ensure this job does *not* get skipped if its dependencies fail,
     # because a skipped job is considered a success by GitHub. So we have to
     # overwrite `if:`. We use `!cancelled()` to ensure the job does still not get run
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/mir/eval.rs b/src/tools/rust-analyzer/crates/hir-ty/src/mir/eval.rs
index 5de08313f418..ec0723c3f8c3 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/mir/eval.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/mir/eval.rs
@@ -1625,7 +1625,7 @@ impl<'db> Evaluator<'db> {
                     };
                     match target_ty {
                         rustc_type_ir::FloatTy::F32 => Owned((value as f32).to_le_bytes().to_vec()),
-                        rustc_type_ir::FloatTy::F64 => Owned((value as f64).to_le_bytes().to_vec()),
+                        rustc_type_ir::FloatTy::F64 => Owned(value.to_le_bytes().to_vec()),
                         rustc_type_ir::FloatTy::F16 | rustc_type_ir::FloatTy::F128 => {
                             not_supported!("unstable floating point type f16 and f128");
                         }

From 00432c0b85ecd3a099fbe4b0f867f3929b099366 Mon Sep 17 00:00:00 2001
From: mikhailofff 
Date: Mon, 9 Feb 2026 18:54:17 +0400
Subject: [PATCH 1506/1843] fix dead links in type checking section of the doc

---
 book/src/development/type_checking.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/book/src/development/type_checking.md b/book/src/development/type_checking.md
index 578836ecc568..55adb0aceadd 100644
--- a/book/src/development/type_checking.md
+++ b/book/src/development/type_checking.md
@@ -146,7 +146,7 @@ in this chapter:
 
 - [Stages of compilation](https://rustc-dev-guide.rust-lang.org/compiler-src.html#the-main-stages-of-compilation)
 - [Diagnostic items](https://rustc-dev-guide.rust-lang.org/diagnostics/diagnostic-items.html)
-- [Type checking](https://rustc-dev-guide.rust-lang.org/type-checking.html)
+- [Type checking](https://rustc-dev-guide.rust-lang.org/hir-typeck/summary.html)
 - [Ty module](https://rustc-dev-guide.rust-lang.org/ty.html)
 
 [Adt]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Adt
@@ -163,5 +163,5 @@ in this chapter:
 [TyKind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html
 [TypeckResults]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypeckResults.html
 [middle_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html
-[hir_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.Ty.html
+[hir_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Ty.html
 [lower_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/fn.lower_ty.html

From 6494dabc33a27dafd61b98dca55915b5cb7f60de Mon Sep 17 00:00:00 2001
From: Shoyu Vanilla 
Date: Tue, 10 Feb 2026 00:57:41 +0900
Subject: [PATCH 1507/1843] fix: Sync `allow_normalize` to rustc

---
 .../hir-ty/src/next_solver/predicate.rs       |  7 +-
 .../crates/hir-ty/src/tests/regression.rs     | 70 +++++++++++++++++++
 .../crates/test-utils/src/minicore.rs         | 24 ++++++-
 3 files changed, 97 insertions(+), 4 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs
index 6f4fae707317..8658d03a9e3e 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/next_solver/predicate.rs
@@ -714,9 +714,9 @@ impl<'db> rustc_type_ir::inherent::Predicate> for Predicate<'db>
     fn allow_normalization(self) -> bool {
         // TODO: this should probably live in rustc_type_ir
         match self.inner().as_ref().skip_binder() {
-            PredicateKind::Clause(ClauseKind::WellFormed(_))
-            | PredicateKind::AliasRelate(..)
-            | PredicateKind::NormalizesTo(..) => false,
+            PredicateKind::Clause(ClauseKind::WellFormed(_)) | PredicateKind::AliasRelate(..) => {
+                false
+            }
             PredicateKind::Clause(ClauseKind::Trait(_))
             | PredicateKind::Clause(ClauseKind::RegionOutlives(_))
             | PredicateKind::Clause(ClauseKind::TypeOutlives(_))
@@ -729,6 +729,7 @@ impl<'db> rustc_type_ir::inherent::Predicate> for Predicate<'db>
             | PredicateKind::Coerce(_)
             | PredicateKind::Clause(ClauseKind::ConstEvaluatable(_))
             | PredicateKind::ConstEquate(_, _)
+            | PredicateKind::NormalizesTo(..)
             | PredicateKind::Ambiguous => true,
         }
     }
diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
index 3b5b4e4fa540..fba582f880fd 100644
--- a/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
+++ b/src/tools/rust-analyzer/crates/hir-ty/src/tests/regression.rs
@@ -2688,3 +2688,73 @@ pub trait FilterT = Self> {
     "#,
     );
 }
+
+#[test]
+fn regression_21605() {
+    check_infer(
+        r#"
+//- minicore: fn, coerce_unsized, dispatch_from_dyn, iterator, iterators
+pub struct Filter<'a, 'b, T>
+where
+    T: 'b,
+    'a: 'b,
+{
+    filter_fn: dyn Fn(&'a T) -> bool,
+    t: Option,
+    b: &'b (),
+}
+
+impl<'a, 'b, T> Filter<'a, 'b, T>
+where
+    T: 'b,
+    'a: 'b,
+{
+    pub fn new(filter_fn: dyn Fn(&T) -> bool) -> Self {
+        Self {
+            filter_fn: filter_fn,
+            t: None,
+            b: &(),
+        }
+    }
+}
+
+pub trait FilterExt {
+    type Output;
+    fn filter(&self, filter: &Filter) -> Self::Output;
+}
+
+impl FilterExt for [T; N]
+where
+    T: IntoIterator,
+{
+    type Output = T;
+    fn filter(&self, filter: &Filter) -> Self::Output {
+        let _ = self.into_iter().filter(filter.filter_fn);
+        loop {}
+    }
+}
+"#,
+        expect![[r#"
+            214..223 'filter_fn': dyn Fn(&'? T) -> bool + 'static
+            253..360 '{     ...     }': Filter<'a, 'b, T>
+            263..354 'Self {...     }': Filter<'a, 'b, T>
+            293..302 'filter_fn': dyn Fn(&'? T) -> bool + 'static
+            319..323 'None': Option
+            340..343 '&()': &'? ()
+            341..343 '()': ()
+            421..425 'self': &'? Self
+            427..433 'filter': &'? Filter<'?, '?, T>
+            580..584 'self': &'? [T; N]
+            586..592 'filter': &'? Filter<'?, '?, T>
+            622..704 '{     ...     }': T
+            636..637 '_': Filter, dyn Fn(&'? T) -> bool + '?>
+            640..644 'self': &'? [T; N]
+            640..656 'self.i...iter()': Iter<'?, T>
+            640..681 'self.i...er_fn)': Filter, dyn Fn(&'? T) -> bool + '?>
+            664..670 'filter': &'? Filter<'?, '?, T>
+            664..680 'filter...ter_fn': dyn Fn(&'? T) -> bool + 'static
+            691..698 'loop {}': !
+            696..698 '{}': ()
+        "#]],
+    );
+}
diff --git a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
index 7d95043867ee..c34475bbdf01 100644
--- a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
+++ b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
@@ -1689,6 +1689,21 @@ pub mod iter {
             }
         }
 
+        pub struct Filter {
+            iter: I,
+            predicate: P,
+        }
+        impl Iterator for Filter
+        where
+            P: FnMut(&I::Item) -> bool,
+        {
+            type Item = I::Item;
+
+            fn next(&mut self) -> Option {
+                loop {}
+            }
+        }
+
         pub struct FilterMap {
             iter: I,
             f: F,
@@ -1705,7 +1720,7 @@ pub mod iter {
             }
         }
     }
-    pub use self::adapters::{FilterMap, Take};
+    pub use self::adapters::{Filter, FilterMap, Take};
 
     mod sources {
         mod repeat {
@@ -1756,6 +1771,13 @@ pub mod iter {
                 {
                     loop {}
                 }
+                fn filter
(self, predicate: P) -> crate::iter::Filter
+                where
+                    Self: Sized,
+                    P: FnMut(&Self::Item) -> bool,
+                {
+                    loop {}
+                }
                 fn filter_map(self, _f: F) -> crate::iter::FilterMap
                 where
                     Self: Sized,

From 73a991fb9d7f68f497f047d4e948bde463f2f6eb Mon Sep 17 00:00:00 2001
From: Keith-Cancel 
Date: Thu, 5 Feb 2026 22:01:57 -0800
Subject: [PATCH 1508/1843] Allow provisional mgca syntax of type const 
 =  to be reconized.

Revert, but without type const.

Update symbol for feature err, then update suggestion output, and lastly update tests that change because of those.

Update these new tests with the correct syntax, and few existing tests with the new outputs the merge with main added.

Fix for tidyfmt and some errors when manually resolving a merge conflicts.

Update these tests to use update error messages and type const syntax.

Update comments and error message to use new syntax instead of old type_const attribute.

Remove the type_const attribute

update some more tests to use the new syntax.

Update these test cases.

update feature gate test

Change gate logic for `mgca_type_const_syntax` to work also if `min_generic_const_args` is enabled.

Create a new feature gate that checks for the feature before expansion.

Make rustfmt handle the `type const` syntax correctly.

Add a convience method to check if a RhsKind is type const.

Rename `Const` discriminant to `Body` for `ConstItemRhsKind`

Give the `TraitItemKind` flag an enum instead of a simple bool to better describe what the flag is for.

Update formatting for these match statements.

Update clippy test to use type const syntax.

Update test to use type const syntax.

update rustfmt to match ast items.

Update clippy to match ast and hir items.

Few more test cases that used old attribute, instead of 'type const'

Update to match the output from the feature gate checks.

tidyfmt adjustments.

Update the is_type_const, so I can constrain record!(..) in encoder.rs

Update conditional compilation test.

Move the feature gate to after expansion to allow for cfg(...) to work.

Update some more tests to use the new syntax.

Update type const tests in associated-const-bindings to use new syntax.

Don't check based off the attribute, but the item here.

Update some tests outside of the const_generics folder that were using #[type_const]

update the tests in associated consts that use #[type_const] to use type const

Update these mgca tests with the type const syntax.

Add a flag to TraitItemKind for detecting type const for now. Maybe later change ItemConstRhs to have optional consts but that touches a lot more lines of code.

Don't need into for these now that it's a query.

Add is_type_const query to handle foreign def ids.

update this test to use type const syntax.

Fix logic here, we only want to lower if there is expression in this case.

Update built-in macros to use ConstItemRhsKind

Update more instance of the old ConstItemRhs.

Rename ConstItemKind to ConstItemRhsKind, I noticed there is a typed called ConstantItemKind, so add the Rhs to the name to avoid confusion.

Update lower to use ConstItemKind

Add an other helper method to check if the rhs kinda has an expr.

Update item parse to use ConstItemKind enum.

Felt the field name could a be little clear when editing a few other things.

Change the ConstItem struct see know if we have a type const or regular const.

Make sure this syntax is properly feature gated.
---
 compiler/rustc_ast/src/ast.rs                 | 37 ++++++---
 compiler/rustc_ast/src/visit.rs               |  2 +-
 compiler/rustc_ast_lowering/src/item.rs       | 35 +++++---
 compiler/rustc_ast_lowering/src/lib.rs        | 19 ++---
 .../rustc_ast_passes/src/ast_validation.rs    | 16 ++--
 compiler/rustc_ast_passes/src/feature_gate.rs | 43 ++++++++++
 .../rustc_ast_pretty/src/pprust/state/item.rs |  8 +-
 .../src/attributes/traits.rs                  |  9 ---
 compiler/rustc_attr_parsing/src/context.rs    |  1 -
 .../src/alloc_error_handler.rs                |  2 +-
 compiler/rustc_builtin_macros/src/eii.rs      |  2 +-
 .../src/global_allocator.rs                   |  2 +-
 .../src/proc_macro_harness.rs                 |  2 +-
 compiler/rustc_builtin_macros/src/test.rs     |  4 +-
 .../src/check_consts/qualifs.rs               |  2 +-
 .../src/const_eval/eval_queries.rs            |  2 +-
 compiler/rustc_expand/src/build.rs            |  4 +-
 compiler/rustc_feature/src/builtin_attrs.rs   |  7 --
 compiler/rustc_feature/src/unstable.rs        |  2 +
 .../rustc_hir/src/attrs/data_structures.rs    |  3 -
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 -
 compiler/rustc_hir/src/hir.rs                 | 31 +++++--
 compiler/rustc_hir/src/intravisit.rs          |  2 +-
 .../rustc_hir_analysis/src/check/check.rs     |  2 +-
 .../src/check/compare_impl_item.rs            |  4 +-
 .../rustc_hir_analysis/src/check/wfcheck.rs   |  2 +-
 compiler/rustc_hir_analysis/src/collect.rs    | 25 +++++-
 .../src/collect/resolve_bound_vars.rs         |  2 +-
 .../rustc_hir_analysis/src/collect/type_of.rs |  8 +-
 .../src/hir_ty_lowering/bounds.rs             |  6 +-
 .../src/hir_ty_lowering/mod.rs                | 13 +--
 .../rustc_hir_analysis/src/hir_wf_check.rs    |  2 +-
 compiler/rustc_hir_analysis/src/lib.rs        |  2 +-
 compiler/rustc_hir_pretty/src/lib.rs          |  2 +-
 compiler/rustc_lint/src/unused.rs             |  4 +-
 .../src/rmeta/decoder/cstore_impl.rs          |  1 +
 compiler/rustc_metadata/src/rmeta/encoder.rs  |  3 +
 compiler/rustc_metadata/src/rmeta/mod.rs      |  1 +
 compiler/rustc_middle/src/queries.rs          | 10 ++-
 compiler/rustc_middle/src/traits/mod.rs       |  4 +-
 compiler/rustc_middle/src/ty/context.rs       | 22 +++--
 compiler/rustc_mir_build/src/builder/mod.rs   |  2 +-
 compiler/rustc_parse/src/parser/item.rs       | 80 ++++++++++++-------
 compiler/rustc_passes/src/check_attr.rs       |  1 -
 compiler/rustc_passes/src/reachable.rs        |  8 +-
 compiler/rustc_resolve/src/late.rs            | 63 +++++++--------
 compiler/rustc_span/src/symbol.rs             |  2 +-
 .../src/traits/normalize.rs                   |  2 +-
 src/librustdoc/clean/mod.rs                   |  4 +-
 .../clippy/clippy_lints/src/non_copy_const.rs |  4 +-
 .../clippy/clippy_lints/src/types/mod.rs      |  2 +-
 .../clippy/clippy_utils/src/ast_utils/mod.rs  | 28 ++++---
 ...duplication_in_bounds_assoc_const_eq.fixed |  4 +-
 ...it_duplication_in_bounds_assoc_const_eq.rs |  4 +-
 src/tools/rustfmt/src/items.rs                | 56 ++++++++-----
 tests/crashes/149809.rs                       |  3 +-
 tests/debuginfo/associated-const-bindings.rs  |  6 +-
 .../ice-associated-const-equality-105952.rs   |  3 +-
 .../auxiliary/assoc-const-equality.rs         |  3 +-
 .../associated-constant-not-allowed-102467.rs |  3 +-
 tests/ui/associated-consts/issue-110933.rs    |  3 +-
 .../type-const-in-array-len-wrong-type.rs     |  3 +-
 .../type-const-in-array-len-wrong-type.stderr |  2 +-
 .../type-const-in-array-len.rs                |  9 +--
 .../duplicate-bound-err.rs                    | 12 +--
 .../duplicate-bound-err.stderr                | 42 +++++-----
 .../associated-type-bounds/duplicate-bound.rs |  6 +-
 .../type-const-inherent-impl-normalize.rs     |  6 +-
 .../type-const-inherent-impl-normalize.stderr | 18 ++++-
 tests/ui/attributes/malformed-attrs.rs        |  2 -
 tests/ui/attributes/malformed-attrs.stderr    | 61 ++++++--------
 .../associated-const-bindings/ambiguity.rs    |  9 +--
 .../ambiguity.stderr                          | 16 ++--
 .../associated-const-bindings/assoc-const.rs  |  6 +-
 .../bound-var-in-ty-not-wf.rs                 |  3 +-
 .../bound-var-in-ty-not-wf.stderr             |  4 +-
 .../bound-var-in-ty.rs                        |  3 +-
 .../coexisting-with-type-binding.rs           |  6 +-
 .../associated-const-bindings/coherence.rs    |  6 +-
 .../coherence.stderr                          |  2 +-
 .../const-projection-err.rs                   |  3 +-
 .../const-projection-err.stderr               |  4 +-
 .../const_evaluatable_unchecked.rs            |  3 +-
 ...dyn-compat-assoc-const-ty-mentions-self.rs |  6 +-
 ...compat-assoc-const-ty-mentions-self.stderr | 12 +--
 .../dyn-compat-basic.rs                       | 15 ++--
 .../dyn-compat-const-mismatch.rs              |  6 +-
 .../dyn-compat-const-mismatch.stderr          |  2 +-
 ...ompat-const-param-default-mentions-self.rs |  6 +-
 ...t-const-param-default-mentions-self.stderr |  2 +-
 ...ection-behind-trait-alias-mentions-self.rs |  3 +-
 ...on-behind-trait-alias-mentions-self.stderr |  2 +-
 ...rojection-from-supertrait-mentions-self.rs |  6 +-
 ...ction-from-supertrait-mentions-self.stderr |  6 +-
 .../dyn-compat-non-type-assoc-const.rs        |  4 +-
 .../dyn-compat-non-type-assoc-const.stderr    |  8 +-
 ...und-on-assoc-const-allowed-and-enforced.rs |  6 +-
 ...on-assoc-const-allowed-and-enforced.stderr | 10 +--
 ...elf-const-projections-in-assoc-const-ty.rs |  6 +-
 ...const-projections-in-assoc-const-ty.stderr |  8 +-
 ...ompat-self-const-projections-in-methods.rs |  9 +--
 ...-const-projections-in-supertrait-bounds.rs |  3 +-
 ...st-projections-in-supertrait-bounds.stderr |  2 +-
 .../dyn-compat-symbol-mangling.rs             |  3 +-
 .../dyn-compat-symbol-mangling.v0.stderr      |  6 +-
 .../dyn-compat-unspecified-assoc-consts.rs    |  3 +-
 ...dyn-compat-unspecified-assoc-consts.stderr | 18 ++---
 .../equality-unused-issue-126729.rs           | 24 ++----
 .../equality_bound_with_infer.rs              |  6 +-
 .../esc-bound-var-in-ty.rs                    |  3 +-
 .../esc-bound-var-in-ty.stderr                |  2 +-
 .../issue-102335-const.rs                     |  3 +-
 .../mismatched-types-with-generic-in-ace.rs   |  6 +-
 ...ismatched-types-with-generic-in-ace.stderr | 16 ++--
 .../normalization-via-param-env.rs            |  3 +-
 .../associated-const-bindings/param-in-ty.rs  |  6 +-
 .../param-in-ty.stderr                        | 22 ++---
 .../projection-unspecified-but-bounded.rs     |  3 +-
 .../projection-unspecified-but-bounded.stderr |  4 +-
 .../associated-const-bindings/supertraits.rs  |  3 +-
 .../using-fnptr-as-type_const.rs              |  3 +-
 .../using-fnptr-as-type_const.stderr          |  6 +-
 .../auxiliary/non_local_type_const.rs         |  3 +-
 .../mgca/adt_expr_arg_simple.rs               |  4 +-
 .../mgca/adt_expr_arg_tuple_expr_fail.rs      |  4 +-
 .../mgca/adt_expr_infers_from_value.rs        |  3 +-
 .../mgca/array-expr-with-assoc-const.rs       |  3 +-
 .../mgca/assoc-const-projection-in-bound.rs   |  6 +-
 .../mgca/assoc-const-without-type_const.rs    |  4 +-
 .../assoc-const-without-type_const.stderr     | 22 ++---
 tests/ui/const-generics/mgca/assoc-const.rs   |  3 +-
 .../mgca/bad-type_const-syntax.rs             | 14 ++--
 .../mgca/bad-type_const-syntax.stderr         | 40 ++++++----
 .../concrete-expr-with-generics-in-env.rs     | 18 ++---
 ...const-arg-coherence-conflicting-methods.rs |  4 +-
 .../mgca/cyclic-type-const-151251.rs          |  3 +-
 .../mgca/cyclic-type-const-151251.stderr      |  6 +-
 .../mgca/explicit_anon_consts.rs              | 24 +++---
 .../mgca/explicit_anon_consts.stderr          | 12 +--
 .../explicit_anon_consts_literals_hack.rs     |  3 +-
 .../mgca/multi_braced_direct_const_args.rs    |  3 +-
 .../non-local-const-without-type_const.rs     |  2 +-
 .../non-local-const-without-type_const.stderr |  4 +-
 .../printing_valtrees_supports_non_values.rs  |  4 +-
 .../const-generics/mgca/projection-error.rs   |  4 +-
 .../mgca/projection-error.stderr              |  4 +-
 .../mgca/tuple_ctor_arg_simple.rs             |  3 +-
 .../mgca/tuple_ctor_erroneous.rs              |  4 +-
 .../mgca/tuple_expr_arg_complex.rs            |  4 +-
 .../mgca/tuple_expr_arg_simple.rs             |  3 +-
 .../type-const-assoc-const-without-body.rs    |  6 +-
 ...type-const-assoc-const-without-body.stderr | 10 +--
 ...const-inherent-assoc-const-without-body.rs |  3 +-
 ...t-inherent-assoc-const-without-body.stderr | 10 +--
 .../mgca/type-const-used-in-trait.rs          |  3 +-
 .../mgca/type_const-array-return.rs           |  6 +-
 ...pe_const-generic-param-in-type.gate.stderr | 36 ++++-----
 ..._const-generic-param-in-type.nogate.stderr | 54 ++++++-------
 .../mgca/type_const-generic-param-in-type.rs  | 27 +++----
 .../mgca/type_const-incemental-compile.rs     |  3 +-
 .../type_const-inherent-const-omitted-type.rs |  3 +-
 ...e_const-inherent-const-omitted-type.stderr | 10 +--
 .../mgca/type_const-mismatched-types.rs       | 12 +--
 .../mgca/type_const-mismatched-types.stderr   | 22 ++---
 .../mgca/type_const-not-constparamty.rs       |  9 +--
 .../mgca/type_const-not-constparamty.stderr   | 18 ++---
 .../mgca/type_const-on-generic-expr.rs        |  8 +-
 .../mgca/type_const-on-generic-expr.stderr    | 12 +--
 .../mgca/type_const-on-generic_expr-2.rs      | 18 ++---
 .../mgca/type_const-on-generic_expr-2.stderr  | 18 ++---
 .../type_const-only-in-impl-omitted-type.rs   |  5 +-
 ...ype_const-only-in-impl-omitted-type.stderr | 12 +--
 .../mgca/type_const-only-in-impl.rs           |  5 +-
 .../mgca/type_const-only-in-impl.stderr       |  6 +-
 .../mgca/type_const-only-in-trait.rs          |  5 +-
 .../mgca/type_const-only-in-trait.stderr      | 12 ++-
 .../ui/const-generics/mgca/type_const-pub.rs  |  3 +-
 .../mgca/type_const-recursive.rs              |  4 +-
 .../mgca/type_const-recursive.stderr          |  4 +-
 .../ui/const-generics/mgca/type_const-use.rs  |  3 +-
 .../unbraced_const_block_const_arg_gated.rs   |  7 +-
 ...nbraced_const_block_const_arg_gated.stderr | 20 +++--
 .../mgca/unmarked-free-const.rs               |  2 +-
 .../mgca/unmarked-free-const.stderr           | 11 +--
 .../mgca/wrong_type_const_arr_diag_trait.rs   |  4 +-
 tests/ui/const-generics/ogca/basic-fail.rs    | 12 +--
 .../ui/const-generics/ogca/basic-fail.stderr  |  2 +-
 tests/ui/const-generics/ogca/basic.rs         | 12 +--
 .../ogca/coherence-ambiguous.rs               |  6 +-
 .../const-generics/ogca/rhs-but-not-root.rs   |  6 +-
 .../ogca/rhs-but-not-root.stderr              |  6 +-
 .../type-relative-path-144547.rs              | 14 ++--
 .../feature-gate-mgca-type-const-syntax.rs    | 17 ++++
 ...feature-gate-mgca-type-const-syntax.stderr | 63 +++++++++++++++
 .../feature-gate-min-generic-const-args.rs    |  6 +-
 ...feature-gate-min-generic-const-args.stderr | 18 ++++-
 .../feature-gate-opaque-generic-const-args.rs |  3 +-
 ...ture-gate-opaque-generic-const-args.stderr |  6 +-
 .../assoc-const-bindings.rs                   | 12 +--
 .../assoc-const-no-infer-ice-115806.rs        |  3 +-
 .../assoc-const-no-infer-ice-115806.stderr    |  2 +-
 .../assoc-const-projection-issue-151878.rs    |  3 +-
 .../overlap-due-to-unsatisfied-const-bound.rs |  7 +-
 ...rlap-due-to-unsatisfied-const-bound.stderr |  2 +-
 204 files changed, 1014 insertions(+), 906 deletions(-)
 create mode 100644 tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.stderr

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index fea87c2360f1..fa323b7cf581 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -3869,27 +3869,44 @@ pub struct ConstItem {
     pub ident: Ident,
     pub generics: Generics,
     pub ty: Box,
-    pub rhs: Option,
+    pub rhs_kind: ConstItemRhsKind,
     pub define_opaque: Option>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, Walkable)]
-pub enum ConstItemRhs {
-    TypeConst(AnonConst),
-    Body(Box),
+pub enum ConstItemRhsKind {
+    Body { rhs: Option> },
+    TypeConst { rhs: Option },
 }
 
-impl ConstItemRhs {
-    pub fn span(&self) -> Span {
-        self.expr().span
+impl ConstItemRhsKind {
+    pub fn new_body(rhs: Box) -> Self {
+        Self::Body { rhs: Some(rhs) }
     }
 
-    pub fn expr(&self) -> &Expr {
+    pub fn span(&self) -> Option {
+        Some(self.expr()?.span)
+    }
+
+    pub fn expr(&self) -> Option<&Expr> {
         match self {
-            ConstItemRhs::TypeConst(anon_const) => &anon_const.value,
-            ConstItemRhs::Body(expr) => expr,
+            Self::Body { rhs: Some(body) } => Some(&body),
+            Self::TypeConst { rhs: Some(anon) } => Some(&anon.value),
+            _ => None,
         }
     }
+
+    pub fn has_expr(&self) -> bool {
+        match self {
+            Self::Body { rhs: Some(_) } => true,
+            Self::TypeConst { rhs: Some(_) } => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_type_const(&self) -> bool {
+        matches!(self, &Self::TypeConst { .. })
+    }
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, Walkable)]
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index 51614460d3c4..8556e8288670 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -427,7 +427,7 @@ macro_rules! common_visitor_and_walkers {
             Const,
             ConstBlockItem,
             ConstItem,
-            ConstItemRhs,
+            ConstItemRhsKind,
             Defaultness,
             Delegation,
             DelegationMac,
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 12eaa99143e9..9922ed8a5c58 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -288,7 +288,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ident,
                 generics,
                 ty,
-                rhs,
+                rhs_kind,
                 define_opaque,
             }) => {
                 let ident = self.lower_ident(*ident);
@@ -301,7 +301,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             ty,
                             ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy),
                         );
-                        let rhs = this.lower_const_item_rhs(attrs, rhs.as_ref(), span);
+                        let rhs = this.lower_const_item_rhs(rhs_kind, span);
                         (ty, rhs)
                     },
                 );
@@ -941,7 +941,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         let (ident, generics, kind, has_default) = match &i.kind {
             AssocItemKind::Const(box ConstItem {
-                ident, generics, ty, rhs, define_opaque, ..
+                ident,
+                generics,
+                ty,
+                rhs_kind,
+                define_opaque,
+                ..
             }) => {
                 let (generics, kind) = self.lower_generics(
                     generics,
@@ -952,15 +957,18 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             ty,
                             ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy),
                         );
-                        let rhs = rhs
-                            .as_ref()
-                            .map(|rhs| this.lower_const_item_rhs(attrs, Some(rhs), i.span));
-                        hir::TraitItemKind::Const(ty, rhs)
+                        // Trait associated consts don't need an expression/body.
+                        let rhs = if rhs_kind.has_expr() {
+                            Some(this.lower_const_item_rhs(rhs_kind, i.span))
+                        } else {
+                            None
+                        };
+                        hir::TraitItemKind::Const(ty, rhs, rhs_kind.is_type_const().into())
                     },
                 );
 
                 if define_opaque.is_some() {
-                    if rhs.is_some() {
+                    if rhs_kind.has_expr() {
                         self.lower_define_opaque(hir_id, &define_opaque);
                     } else {
                         self.dcx().span_err(
@@ -970,7 +978,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     }
                 }
 
-                (*ident, generics, kind, rhs.is_some())
+                (*ident, generics, kind, rhs_kind.has_expr())
             }
             AssocItemKind::Fn(box Fn {
                 sig, ident, generics, body: None, define_opaque, ..
@@ -1154,7 +1162,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         let (ident, (generics, kind)) = match &i.kind {
             AssocItemKind::Const(box ConstItem {
-                ident, generics, ty, rhs, define_opaque, ..
+                ident,
+                generics,
+                ty,
+                rhs_kind,
+                define_opaque,
+                ..
             }) => (
                 *ident,
                 self.lower_generics(
@@ -1167,7 +1180,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy),
                         );
                         this.lower_define_opaque(hir_id, &define_opaque);
-                        let rhs = this.lower_const_item_rhs(attrs, rhs.as_ref(), i.span);
+                        let rhs = this.lower_const_item_rhs(rhs_kind, i.span);
                         hir::ImplItemKind::Const(ty, rhs)
                     },
                 ),
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 6dded9936fba..ff5edfc79943 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -2374,15 +2374,20 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
     fn lower_const_item_rhs(
         &mut self,
-        attrs: &[hir::Attribute],
-        rhs: Option<&ConstItemRhs>,
+        rhs_kind: &ConstItemRhsKind,
         span: Span,
     ) -> hir::ConstItemRhs<'hir> {
-        match rhs {
-            Some(ConstItemRhs::TypeConst(anon)) => {
+        match rhs_kind {
+            ConstItemRhsKind::Body { rhs: Some(body) } => {
+                hir::ConstItemRhs::Body(self.lower_const_body(span, Some(body)))
+            }
+            ConstItemRhsKind::Body { rhs: None } => {
+                hir::ConstItemRhs::Body(self.lower_const_body(span, None))
+            }
+            ConstItemRhsKind::TypeConst { rhs: Some(anon) } => {
                 hir::ConstItemRhs::TypeConst(self.lower_anon_const_to_const_arg_and_alloc(anon))
             }
-            None if find_attr!(attrs, AttributeKind::TypeConst(_)) => {
+            ConstItemRhsKind::TypeConst { rhs: None } => {
                 let const_arg = ConstArg {
                     hir_id: self.next_id(),
                     kind: hir::ConstArgKind::Error(
@@ -2392,10 +2397,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 };
                 hir::ConstItemRhs::TypeConst(self.arena.alloc(const_arg))
             }
-            Some(ConstItemRhs::Body(body)) => {
-                hir::ConstItemRhs::Body(self.lower_const_body(span, Some(body)))
-            }
-            None => hir::ConstItemRhs::Body(self.lower_const_body(span, None)),
         }
     }
 
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 2457e0a777e4..b9fb20b68971 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -1361,9 +1361,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     }
                 });
             }
-            ItemKind::Const(box ConstItem { defaultness, ident, rhs, .. }) => {
+            ItemKind::Const(box ConstItem { defaultness, ident, rhs_kind, .. }) => {
                 self.check_defaultness(item.span, *defaultness);
-                if rhs.is_none() {
+                if !rhs_kind.has_expr() {
                     self.dcx().emit_err(errors::ConstWithoutBody {
                         span: item.span,
                         replace_span: self.ending_semi_or_hi(item.span),
@@ -1715,11 +1715,13 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
         if let AssocCtxt::Impl { .. } = ctxt {
             match &item.kind {
-                AssocItemKind::Const(box ConstItem { rhs: None, .. }) => {
-                    self.dcx().emit_err(errors::AssocConstWithoutBody {
-                        span: item.span,
-                        replace_span: self.ending_semi_or_hi(item.span),
-                    });
+                AssocItemKind::Const(box ConstItem { rhs_kind, .. }) => {
+                    if !rhs_kind.has_expr() {
+                        self.dcx().emit_err(errors::AssocConstWithoutBody {
+                            span: item.span,
+                            replace_span: self.ending_semi_or_hi(item.span),
+                        });
+                    }
                 }
                 AssocItemKind::Fn(box Fn { body, .. }) => {
                     if body.is_none() && !self.is_sdylib_interface {
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index ce9001d65931..eac7f03d8450 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -249,6 +249,14 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ast::ItemKind::TyAlias(box ast::TyAlias { ty: Some(ty), .. }) => {
                 self.check_impl_trait(ty, false)
             }
+            ast::ItemKind::Const(box ast::ConstItem {
+                rhs_kind: ast::ConstItemRhsKind::TypeConst { .. },
+                ..
+            }) => {
+                // Make sure this is only allowed if the feature gate is enabled.
+                // #![feature(min_generic_const_args)]
+                gate!(&self, min_generic_const_args, i.span, "top-level `type const` are unstable");
+            }
 
             _ => {}
         }
@@ -422,6 +430,20 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 }
                 false
             }
+            ast::AssocItemKind::Const(box ast::ConstItem {
+                rhs_kind: ast::ConstItemRhsKind::TypeConst { .. },
+                ..
+            }) => {
+                // Make sure this is only allowed if the feature gate is enabled.
+                // #![feature(min_generic_const_args)]
+                gate!(
+                    &self,
+                    min_generic_const_args,
+                    i.span,
+                    "associated `type const` are unstable"
+                );
+                false
+            }
             _ => false,
         };
         if let ast::Defaultness::Default(_) = i.kind.defaultness() {
@@ -528,6 +550,27 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
             }
         }
     }
+    // `mgca_type_const_syntax` is part of `min_generic_const_args` so either
+    // or both are enabled we don't need to emit a feature error.
+    if let Some(spans) = spans.get(&sym::mgca_type_const_syntax) {
+        for span in spans {
+            if visitor.features.min_generic_const_args()
+                || visitor.features.mgca_type_const_syntax()
+                || span.allows_unstable(sym::min_generic_const_args)
+                || span.allows_unstable(sym::mgca_type_const_syntax)
+            {
+                continue;
+            }
+            feature_err(
+                &visitor.sess,
+                sym::min_generic_const_args,
+                *span,
+                "`type const` syntax is experimental",
+            )
+            .emit();
+        }
+    }
+
     gate_all!(global_registration, "global registration is experimental");
     gate_all!(return_type_notation, "return type notation is experimental");
     gate_all!(pin_ergonomics, "pinned reference syntax is experimental");
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
index 68054b06e39f..c7f110a2e003 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
@@ -221,7 +221,7 @@ impl<'a> State<'a> {
                 ident,
                 generics,
                 ty,
-                rhs,
+                rhs_kind,
                 define_opaque,
             }) => {
                 self.print_item_const(
@@ -229,7 +229,7 @@ impl<'a> State<'a> {
                     None,
                     generics,
                     ty,
-                    rhs.as_ref().map(|ct| ct.expr()),
+                    rhs_kind.expr(),
                     &item.vis,
                     ast::Safety::Default,
                     *defaultness,
@@ -573,7 +573,7 @@ impl<'a> State<'a> {
                 ident,
                 generics,
                 ty,
-                rhs,
+                rhs_kind,
                 define_opaque,
             }) => {
                 self.print_item_const(
@@ -581,7 +581,7 @@ impl<'a> State<'a> {
                     None,
                     generics,
                     ty,
-                    rhs.as_ref().map(|ct| ct.expr()),
+                    rhs_kind.expr(),
                     vis,
                     ast::Safety::Default,
                     *defaultness,
diff --git a/compiler/rustc_attr_parsing/src/attributes/traits.rs b/compiler/rustc_attr_parsing/src/attributes/traits.rs
index bfea02e789a0..ceaa43948d67 100644
--- a/compiler/rustc_attr_parsing/src/attributes/traits.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/traits.rs
@@ -66,15 +66,6 @@ impl NoArgsAttributeParser for ParenSugarParser {
     const CREATE: fn(Span) -> AttributeKind = AttributeKind::RustcParenSugar;
 }
 
-pub(crate) struct TypeConstParser;
-impl NoArgsAttributeParser for TypeConstParser {
-    const PATH: &[Symbol] = &[sym::type_const];
-    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
-    const ALLOWED_TARGETS: AllowedTargets =
-        AllowedTargets::AllowList(&[Allow(Target::Const), Allow(Target::AssocConst)]);
-    const CREATE: fn(Span) -> AttributeKind = AttributeKind::TypeConst;
-}
-
 // Markers
 
 pub(crate) struct MarkerParser;
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index afbb70654f4d..5c3368e4c930 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -297,7 +297,6 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
-        Single>,
         Single>,
         // tidy-alphabetical-end
     ];
diff --git a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
index 12ccc4a6de04..c87f1e41043d 100644
--- a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
+++ b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
@@ -43,7 +43,7 @@ pub(crate) fn expand(
 
     // Generate anonymous constant serving as container for the allocator methods.
     let const_ty = ecx.ty(sig_span, TyKind::Tup(ThinVec::new()));
-    let const_body = ast::ConstItemRhs::Body(ecx.expr_block(ecx.block(span, stmts)));
+    let const_body = ast::ConstItemRhsKind::new_body(ecx.expr_block(ecx.block(span, stmts)));
     let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body);
     let const_item = if is_stmt {
         Annotatable::Stmt(Box::new(ecx.stmt_item(span, const_item)))
diff --git a/compiler/rustc_builtin_macros/src/eii.rs b/compiler/rustc_builtin_macros/src/eii.rs
index 538f5afae5fc..7b651ed84828 100644
--- a/compiler/rustc_builtin_macros/src/eii.rs
+++ b/compiler/rustc_builtin_macros/src/eii.rs
@@ -232,7 +232,7 @@ fn generate_default_impl(
             span,
             underscore,
             unit,
-            ast::ConstItemRhs::Body(ecx.expr_block(ecx.block(span, stmts))),
+            ast::ConstItemRhsKind::new_body(ecx.expr_block(ecx.block(span, stmts))),
         )
     };
 
diff --git a/compiler/rustc_builtin_macros/src/global_allocator.rs b/compiler/rustc_builtin_macros/src/global_allocator.rs
index bb031dafdcfd..9e22d408c125 100644
--- a/compiler/rustc_builtin_macros/src/global_allocator.rs
+++ b/compiler/rustc_builtin_macros/src/global_allocator.rs
@@ -47,7 +47,7 @@ pub(crate) fn expand(
 
     // Generate anonymous constant serving as container for the allocator methods.
     let const_ty = ecx.ty(ty_span, TyKind::Tup(ThinVec::new()));
-    let const_body = ast::ConstItemRhs::Body(ecx.expr_block(ecx.block(span, stmts)));
+    let const_body = ast::ConstItemRhsKind::new_body(ecx.expr_block(ecx.block(span, stmts)));
     let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body);
     let const_item = if is_stmt {
         Annotatable::Stmt(Box::new(ecx.stmt_item(span, const_item)))
diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
index e30d506a31b9..24a5d79958c6 100644
--- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
+++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
@@ -385,7 +385,7 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box {
         cx.attr_nested_word(sym::allow, sym::deprecated, span),
     ]);
 
-    let block = ast::ConstItemRhs::Body(cx.expr_block(
+    let block = ast::ConstItemRhsKind::new_body(cx.expr_block(
         cx.block(span, thin_vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]),
     ));
 
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index d5f774865a9e..a9718d53ac49 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -289,7 +289,7 @@ pub(crate) fn expand_test_or_bench(
                     ty: cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))),
                     define_opaque: None,
                     // test::TestDescAndFn {
-                    rhs: Some(ast::ConstItemRhs::Body(
+                    rhs_kind: ast::ConstItemRhsKind::new_body(
                         cx.expr_struct(
                             sp,
                             test_path("TestDescAndFn"),
@@ -371,7 +371,7 @@ pub(crate) fn expand_test_or_bench(
                         field("testfn", test_fn), // }
                     ],
                         ), // }
-                    )),
+                    ),
                 }
                 .into(),
             ),
diff --git a/compiler/rustc_const_eval/src/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/check_consts/qualifs.rs
index fe3891d0dd7e..462254f064cf 100644
--- a/compiler/rustc_const_eval/src/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/check_consts/qualifs.rs
@@ -345,7 +345,7 @@ where
     let uneval = match constant.const_ {
         Const::Ty(_, ct) => match ct.kind() {
             ty::ConstKind::Param(_) | ty::ConstKind::Error(_) => None,
-            // Unevaluated consts in MIR bodies don't have associated MIR (e.g. `#[type_const]`).
+            // Unevaluated consts in MIR bodies don't have associated MIR (e.g. `type const`).
             ty::ConstKind::Unevaluated(_) => None,
             // FIXME(mgca): Investigate whether using `None` for `ConstKind::Value` is overly
             // strict, and if instead we should be doing some kind of value-based analysis.
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 25da90298791..9f39750d8125 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -394,7 +394,7 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
     typing_env: ty::TypingEnv<'tcx>,
 ) -> Result {
     let def = cid.instance.def.def_id();
-    // #[type_const] don't have bodys
+    // `type const` don't have bodys
     debug_assert!(!tcx.is_type_const(def), "CTFE tried to evaluate type-const: {:?}", def);
 
     let is_static = tcx.is_static(def);
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index e5c06889f3e0..7320ee12abe3 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -727,7 +727,7 @@ impl<'a> ExtCtxt<'a> {
         span: Span,
         ident: Ident,
         ty: Box,
-        rhs: ast::ConstItemRhs,
+        rhs_kind: ast::ConstItemRhsKind,
     ) -> Box {
         let defaultness = ast::Defaultness::Final;
         self.item(
@@ -740,7 +740,7 @@ impl<'a> ExtCtxt<'a> {
                     // FIXME(generic_const_items): Pass the generics as a parameter.
                     generics: ast::Generics::default(),
                     ty,
-                    rhs: Some(rhs),
+                    rhs_kind,
                     define_opaque: None,
                 }
                 .into(),
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 182ef6816337..d74e8442545b 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -888,13 +888,6 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         EncodeCrossCrate::Yes, experimental!(patchable_function_entry)
     ),
 
-    // Probably temporary component of min_generic_const_args.
-    // `#[type_const] const ASSOC: usize;`
-    gated!(
-        type_const, Normal, template!(Word), ErrorFollowing,
-        EncodeCrossCrate::Yes, min_generic_const_args, experimental!(type_const),
-    ),
-
     // The `#[loop_match]` and `#[const_continue]` attributes are part of the
     // lang experiment for RFC 3720 tracked in:
     //
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index a941eb1f459e..add09c3ea58b 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -561,6 +561,8 @@ declare_features! (
     (unstable, macro_metavar_expr_concat, "1.81.0", Some(124225)),
     /// Allows `#[marker]` on certain traits allowing overlapping implementations.
     (unstable, marker_trait_attr, "1.30.0", Some(29864)),
+    /// Enable mgca `type const` syntax before expansion.
+    (incomplete, mgca_type_const_syntax, "CURRENT_RUSTC_VERSION", Some(132980)),
     /// Enables the generic const args MVP (only bare paths, not arbitrary computation).
     (incomplete, min_generic_const_args, "1.84.0", Some(132980)),
     /// A minimal, sound subset of specialization intended to be used by the
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index ef790fe76b74..58a748b9568a 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1292,9 +1292,6 @@ pub enum AttributeKind {
     /// Represents `#[track_caller]`
     TrackCaller(Span),
 
-    /// Represents `#[type_const]`.
-    TypeConst(Span),
-
     /// Represents `#[type_length_limit]`
     TypeLengthLimit { attr_span: Span, limit_span: Span, limit: Limit },
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index cba7679d2df3..7d8ac6df9e3a 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -169,7 +169,6 @@ impl AttributeKind {
             TargetFeature { .. } => No,
             ThreadLocal => No,
             TrackCaller(..) => Yes,
-            TypeConst(..) => Yes,
             TypeLengthLimit { .. } => No,
             UnstableFeatureBound(..) => No,
             Used { .. } => No,
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index cf96f33e9891..68fc7653b636 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -3199,7 +3199,7 @@ impl<'hir> TraitItem<'hir> {
 
     expect_methods_self_kind! {
         expect_const, (&'hir Ty<'hir>, Option>),
-            TraitItemKind::Const(ty, rhs), (ty, *rhs);
+            TraitItemKind::Const(ty, rhs, _), (ty, *rhs);
 
         expect_fn, (&FnSig<'hir>, &TraitFn<'hir>),
             TraitItemKind::Fn(ty, trfn), (ty, trfn);
@@ -3219,11 +3219,32 @@ pub enum TraitFn<'hir> {
     Provided(BodyId),
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, HashStable_Generic)]
+pub enum IsTypeConst {
+    No,
+    Yes,
+}
+
+impl From for IsTypeConst {
+    fn from(value: bool) -> Self {
+        if value { Self::Yes } else { Self::No }
+    }
+}
+
+impl From for bool {
+    fn from(value: IsTypeConst) -> Self {
+        matches!(value, IsTypeConst::Yes)
+    }
+}
+
 /// Represents a trait method or associated constant or type
 #[derive(Debug, Clone, Copy, HashStable_Generic)]
 pub enum TraitItemKind<'hir> {
+    // FIXME(mgca) eventually want to move the option that is around `ConstItemRhs<'hir>`
+    // into `ConstItemRhs`, much like `ast::ConstItemRhsKind`, but for now mark whether
+    // this node is a TypeConst with a flag.
     /// An associated constant with an optional value (otherwise `impl`s must contain a value).
-    Const(&'hir Ty<'hir>, Option>),
+    Const(&'hir Ty<'hir>, Option>, IsTypeConst),
     /// An associated function with an optional body.
     Fn(FnSig<'hir>, TraitFn<'hir>),
     /// An associated type with (possibly empty) bounds and optional concrete
@@ -4686,7 +4707,7 @@ impl<'hir> OwnerNode<'hir> {
             | OwnerNode::TraitItem(TraitItem {
                 kind:
                     TraitItemKind::Fn(_, TraitFn::Provided(body))
-                    | TraitItemKind::Const(_, Some(ConstItemRhs::Body(body))),
+                    | TraitItemKind::Const(_, Some(ConstItemRhs::Body(body)), _),
                 ..
             })
             | OwnerNode::ImplItem(ImplItem {
@@ -4913,7 +4934,7 @@ impl<'hir> Node<'hir> {
                 _ => None,
             },
             Node::TraitItem(it) => match it.kind {
-                TraitItemKind::Const(ty, _) => Some(ty),
+                TraitItemKind::Const(ty, _, _) => Some(ty),
                 TraitItemKind::Type(_, ty) => ty,
                 _ => None,
             },
@@ -4956,7 +4977,7 @@ impl<'hir> Node<'hir> {
             | Node::TraitItem(TraitItem {
                 owner_id,
                 kind:
-                    TraitItemKind::Const(.., Some(ConstItemRhs::Body(body)))
+                    TraitItemKind::Const(_, Some(ConstItemRhs::Body(body)), _)
                     | TraitItemKind::Fn(_, TraitFn::Provided(body)),
                 ..
             })
diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs
index bd863abaceb4..21bcf53b5619 100644
--- a/compiler/rustc_hir/src/intravisit.rs
+++ b/compiler/rustc_hir/src/intravisit.rs
@@ -1271,7 +1271,7 @@ pub fn walk_trait_item<'v, V: Visitor<'v>>(
     try_visit!(visitor.visit_defaultness(&defaultness));
     try_visit!(visitor.visit_id(hir_id));
     match *kind {
-        TraitItemKind::Const(ref ty, default) => {
+        TraitItemKind::Const(ref ty, default, _) => {
             try_visit!(visitor.visit_ty_unambig(ty));
             visit_opt!(visitor, visit_const_item_rhs, default);
         }
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index e0bfae9617ff..1bee7a72229a 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -923,7 +923,7 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(),
                 );
                 check_where_clauses(wfcx, def_id);
 
-                if tcx.is_type_const(def_id.into()) {
+                if tcx.is_type_const(def_id) {
                     wfcheck::check_type_const(wfcx, def_id, ty, true)?;
                 }
                 Ok(())
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
index cd9fdcb59fb5..c29ebe67a158 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
@@ -2060,14 +2060,14 @@ fn compare_type_const<'tcx>(
             .dcx()
             .struct_span_err(
                 tcx.def_span(impl_const_item.def_id),
-                "implementation of `#[type_const]` const must be marked with `#[type_const]`",
+                "implementation of a `type const` must also be marked as `type const`",
             )
             .with_span_note(
                 MultiSpan::from_spans(vec![
                     tcx.def_span(trait_const_item.def_id),
                     trait_type_const_span,
                 ]),
-                "trait declaration of const is marked with `#[type_const]`",
+                "trait declaration of const is marked as `type const`",
             )
             .emit());
     }
diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index f917c799ef5a..ceb0e138a0da 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -955,7 +955,7 @@ pub(crate) fn check_associated_item(
                 wfcx.register_wf_obligation(span, loc, ty.into());
 
                 let has_value = item.defaultness(tcx).has_value();
-                if tcx.is_type_const(def_id.into()) {
+                if tcx.is_type_const(def_id) {
                     check_type_const(wfcx, def_id, ty, has_value)?;
                 }
 
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index f3086266eec0..e72fde9f0568 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -95,6 +95,7 @@ pub(crate) fn provide(providers: &mut Providers) {
         const_param_default,
         anon_const_kind,
         const_of_item,
+        is_rhs_type_const,
         ..*providers
     };
 }
@@ -1549,7 +1550,8 @@ fn is_anon_const_rhs_of_const_item<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId)
     let (Node::Item(hir::Item { kind: hir::ItemKind::Const(_, _, _, ct_rhs), .. })
     | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(_, ct_rhs), .. })
     | Node::TraitItem(hir::TraitItem {
-        kind: hir::TraitItemKind::Const(_, Some(ct_rhs)), ..
+        kind: hir::TraitItemKind::Const(_, Some(ct_rhs), _),
+        ..
     })) = grandparent_node
     else {
         return false;
@@ -1594,7 +1596,7 @@ fn const_of_item<'tcx>(
     let ct_rhs = match tcx.hir_node_by_def_id(def_id) {
         hir::Node::Item(hir::Item { kind: hir::ItemKind::Const(.., ct), .. }) => *ct,
         hir::Node::TraitItem(hir::TraitItem {
-            kind: hir::TraitItemKind::Const(.., ct), ..
+            kind: hir::TraitItemKind::Const(_, ct, _), ..
         }) => ct.expect("no default value for trait assoc const"),
         hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(.., ct), .. }) => *ct,
         _ => {
@@ -1624,3 +1626,22 @@ fn const_of_item<'tcx>(
         ty::EarlyBinder::bind(ct)
     }
 }
+
+/// Check if a Const or AssocConst is a type const (mgca)
+fn is_rhs_type_const<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> bool {
+    match tcx.hir_node_by_def_id(def) {
+        hir::Node::Item(hir::Item {
+            kind: hir::ItemKind::Const(_, _, _, hir::ConstItemRhs::TypeConst(_)),
+            ..
+        })
+        | hir::Node::ImplItem(hir::ImplItem {
+            kind: hir::ImplItemKind::Const(_, hir::ConstItemRhs::TypeConst(_)),
+            ..
+        })
+        | hir::Node::TraitItem(hir::TraitItem {
+            kind: hir::TraitItemKind::Const(_, _, hir::IsTypeConst::Yes),
+            ..
+        }) => return true,
+        _ => return false,
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
index d4a01d26a1fc..3e687700f11c 100644
--- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
+++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
@@ -859,7 +859,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                     }
                 })
             }
-            Const(_, _) => self.visit_early(trait_item.hir_id(), trait_item.generics, |this| {
+            Const(_, _, _) => self.visit_early(trait_item.hir_id(), trait_item.generics, |this| {
                 intravisit::walk_trait_item(this, trait_item)
             }),
         }
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs
index 910176a0689c..23df419d06a7 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs
@@ -63,7 +63,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                 let args = ty::GenericArgs::identity_for_item(tcx, def_id);
                 Ty::new_fn_def(tcx, def_id.to_def_id(), args)
             }
-            TraitItemKind::Const(ty, rhs) => rhs
+            TraitItemKind::Const(ty, rhs, _) => rhs
                 .and_then(|rhs| {
                     ty.is_suggestable_infer_ty().then(|| {
                         infer_placeholder_type(
@@ -420,9 +420,9 @@ fn infer_placeholder_type<'tcx>(
     kind: &'static str,
 ) -> Ty<'tcx> {
     let tcx = cx.tcx();
-    // If the type is omitted on a #[type_const] we can't run
+    // If the type is omitted on a `type const` we can't run
     // type check on since that requires the const have a body
-    // which type_consts don't.
+    // which `type const`s don't.
     let ty = if tcx.is_type_const(def_id.to_def_id()) {
         if let Some(trait_item_def_id) = tcx.trait_item_of(def_id.to_def_id()) {
             tcx.type_of(trait_item_def_id).instantiate_identity()
@@ -430,7 +430,7 @@ fn infer_placeholder_type<'tcx>(
             Ty::new_error_with_message(
                 tcx,
                 ty_span,
-                "constant with #[type_const] requires an explicit type",
+                "constant with `type const` requires an explicit type",
             )
         }
     } else {
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
index a1c3af5f999d..4a03b290c33d 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
@@ -607,14 +607,14 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                             if tcx.features().min_generic_const_args() {
                                 let mut err = self.dcx().struct_span_err(
                                     constraint.span,
-                                    "use of trait associated const without `#[type_const]`",
+                                    "use of trait associated const not defined as `type const`",
                                 );
-                                err.note("the declaration in the trait must be marked with `#[type_const]`");
+                                err.note("the declaration in the trait must begin with `type const` not just `const` alone");
                                 return Err(err.emit());
                             } else {
                                 let err = self.dcx().span_delayed_bug(
                                     constraint.span,
-                                    "use of trait associated const without `#[type_const]`",
+                                    "use of trait associated const defined as `type const`",
                                 );
                                 return Err(err);
                             }
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index 7a03d8b220c5..f58461e5d0e1 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -2849,19 +2849,20 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         if tcx.is_type_const(def_id) {
             Ok(())
         } else {
-            let mut err = self
-                .dcx()
-                .struct_span_err(span, "use of `const` in the type system without `#[type_const]`");
+            let mut err = self.dcx().struct_span_err(
+                span,
+                "use of `const` in the type system not defined as `type const`",
+            );
             if def_id.is_local() {
                 let name = tcx.def_path_str(def_id);
                 err.span_suggestion(
                     tcx.def_span(def_id).shrink_to_lo(),
-                    format!("add `#[type_const]` attribute to `{name}`"),
-                    format!("#[type_const]\n"),
+                    format!("add `type` before `const` for `{name}`"),
+                    format!("type "),
                     Applicability::MaybeIncorrect,
                 );
             } else {
-                err.note("only consts marked with `#[type_const]` may be used in types");
+                err.note("only consts marked defined as `type const` may be used in types");
             }
             Err(err.emit())
         }
diff --git a/compiler/rustc_hir_analysis/src/hir_wf_check.rs b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
index d414f4dbcc24..08da8d19344d 100644
--- a/compiler/rustc_hir_analysis/src/hir_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
@@ -139,7 +139,7 @@ pub(super) fn diagnostic_hir_wf_check<'tcx>(
             },
             hir::Node::TraitItem(item) => match item.kind {
                 hir::TraitItemKind::Type(_, ty) => ty.into_iter().collect(),
-                hir::TraitItemKind::Const(ty, _) => vec![ty],
+                hir::TraitItemKind::Const(ty, _, _) => vec![ty],
                 ref item => bug!("Unexpected TraitItem {:?}", item),
             },
             hir::Node::Item(item) => match item.kind {
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index 28130383e94c..6214106d422b 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -233,7 +233,7 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
             }
             DefKind::Const
                 if !tcx.generics_of(item_def_id).own_requires_monomorphization()
-                    && !tcx.is_type_const(item_def_id.into()) =>
+                    && !tcx.is_type_const(item_def_id) =>
             {
                 // FIXME(generic_const_items): Passing empty instead of identity args is fishy but
                 //                             seems to be fine for now. Revisit this!
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index d045ec0eb6c7..fc6cc4e67251 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -937,7 +937,7 @@ impl<'a> State<'a> {
         self.maybe_print_comment(ti.span.lo());
         self.print_attrs(self.attrs(ti.hir_id()));
         match ti.kind {
-            hir::TraitItemKind::Const(ty, default) => {
+            hir::TraitItemKind::Const(ty, default, _) => {
                 self.print_associated_const(ti.ident, ti.generics, ty, default);
             }
             hir::TraitItemKind::Fn(ref sig, hir::TraitFn::Required(arg_idents)) => {
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 51d9a4641cd5..8b2a6d1d2ab5 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -1058,8 +1058,8 @@ trait UnusedDelimLint {
     fn check_item(&mut self, cx: &EarlyContext<'_>, item: &ast::Item) {
         use ast::ItemKind::*;
 
-        let expr = if let Const(box ast::ConstItem { rhs: Some(rhs), .. }) = &item.kind {
-            rhs.expr()
+        let expr = if let Const(box ast::ConstItem { rhs_kind, .. }) = &item.kind {
+            if let Some(e) = rhs_kind.expr() { e } else { return }
         } else if let Static(box ast::StaticItem { expr: Some(expr), .. }) = &item.kind {
             expr
         } else {
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index c6c87534851c..6ea9a2852804 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -418,6 +418,7 @@ provide! { tcx, def_id, other, cdata,
     }
     anon_const_kind => { table }
     const_of_item => { table }
+    is_rhs_type_const => { table }
 }
 
 pub(in crate::rmeta) fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index d8652539d0c5..4988cafdd363 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -1630,6 +1630,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 let table = tcx.associated_types_for_impl_traits_in_trait_or_impl(def_id);
                 record!(self.tables.associated_types_for_impl_traits_in_trait_or_impl[def_id] <- table);
             }
+            if let DefKind::AssocConst | DefKind::Const = def_kind {
+                record!(self.tables.is_rhs_type_const[def_id] <- self.tcx.is_rhs_type_const(def_id));
+            }
         }
 
         for (def_id, impls) in &tcx.crate_inherent_impls(()).0.inherent_impls {
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index af6df0cd6eb6..408b50ae48df 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -476,6 +476,7 @@ define_tables! {
     anon_const_kind: Table>,
     const_of_item: Table>>>,
     associated_types_for_impl_traits_in_trait_or_impl: Table>>>,
+    is_rhs_type_const: Table>,
 }
 
 #[derive(TyEncodable, TyDecodable)]
diff --git a/compiler/rustc_middle/src/queries.rs b/compiler/rustc_middle/src/queries.rs
index de2298914438..771c300989f2 100644
--- a/compiler/rustc_middle/src/queries.rs
+++ b/compiler/rustc_middle/src/queries.rs
@@ -293,13 +293,13 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// Returns the const of the RHS of a (free or assoc) const item, if it is a `#[type_const]`.
+    /// Returns the const of the RHS of a (free or assoc) const item, if it is a `type const`.
     ///
     /// When a const item is used in a type-level expression, like in equality for an assoc const
     /// projection, this allows us to retrieve the typesystem-appropriate representation of the
     /// const value.
     ///
-    /// This query will ICE if given a const that is not marked with `#[type_const]`.
+    /// This query will ICE if given a const that is not marked with `type const`.
     query const_of_item(def_id: DefId) -> ty::EarlyBinder<'tcx, ty::Const<'tcx>> {
         desc { |tcx| "computing the type-level value for `{}`", tcx.def_path_str(def_id)  }
         cache_on_disk_if { def_id.is_local() }
@@ -2764,6 +2764,12 @@ rustc_queries! {
         cache_on_disk_if { *cnum == LOCAL_CRATE }
         separate_provide_extern
     }
+
+    query is_rhs_type_const(def_id: DefId) -> bool {
+        desc { |tcx| "checking whether `{}` is a rhs type const", tcx.def_path_str(def_id) }
+        cache_on_disk_if { def_id.is_local() }
+        separate_provide_extern
+    }
 }
 
 rustc_with_all_queries! { define_callbacks! }
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 75323bf5c8cc..2fca85c9f2f1 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -845,7 +845,7 @@ impl DynCompatibilityViolation {
                 format!("it contains generic associated const `{name}`").into()
             }
             Self::AssocConst(name, AssocConstViolation::NonType, _) => {
-                format!("it contains associated const `{name}` that's not marked `#[type_const]`")
+                format!("it contains associated const `{name}` that's not defined as `type const`")
                     .into()
             }
             Self::AssocConst(name, AssocConstViolation::TypeReferencesSelf, _) => format!(
@@ -999,7 +999,7 @@ pub enum AssocConstViolation {
     /// Has own generic parameters (GAC).
     Generic,
 
-    /// Isn't marked `#[type_const]`.
+    /// Isn't defined as `type const`.
     NonType,
 
     /// Its type mentions the `Self` type parameter.
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 1bb83530487d..45bb584bf595 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -1890,14 +1890,22 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn type_const_span(self, def_id: DefId) -> Option {
-        matches!(self.def_kind(def_id), DefKind::Const | DefKind::AssocConst)
-            .then(|| find_attr!(self.get_all_attrs(def_id), AttributeKind::TypeConst(sp) => *sp))
-            .flatten()
+        if !self.is_type_const(def_id) {
+            return None;
+        }
+        Some(self.def_span(def_id))
     }
 
-    /// Check if the given `def_id` is a const with the `#[type_const]` attribute.
-    pub fn is_type_const(self, def_id: DefId) -> bool {
-        self.type_const_span(def_id).is_some()
+    /// Check if the given `def_id` is a `type const` (mgca)
+    pub fn is_type_const>(self, def_id: I) -> bool {
+        // No need to call the query directly in this case always false.
+        if !(matches!(
+            self.def_kind(def_id.into_query_param()),
+            DefKind::Const | DefKind::AssocConst
+        )) {
+            return false;
+        }
+        self.is_rhs_type_const(def_id)
     }
 
     /// Returns the movability of the coroutine of `def_id`, or panics
@@ -2923,7 +2931,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ) -> bool {
         let generics = self.generics_of(def_id);
 
-        // IATs and IACs (inherent associated types/consts with #[type_const]) themselves have a
+        // IATs and IACs (inherent associated types/consts with `type const`) themselves have a
         // weird arg setup (self + own args), but nested items *in* IATs (namely: opaques, i.e.
         // ATPITs) do not.
         let is_inherent_assoc_ty = matches!(self.def_kind(def_id), DefKind::AssocTy)
diff --git a/compiler/rustc_mir_build/src/builder/mod.rs b/compiler/rustc_mir_build/src/builder/mod.rs
index 317df4e64fdb..fbd7aa90f49c 100644
--- a/compiler/rustc_mir_build/src/builder/mod.rs
+++ b/compiler/rustc_mir_build/src/builder/mod.rs
@@ -578,7 +578,7 @@ fn construct_const<'a, 'tcx>(
         })
         | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(ty, _), span, .. })
         | Node::TraitItem(hir::TraitItem {
-            kind: hir::TraitItemKind::Const(ty, Some(_)),
+            kind: hir::TraitItemKind::Const(ty, Some(_), _),
             span,
             ..
         }) => (*span, ty.span),
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 9ac9bd5c0e46..5b81acb0f91f 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -6,9 +6,7 @@ use rustc_ast::ast::*;
 use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_ast::util::case::Case;
-use rustc_ast::{
-    attr, {self as ast},
-};
+use rustc_ast::{self as ast};
 use rustc_ast_pretty::pprust;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, PResult, StashKey, inline_fluent, struct_span_code_err};
@@ -286,13 +284,13 @@ impl<'a> Parser<'a> {
             // CONST ITEM
             self.recover_const_mut(const_span);
             self.recover_missing_kw_before_item()?;
-            let (ident, generics, ty, rhs) = self.parse_const_item(attrs)?;
+            let (ident, generics, ty, rhs_kind) = self.parse_const_item(false)?;
             ItemKind::Const(Box::new(ConstItem {
                 defaultness: def_(),
                 ident,
                 generics,
                 ty,
-                rhs,
+                rhs_kind,
                 define_opaque: None,
             }))
         } else if let Some(kind) = self.is_reuse_item() {
@@ -303,8 +301,26 @@ impl<'a> Parser<'a> {
             // MODULE ITEM
             self.parse_item_mod(attrs)?
         } else if self.eat_keyword_case(exp!(Type), case) {
-            // TYPE ITEM
-            self.parse_type_alias(def_())?
+            if let Const::Yes(const_span) = self.parse_constness(case) {
+                // TYPE CONST (mgca)
+                self.recover_const_mut(const_span);
+                self.recover_missing_kw_before_item()?;
+                let (ident, generics, ty, rhs_kind) = self.parse_const_item(true)?;
+                // Make sure this is only allowed if the feature gate is enabled.
+                // #![feature(mgca_type_const_syntax)]
+                self.psess.gated_spans.gate(sym::mgca_type_const_syntax, lo.to(const_span));
+                ItemKind::Const(Box::new(ConstItem {
+                    defaultness: def_(),
+                    ident,
+                    generics,
+                    ty,
+                    rhs_kind,
+                    define_opaque: None,
+                }))
+            } else {
+                // TYPE ITEM
+                self.parse_type_alias(def_())?
+            }
         } else if self.eat_keyword_case(exp!(Enum), case) {
             // ENUM ITEM
             self.parse_item_enum()?
@@ -1113,13 +1129,12 @@ impl<'a> Parser<'a> {
                             define_opaque,
                         }) => {
                             self.dcx().emit_err(errors::AssociatedStaticItemNotAllowed { span });
-                            let rhs = expr.map(ConstItemRhs::Body);
                             AssocItemKind::Const(Box::new(ConstItem {
                                 defaultness: Defaultness::Final,
                                 ident,
                                 generics: Generics::default(),
                                 ty,
-                                rhs,
+                                rhs_kind: ConstItemRhsKind::Body { rhs: expr },
                                 define_opaque,
                             }))
                         }
@@ -1360,7 +1375,7 @@ impl<'a> Parser<'a> {
                 let kind = match ForeignItemKind::try_from(kind) {
                     Ok(kind) => kind,
                     Err(kind) => match kind {
-                        ItemKind::Const(box ConstItem { ident, ty, rhs, .. }) => {
+                        ItemKind::Const(box ConstItem { ident, ty, rhs_kind, .. }) => {
                             let const_span = Some(span.with_hi(ident.span.lo()))
                                 .filter(|span| span.can_be_used_for_suggestions());
                             self.dcx().emit_err(errors::ExternItemCannotBeConst {
@@ -1371,10 +1386,13 @@ impl<'a> Parser<'a> {
                                 ident,
                                 ty,
                                 mutability: Mutability::Not,
-                                expr: rhs.map(|b| match b {
-                                    ConstItemRhs::TypeConst(anon_const) => anon_const.value,
-                                    ConstItemRhs::Body(expr) => expr,
-                                }),
+                                expr: match rhs_kind {
+                                    ConstItemRhsKind::Body { rhs } => rhs,
+                                    ConstItemRhsKind::TypeConst { rhs: Some(anon) } => {
+                                        Some(anon.value)
+                                    }
+                                    ConstItemRhsKind::TypeConst { rhs: None } => None,
+                                },
                                 safety: Safety::Default,
                                 define_opaque: None,
                             }))
@@ -1516,13 +1534,16 @@ impl<'a> Parser<'a> {
 
     /// Parse a constant item with the prefix `"const"` already parsed.
     ///
+    /// If `const_arg` is true, any expression assigned to the const will be parsed
+    /// as a const_arg instead of a body expression.
+    ///
     /// ```ebnf
     /// Const = "const" ($ident | "_") Generics ":" $ty (= $expr)? WhereClause ";" ;
     /// ```
     fn parse_const_item(
         &mut self,
-        attrs: &[Attribute],
-    ) -> PResult<'a, (Ident, Generics, Box, Option)> {
+        const_arg: bool,
+    ) -> PResult<'a, (Ident, Generics, Box, ConstItemRhsKind)> {
         let ident = self.parse_ident_or_underscore()?;
 
         let mut generics = self.parse_generics()?;
@@ -1549,16 +1570,15 @@ impl<'a> Parser<'a> {
         let before_where_clause =
             if self.may_recover() { self.parse_where_clause()? } else { WhereClause::default() };
 
-        let rhs = if self.eat(exp!(Eq)) {
-            if attr::contains_name(attrs, sym::type_const) {
-                let ct =
-                    self.parse_expr_anon_const(|this, expr| this.mgca_direct_lit_hack(expr))?;
-                Some(ConstItemRhs::TypeConst(ct))
-            } else {
-                Some(ConstItemRhs::Body(self.parse_expr()?))
-            }
-        } else {
-            None
+        let rhs = match (self.eat(exp!(Eq)), const_arg) {
+            (true, true) => ConstItemRhsKind::TypeConst {
+                rhs: Some(
+                    self.parse_expr_anon_const(|this, expr| this.mgca_direct_lit_hack(expr))?,
+                ),
+            },
+            (true, false) => ConstItemRhsKind::Body { rhs: Some(self.parse_expr()?) },
+            (false, true) => ConstItemRhsKind::TypeConst { rhs: None },
+            (false, false) => ConstItemRhsKind::Body { rhs: None },
         };
 
         let after_where_clause = self.parse_where_clause()?;
@@ -1567,18 +1587,18 @@ impl<'a> Parser<'a> {
         // Users may be tempted to write such code if they are still used to the deprecated
         // where-clause location on type aliases and associated types. See also #89122.
         if before_where_clause.has_where_token
-            && let Some(rhs) = &rhs
+            && let Some(rhs_span) = rhs.span()
         {
             self.dcx().emit_err(errors::WhereClauseBeforeConstBody {
                 span: before_where_clause.span,
                 name: ident.span,
-                body: rhs.span(),
+                body: rhs_span,
                 sugg: if !after_where_clause.has_where_token {
-                    self.psess.source_map().span_to_snippet(rhs.span()).ok().map(|body_s| {
+                    self.psess.source_map().span_to_snippet(rhs_span).ok().map(|body_s| {
                         errors::WhereClauseBeforeConstBodySugg {
                             left: before_where_clause.span.shrink_to_lo(),
                             snippet: body_s,
-                            right: before_where_clause.span.shrink_to_hi().to(rhs.span()),
+                            right: before_where_clause.span.shrink_to_hi().to(rhs_span),
                         }
                     })
                 } else {
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index dd0de90d575b..e53922e3734b 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -356,7 +356,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcVarianceOfOpaques
                     | AttributeKind::ShouldPanic { .. }
                     | AttributeKind::ThreadLocal
-                    | AttributeKind::TypeConst{..}
                     | AttributeKind::TypeLengthLimit { .. }
                     | AttributeKind::UnstableFeatureBound(..)
                     | AttributeKind::Used { .. }
diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs
index d9565e2dae0e..225487cc989f 100644
--- a/compiler/rustc_passes/src/reachable.rs
+++ b/compiler/rustc_passes/src/reachable.rs
@@ -145,7 +145,7 @@ impl<'tcx> ReachableContext<'tcx> {
                 _ => false,
             },
             Node::TraitItem(trait_method) => match trait_method.kind {
-                hir::TraitItemKind::Const(_, ref default) => default.is_some(),
+                hir::TraitItemKind::Const(_, ref default, _) => default.is_some(),
                 hir::TraitItemKind::Fn(_, hir::TraitFn::Provided(_)) => true,
                 hir::TraitItemKind::Fn(_, hir::TraitFn::Required(_))
                 | hir::TraitItemKind::Type(..) => false,
@@ -209,7 +209,7 @@ impl<'tcx> ReachableContext<'tcx> {
                             self.visit_nested_body(body);
                         }
                     }
-                    // For #[type_const] we want to evaluate the RHS.
+                    // For `type const` we want to evaluate the RHS.
                     hir::ItemKind::Const(_, _, _, init @ hir::ConstItemRhs::TypeConst(_)) => {
                         self.visit_const_item_rhs(init);
                     }
@@ -258,11 +258,11 @@ impl<'tcx> ReachableContext<'tcx> {
             }
             Node::TraitItem(trait_method) => {
                 match trait_method.kind {
-                    hir::TraitItemKind::Const(_, None)
+                    hir::TraitItemKind::Const(_, None, _)
                     | hir::TraitItemKind::Fn(_, hir::TraitFn::Required(_)) => {
                         // Keep going, nothing to get exported
                     }
-                    hir::TraitItemKind::Const(_, Some(rhs)) => self.visit_const_item_rhs(rhs),
+                    hir::TraitItemKind::Const(_, Some(rhs), _) => self.visit_const_item_rhs(rhs),
                     hir::TraitItemKind::Fn(_, hir::TraitFn::Provided(body_id)) => {
                         self.visit_nested_body(body_id);
                     }
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 4faf7715de6e..fe6346f6c6e9 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -2847,11 +2847,10 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 ident,
                 generics,
                 ty,
-                rhs,
+                rhs_kind,
                 define_opaque,
                 defaultness: _,
             }) => {
-                let is_type_const = attr::contains_name(&item.attrs, sym::type_const);
                 self.with_generic_param_rib(
                     &generics.params,
                     RibKind::Item(
@@ -2871,7 +2870,7 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                         this.with_lifetime_rib(
                             LifetimeRibKind::Elided(LifetimeRes::Static),
                             |this| {
-                                if is_type_const
+                                if rhs_kind.is_type_const()
                                     && !this.r.tcx.features().generic_const_parameter_types()
                                 {
                                     this.with_rib(TypeNS, RibKind::ConstParamTy, |this| {
@@ -2888,12 +2887,10 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                             },
                         );
 
-                        if let Some(rhs) = rhs {
-                            this.resolve_const_item_rhs(
-                                rhs,
-                                Some((*ident, ConstantItemKind::Const)),
-                            );
-                        }
+                        this.resolve_const_item_rhs(
+                            rhs_kind,
+                            Some((*ident, ConstantItemKind::Const)),
+                        );
                     },
                 );
                 self.resolve_define_opaques(define_opaque);
@@ -3242,11 +3239,10 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 AssocItemKind::Const(box ast::ConstItem {
                     generics,
                     ty,
-                    rhs,
+                    rhs_kind,
                     define_opaque,
                     ..
                 }) => {
-                    let is_type_const = attr::contains_name(&item.attrs, sym::type_const);
                     self.with_generic_param_rib(
                         &generics.params,
                         RibKind::AssocItem,
@@ -3261,7 +3257,7 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                                 },
                                 |this| {
                                     this.visit_generics(generics);
-                                    if is_type_const
+                                    if rhs_kind.is_type_const()
                                         && !this.r.tcx.features().generic_const_parameter_types()
                                     {
                                         this.with_rib(TypeNS, RibKind::ConstParamTy, |this| {
@@ -3278,14 +3274,13 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
 
                                     // Only impose the restrictions of `ConstRibKind` for an
                                     // actual constant expression in a provided default.
-                                    if let Some(rhs) = rhs {
-                                        // We allow arbitrary const expressions inside of associated consts,
-                                        // even if they are potentially not const evaluatable.
-                                        //
-                                        // Type parameters can already be used and as associated consts are
-                                        // not used as part of the type system, this is far less surprising.
-                                        this.resolve_const_item_rhs(rhs, None);
-                                    }
+                                    //
+                                    // We allow arbitrary const expressions inside of associated consts,
+                                    // even if they are potentially not const evaluatable.
+                                    //
+                                    // Type parameters can already be used and as associated consts are
+                                    // not used as part of the type system, this is far less surprising.
+                                    this.resolve_const_item_rhs(rhs_kind, None);
                                 },
                             )
                         },
@@ -3463,12 +3458,11 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 ident,
                 generics,
                 ty,
-                rhs,
+                rhs_kind,
                 define_opaque,
                 ..
             }) => {
                 debug!("resolve_implementation AssocItemKind::Const");
-                let is_type_const = attr::contains_name(&item.attrs, sym::type_const);
                 self.with_generic_param_rib(
                     &generics.params,
                     RibKind::AssocItem,
@@ -3505,7 +3499,7 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                                         );
 
                                         this.visit_generics(generics);
-                                        if is_type_const
+                                        if rhs_kind.is_type_const()
                                             && !this
                                                 .r
                                                 .tcx
@@ -3527,14 +3521,12 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                                         } else {
                                             this.visit_ty(ty);
                                         }
-                                        if let Some(rhs) = rhs {
-                                            // We allow arbitrary const expressions inside of associated consts,
-                                            // even if they are potentially not const evaluatable.
-                                            //
-                                            // Type parameters can already be used and as associated consts are
-                                            // not used as part of the type system, this is far less surprising.
-                                            this.resolve_const_item_rhs(rhs, None);
-                                        }
+                                        // We allow arbitrary const expressions inside of associated consts,
+                                        // even if they are potentially not const evaluatable.
+                                        //
+                                        // Type parameters can already be used and as associated consts are
+                                        // not used as part of the type system, this is far less surprising.
+                                        this.resolve_const_item_rhs(rhs_kind, None);
                                     },
                                 )
                             },
@@ -3756,18 +3748,19 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
 
     fn resolve_const_item_rhs(
         &mut self,
-        rhs: &'ast ConstItemRhs,
+        rhs_kind: &'ast ConstItemRhsKind,
         item: Option<(Ident, ConstantItemKind)>,
     ) {
-        self.with_lifetime_rib(LifetimeRibKind::Elided(LifetimeRes::Infer), |this| match rhs {
-            ConstItemRhs::TypeConst(anon_const) => {
+        self.with_lifetime_rib(LifetimeRibKind::Elided(LifetimeRes::Infer), |this| match rhs_kind {
+            ConstItemRhsKind::TypeConst { rhs: Some(anon_const) } => {
                 this.resolve_anon_const(anon_const, AnonConstKind::ConstArg(IsRepeatExpr::No));
             }
-            ConstItemRhs::Body(expr) => {
+            ConstItemRhsKind::Body { rhs: Some(expr) } => {
                 this.with_constant_rib(IsRepeatExpr::No, ConstantHasGenerics::Yes, item, |this| {
                     this.visit_expr(expr)
                 });
             }
+            _ => (),
         })
     }
 
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 6aa2eae556e2..2c34c99b0539 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1453,6 +1453,7 @@ symbols! {
         meta,
         meta_sized,
         metadata_type,
+        mgca_type_const_syntax,
         min_const_fn,
         min_const_generics,
         min_const_unsafe_fn,
@@ -2339,7 +2340,6 @@ symbols! {
         type_ascribe,
         type_ascription,
         type_changing_struct_update,
-        type_const,
         type_id,
         type_id_eq,
         type_info,
diff --git a/compiler/rustc_trait_selection/src/traits/normalize.rs b/compiler/rustc_trait_selection/src/traits/normalize.rs
index 46d95fdfabfe..3f4c75a5b133 100644
--- a/compiler/rustc_trait_selection/src/traits/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/normalize.rs
@@ -453,7 +453,7 @@ impl<'a, 'b, 'tcx> TypeFolder> for AssocTypeNormalizer<'a, 'b, 'tcx
         // been emitted earlier in compilation.
         //
         // That's because we can only end up with an Unevaluated ty::Const for a const item
-        // if it was marked with `#[type_const]`. Using this attribute without the mgca
+        // if it was marked with `type const`. Using this attribute without the mgca
         // feature gate causes a parse error.
         let ct = match tcx.def_kind(uv.def) {
             DefKind::AssocConst => match tcx.def_kind(tcx.parent(uv.def)) {
diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs
index 753fba3ed482..c09e17d3787e 100644
--- a/src/librustdoc/clean/mod.rs
+++ b/src/librustdoc/clean/mod.rs
@@ -1209,14 +1209,14 @@ fn clean_trait_item<'tcx>(trait_item: &hir::TraitItem<'tcx>, cx: &mut DocContext
     let local_did = trait_item.owner_id.to_def_id();
     cx.with_param_env(local_did, |cx| {
         let inner = match trait_item.kind {
-            hir::TraitItemKind::Const(ty, Some(default)) => {
+            hir::TraitItemKind::Const(ty, Some(default), _) => {
                 ProvidedAssocConstItem(Box::new(Constant {
                     generics: enter_impl_trait(cx, |cx| clean_generics(trait_item.generics, cx)),
                     kind: clean_const_item_rhs(default, local_did),
                     type_: clean_ty(ty, cx),
                 }))
             }
-            hir::TraitItemKind::Const(ty, None) => {
+            hir::TraitItemKind::Const(ty, None, _) => {
                 let generics = enter_impl_trait(cx, |cx| clean_generics(trait_item.generics, cx));
                 RequiredAssocConstItem(generics, Box::new(clean_ty(ty, cx)))
             }
diff --git a/src/tools/clippy/clippy_lints/src/non_copy_const.rs b/src/tools/clippy/clippy_lints/src/non_copy_const.rs
index f99748127a8f..e09fc0a76366 100644
--- a/src/tools/clippy/clippy_lints/src/non_copy_const.rs
+++ b/src/tools/clippy/clippy_lints/src/non_copy_const.rs
@@ -739,7 +739,7 @@ impl<'tcx> LateLintPass<'tcx> for NonCopyConst<'tcx> {
     }
 
     fn check_trait_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx TraitItem<'_>) {
-        if let TraitItemKind::Const(_, ct_rhs_opt) = item.kind
+        if let TraitItemKind::Const(_, ct_rhs_opt, _) = item.kind
             && let ty = cx.tcx.type_of(item.owner_id).instantiate_identity()
             && match self.is_ty_freeze(cx.tcx, cx.typing_env(), ty) {
                 IsFreeze::No => true,
@@ -931,7 +931,7 @@ fn get_const_hir_value<'tcx>(
         {
             match tcx.hir_node(tcx.local_def_id_to_hir_id(did)) {
                 Node::ImplItem(item) if let ImplItemKind::Const(.., ct_rhs) = item.kind => (did, ct_rhs),
-                Node::TraitItem(item) if let TraitItemKind::Const(.., Some(ct_rhs)) = item.kind => (did, ct_rhs),
+                Node::TraitItem(item) if let TraitItemKind::Const(_, Some(ct_rhs), _) = item.kind => (did, ct_rhs),
                 _ => return None,
             }
         },
diff --git a/src/tools/clippy/clippy_lints/src/types/mod.rs b/src/tools/clippy/clippy_lints/src/types/mod.rs
index 7018146f184b..7d97ce97f487 100644
--- a/src/tools/clippy/clippy_lints/src/types/mod.rs
+++ b/src/tools/clippy/clippy_lints/src/types/mod.rs
@@ -514,7 +514,7 @@ impl<'tcx> LateLintPass<'tcx> for Types {
         };
 
         match item.kind {
-            TraitItemKind::Const(ty, _) | TraitItemKind::Type(_, Some(ty)) => {
+            TraitItemKind::Const(ty, _, _) | TraitItemKind::Type(_, Some(ty)) => {
                 self.check_ty(cx, ty, context);
             },
             TraitItemKind::Fn(ref sig, trait_method) => {
diff --git a/src/tools/clippy/clippy_utils/src/ast_utils/mod.rs b/src/tools/clippy/clippy_utils/src/ast_utils/mod.rs
index cf8716398efb..3b043f7565ef 100644
--- a/src/tools/clippy/clippy_utils/src/ast_utils/mod.rs
+++ b/src/tools/clippy/clippy_utils/src/ast_utils/mod.rs
@@ -355,7 +355,7 @@ pub fn eq_item_kind(l: &ItemKind, r: &ItemKind) -> bool {
                 ident: li,
                 generics: lg,
                 ty: lt,
-                rhs: lb,
+                rhs_kind: lb,
                 define_opaque: _,
             }),
             Const(box ConstItem {
@@ -363,7 +363,7 @@ pub fn eq_item_kind(l: &ItemKind, r: &ItemKind) -> bool {
                 ident: ri,
                 generics: rg,
                 ty: rt,
-                rhs: rb,
+                rhs_kind: rb,
                 define_opaque: _,
             }),
         ) => {
@@ -371,7 +371,7 @@ pub fn eq_item_kind(l: &ItemKind, r: &ItemKind) -> bool {
                 && eq_id(*li, *ri)
                 && eq_generics(lg, rg)
                 && eq_ty(lt, rt)
-                && both(lb.as_ref(), rb.as_ref(), eq_const_item_rhs)
+                && both(Some(lb), Some(rb), eq_const_item_rhs)
         },
         (
             Fn(box ast::Fn {
@@ -615,7 +615,7 @@ pub fn eq_assoc_item_kind(l: &AssocItemKind, r: &AssocItemKind) -> bool {
                 ident: li,
                 generics: lg,
                 ty: lt,
-                rhs: lb,
+                rhs_kind: lb,
                 define_opaque: _,
             }),
             Const(box ConstItem {
@@ -623,7 +623,7 @@ pub fn eq_assoc_item_kind(l: &AssocItemKind, r: &AssocItemKind) -> bool {
                 ident: ri,
                 generics: rg,
                 ty: rt,
-                rhs: rb,
+                rhs_kind: rb,
                 define_opaque: _,
             }),
         ) => {
@@ -631,7 +631,7 @@ pub fn eq_assoc_item_kind(l: &AssocItemKind, r: &AssocItemKind) -> bool {
                 && eq_id(*li, *ri)
                 && eq_generics(lg, rg)
                 && eq_ty(lt, rt)
-                && both(lb.as_ref(), rb.as_ref(), eq_const_item_rhs)
+                && both(Some(lb), Some(rb), eq_const_item_rhs)
         },
         (
             Fn(box ast::Fn {
@@ -791,12 +791,18 @@ pub fn eq_anon_const(l: &AnonConst, r: &AnonConst) -> bool {
     eq_expr(&l.value, &r.value)
 }
 
-pub fn eq_const_item_rhs(l: &ConstItemRhs, r: &ConstItemRhs) -> bool {
-    use ConstItemRhs::*;
+pub fn eq_const_item_rhs(l: &ConstItemRhsKind, r: &ConstItemRhsKind) -> bool {
+    use ConstItemRhsKind::*;
     match (l, r) {
-        (TypeConst(l), TypeConst(r)) => eq_anon_const(l, r),
-        (Body(l), Body(r)) => eq_expr(l, r),
-        (TypeConst(..), Body(..)) | (Body(..), TypeConst(..)) => false,
+        (TypeConst { rhs: Some(l) }, TypeConst { rhs: Some(r) }) => eq_anon_const(l, r),
+        (TypeConst { rhs: None }, TypeConst { rhs: None }) => true,
+        (TypeConst { rhs: Some(..) }, TypeConst { rhs: None }) => false,
+        (TypeConst { rhs: None }, TypeConst { rhs: Some(..) }) => false,
+        (Body { rhs: Some(l) }, Body { rhs: Some(r) }) => eq_expr(l, r),
+        (Body { rhs: None }, Body { rhs: None }) => true,
+        (Body { rhs: None }, Body { rhs: Some(..) }) => false,
+        (Body { rhs: Some(..) }, Body { rhs: None }) => false,
+        (TypeConst {..}, Body { .. }) | ( Body { .. }, TypeConst { .. }) => false,
     }
 }
 
diff --git a/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed b/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed
index f8be3331317c..8d63fc44e7f8 100644
--- a/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed
+++ b/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed
@@ -3,8 +3,8 @@
 #![feature(min_generic_const_args)]
 
 trait AssocConstTrait {
-    #[type_const]
-    const ASSOC: usize;
+
+    type const ASSOC: usize;
 }
 fn assoc_const_args()
 where
diff --git a/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs b/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs
index a0d7a653993f..36a83619c0f9 100644
--- a/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs
+++ b/src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs
@@ -3,8 +3,8 @@
 #![feature(min_generic_const_args)]
 
 trait AssocConstTrait {
-    #[type_const]
-    const ASSOC: usize;
+
+    type const ASSOC: usize;
 }
 fn assoc_const_args()
 where
diff --git a/src/tools/rustfmt/src/items.rs b/src/tools/rustfmt/src/items.rs
index c9e32492c50b..1aa85ce2ce55 100644
--- a/src/tools/rustfmt/src/items.rs
+++ b/src/tools/rustfmt/src/items.rs
@@ -2028,12 +2028,16 @@ impl<'a> StaticParts<'a> {
                 ),
                 ast::ItemKind::Const(c) => (
                     Some(c.defaultness),
-                    "const",
+                    if c.rhs_kind.is_type_const() {
+                        "type const"
+                    } else {
+                        "const"
+                    },
                     ast::Safety::Default,
                     c.ident,
                     &c.ty,
                     ast::Mutability::Not,
-                    c.rhs.as_ref().map(|rhs| rhs.expr()),
+                    c.rhs_kind.expr(),
                     Some(&c.generics),
                 ),
                 _ => unreachable!(),
@@ -2053,17 +2057,25 @@ impl<'a> StaticParts<'a> {
     }
 
     pub(crate) fn from_trait_item(ti: &'a ast::AssocItem, ident: Ident) -> Self {
-        let (defaultness, ty, expr_opt, generics) = match &ti.kind {
-            ast::AssocItemKind::Const(c) => (
-                c.defaultness,
-                &c.ty,
-                c.rhs.as_ref().map(|rhs| rhs.expr()),
-                Some(&c.generics),
-            ),
+        let (defaultness, ty, expr_opt, generics, prefix) = match &ti.kind {
+            ast::AssocItemKind::Const(c) => {
+                let prefix = if c.rhs_kind.is_type_const() {
+                    "type const"
+                } else {
+                    "const"
+                };
+                (
+                    c.defaultness,
+                    &c.ty,
+                    c.rhs_kind.expr(),
+                    Some(&c.generics),
+                    prefix,
+                )
+            }
             _ => unreachable!(),
         };
         StaticParts {
-            prefix: "const",
+            prefix,
             safety: ast::Safety::Default,
             vis: &ti.vis,
             ident,
@@ -2077,17 +2089,25 @@ impl<'a> StaticParts<'a> {
     }
 
     pub(crate) fn from_impl_item(ii: &'a ast::AssocItem, ident: Ident) -> Self {
-        let (defaultness, ty, expr_opt, generics) = match &ii.kind {
-            ast::AssocItemKind::Const(c) => (
-                c.defaultness,
-                &c.ty,
-                c.rhs.as_ref().map(|rhs| rhs.expr()),
-                Some(&c.generics),
-            ),
+        let (defaultness, ty, expr_opt, generics, prefix) = match &ii.kind {
+            ast::AssocItemKind::Const(c) => {
+                let prefix = if c.rhs_kind.is_type_const() {
+                    "type const"
+                } else {
+                    "const"
+                };
+                (
+                    c.defaultness,
+                    &c.ty,
+                    c.rhs_kind.expr(),
+                    Some(&c.generics),
+                    prefix,
+                )
+            }
             _ => unreachable!(),
         };
         StaticParts {
-            prefix: "const",
+            prefix,
             safety: ast::Safety::Default,
             vis: &ii.vis,
             ident,
diff --git a/tests/crashes/149809.rs b/tests/crashes/149809.rs
index 2b948e9079c3..f70498f11c87 100644
--- a/tests/crashes/149809.rs
+++ b/tests/crashes/149809.rs
@@ -5,8 +5,7 @@ struct Qux<'a> {
     x: &'a (),
 }
 impl<'a> Qux<'a> {
-    #[type_const]
-    const LEN: usize = 4;
+    type const LEN: usize = 4;
     fn foo(_: [u8; Qux::LEN]) {}
 }
 
diff --git a/tests/debuginfo/associated-const-bindings.rs b/tests/debuginfo/associated-const-bindings.rs
index b60fd66235af..88c17cee8025 100644
--- a/tests/debuginfo/associated-const-bindings.rs
+++ b/tests/debuginfo/associated-const-bindings.rs
@@ -14,12 +14,10 @@
 #![expect(unused_variables, incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 impl Trait for () {
-    #[type_const]
-    const N: usize = 101;
+    type const N: usize = 101;
 }
 
 fn main() {
diff --git a/tests/rustdoc-html/constant/ice-associated-const-equality-105952.rs b/tests/rustdoc-html/constant/ice-associated-const-equality-105952.rs
index 310e56b917fc..bdb82b91ec27 100644
--- a/tests/rustdoc-html/constant/ice-associated-const-equality-105952.rs
+++ b/tests/rustdoc-html/constant/ice-associated-const-equality-105952.rs
@@ -10,8 +10,7 @@ pub enum ParseMode {
     Raw,
 }
 pub trait Parse {
-    #[type_const]
-    const PARSE_MODE: ParseMode;
+    type const PARSE_MODE: ParseMode;
 }
 pub trait RenderRaw {}
 
diff --git a/tests/rustdoc-html/inline_cross/auxiliary/assoc-const-equality.rs b/tests/rustdoc-html/inline_cross/auxiliary/assoc-const-equality.rs
index 598d2b5bf29d..939740d8f918 100644
--- a/tests/rustdoc-html/inline_cross/auxiliary/assoc-const-equality.rs
+++ b/tests/rustdoc-html/inline_cross/auxiliary/assoc-const-equality.rs
@@ -4,6 +4,5 @@
 pub fn accept(_: impl Trait) {}
 
 pub trait Trait {
-    #[type_const]
-    const K: i32;
+    type const K: i32;
 }
diff --git a/tests/rustdoc-ui/associated-constant-not-allowed-102467.rs b/tests/rustdoc-ui/associated-constant-not-allowed-102467.rs
index 5b8a90d64121..4168a5653dd5 100644
--- a/tests/rustdoc-ui/associated-constant-not-allowed-102467.rs
+++ b/tests/rustdoc-ui/associated-constant-not-allowed-102467.rs
@@ -11,8 +11,7 @@ trait T {
 }
 
 trait S {
-    #[type_const]
-    const C: i32;
+    type const C: i32;
 }
 
 fn main() {}
diff --git a/tests/ui/associated-consts/issue-110933.rs b/tests/ui/associated-consts/issue-110933.rs
index 9a013ee71274..0115fb7d6e65 100644
--- a/tests/ui/associated-consts/issue-110933.rs
+++ b/tests/ui/associated-consts/issue-110933.rs
@@ -4,8 +4,7 @@
 #![allow(incomplete_features)]
 
 pub trait Trait {
-    #[type_const]
-    const ASSOC: usize;
+    type const ASSOC: usize;
 }
 
 pub fn foo<
diff --git a/tests/ui/associated-consts/type-const-in-array-len-wrong-type.rs b/tests/ui/associated-consts/type-const-in-array-len-wrong-type.rs
index 1a37f5957f25..cb0b8a102eb4 100644
--- a/tests/ui/associated-consts/type-const-in-array-len-wrong-type.rs
+++ b/tests/ui/associated-consts/type-const-in-array-len-wrong-type.rs
@@ -8,8 +8,7 @@
 struct OnDiskDirEntry<'a>(&'a ());
 
 impl<'a> OnDiskDirEntry<'a> {
-    #[type_const]
-    const LFN_FRAGMENT_LEN: i64 = 2;
+    type const LFN_FRAGMENT_LEN: i64 = 2;
 
     fn lfn_contents() -> [char; Self::LFN_FRAGMENT_LEN] {
         //~^ ERROR the constant `2` is not of type `usize`
diff --git a/tests/ui/associated-consts/type-const-in-array-len-wrong-type.stderr b/tests/ui/associated-consts/type-const-in-array-len-wrong-type.stderr
index 607b7789ae67..b69a1b7fd7de 100644
--- a/tests/ui/associated-consts/type-const-in-array-len-wrong-type.stderr
+++ b/tests/ui/associated-consts/type-const-in-array-len-wrong-type.stderr
@@ -24,7 +24,7 @@ LL | #![feature(inherent_associated_types)]
    = note: see issue #8995  for more information
 
 error: the constant `2` is not of type `usize`
-  --> $DIR/type-const-in-array-len-wrong-type.rs:14:26
+  --> $DIR/type-const-in-array-len-wrong-type.rs:13:26
    |
 LL |     fn lfn_contents() -> [char; Self::LFN_FRAGMENT_LEN] {
    |                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `usize`, found `i64`
diff --git a/tests/ui/associated-consts/type-const-in-array-len.rs b/tests/ui/associated-consts/type-const-in-array-len.rs
index 69dd3f443b87..d33eacaade2d 100644
--- a/tests/ui/associated-consts/type-const-in-array-len.rs
+++ b/tests/ui/associated-consts/type-const-in-array-len.rs
@@ -8,8 +8,7 @@
 // Test case from #138226: generic impl with multiple type parameters
 struct Foo(A, B);
 impl Foo {
-    #[type_const]
-    const LEN: usize = 4;
+    type const LEN: usize = 4;
 
     fn foo() {
         let _ = [5; Self::LEN];
@@ -19,8 +18,7 @@ impl Foo {
 // Test case from #138226: generic impl with const parameter
 struct Bar;
 impl Bar {
-    #[type_const]
-    const LEN: usize = 4;
+    type const LEN: usize = 4;
 
     fn bar() {
         let _ = [0; Self::LEN];
@@ -30,8 +28,7 @@ impl Bar {
 // Test case from #150960: non-generic impl with const block
 struct Baz;
 impl Baz {
-    #[type_const]
-    const LEN: usize = 4;
+    type const LEN: usize = 4;
 
     fn baz() {
         let _ = [0; { Self::LEN }];
diff --git a/tests/ui/associated-type-bounds/duplicate-bound-err.rs b/tests/ui/associated-type-bounds/duplicate-bound-err.rs
index 07eb86108bff..56403fdf6630 100644
--- a/tests/ui/associated-type-bounds/duplicate-bound-err.rs
+++ b/tests/ui/associated-type-bounds/duplicate-bound-err.rs
@@ -50,8 +50,7 @@ fn mismatch_2() -> impl Iterator {
 trait Trait {
     type Gat;
 
-    #[type_const]
-    const ASSOC: i32;
+    type const ASSOC: i32;
 
     fn foo() -> impl Sized;
 }
@@ -59,8 +58,7 @@ trait Trait {
 impl Trait for () {
     type Gat = ();
 
-    #[type_const]
-    const ASSOC: i32 = 3;
+    type const ASSOC: i32 = 3;
 
     fn foo() {}
 }
@@ -68,8 +66,7 @@ impl Trait for () {
 impl Trait for u32 {
     type Gat = ();
 
-    #[type_const]
-    const ASSOC: i32 = 4;
+    type const ASSOC: i32 = 4;
 
     fn foo() -> u32 {
         42
@@ -89,8 +86,7 @@ type MustFail = dyn Iterator;
 //~| ERROR conflicting associated type bindings
 
 trait Trait2 {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 type MustFail2 = dyn Trait2;
diff --git a/tests/ui/associated-type-bounds/duplicate-bound-err.stderr b/tests/ui/associated-type-bounds/duplicate-bound-err.stderr
index 695bb8ad6066..e6bf93970d0a 100644
--- a/tests/ui/associated-type-bounds/duplicate-bound-err.stderr
+++ b/tests/ui/associated-type-bounds/duplicate-bound-err.stderr
@@ -100,7 +100,7 @@ LL |     iter::empty::()
    |     ----------------------- return type was inferred to be `std::iter::Empty` here
 
 error[E0271]: expected `IntoIter` to be an iterator that yields `i32`, but it yields `u32`
-  --> $DIR/duplicate-bound-err.rs:111:17
+  --> $DIR/duplicate-bound-err.rs:107:17
    |
 LL |     fn foo() -> impl Iterator {
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i32`, found `u32`
@@ -109,19 +109,19 @@ LL |         [2u32].into_iter()
    |         ------------------ return type was inferred to be `std::array::IntoIter` here
 
 error[E0271]: expected `impl Iterator` to be an iterator that yields `i32`, but it yields `u32`
-  --> $DIR/duplicate-bound-err.rs:111:17
+  --> $DIR/duplicate-bound-err.rs:107:17
    |
 LL |     fn foo() -> impl Iterator {
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i32`, found `u32`
    |
 note: required by a bound in `Trait3::foo::{anon_assoc#0}`
-  --> $DIR/duplicate-bound-err.rs:107:31
+  --> $DIR/duplicate-bound-err.rs:103:31
    |
 LL |     fn foo() -> impl Iterator;
    |                               ^^^^^^^^^^ required by this bound in `Trait3::foo::{anon_assoc#0}`
 
 error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
-  --> $DIR/duplicate-bound-err.rs:87:42
+  --> $DIR/duplicate-bound-err.rs:84:42
    |
 LL | type MustFail = dyn Iterator;
    |                              ----------  ^^^^^^^^^^ re-bound here
@@ -129,7 +129,7 @@ LL | type MustFail = dyn Iterator;
    |                              `Item` bound here first
 
 error: conflicting associated type bindings for `Item`
-  --> $DIR/duplicate-bound-err.rs:87:17
+  --> $DIR/duplicate-bound-err.rs:84:17
    |
 LL | type MustFail = dyn Iterator;
    |                 ^^^^^^^^^^^^^----------^^----------^
@@ -138,7 +138,7 @@ LL | type MustFail = dyn Iterator;
    |                              `Item` is specified to be `i32` here
 
 error[E0719]: the value of the associated type `ASSOC` in trait `Trait2` is already specified
-  --> $DIR/duplicate-bound-err.rs:96:43
+  --> $DIR/duplicate-bound-err.rs:92:43
    |
 LL | type MustFail2 = dyn Trait2;
    |                             ------------  ^^^^^^^^^^^^ re-bound here
@@ -146,7 +146,7 @@ LL | type MustFail2 = dyn Trait2;
    |                             `ASSOC` bound here first
 
 error: conflicting associated constant bindings for `ASSOC`
-  --> $DIR/duplicate-bound-err.rs:96:18
+  --> $DIR/duplicate-bound-err.rs:92:18
    |
 LL | type MustFail2 = dyn Trait2;
    |                  ^^^^^^^^^^^------------^^------------^
@@ -155,7 +155,7 @@ LL | type MustFail2 = dyn Trait2;
    |                             `ASSOC` is specified to be `3` here
 
 error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
-  --> $DIR/duplicate-bound-err.rs:100:43
+  --> $DIR/duplicate-bound-err.rs:96:43
    |
 LL | type MustFail3 = dyn Iterator;
    |                               ----------  ^^^^^^^^^^ re-bound here
@@ -163,7 +163,7 @@ LL | type MustFail3 = dyn Iterator;
    |                               `Item` bound here first
 
 error[E0719]: the value of the associated type `ASSOC` in trait `Trait2` is already specified
-  --> $DIR/duplicate-bound-err.rs:103:43
+  --> $DIR/duplicate-bound-err.rs:99:43
    |
 LL | type MustFail4 = dyn Trait2;
    |                             ------------  ^^^^^^^^^^^^ re-bound here
@@ -171,7 +171,7 @@ LL | type MustFail4 = dyn Trait2;
    |                             `ASSOC` bound here first
 
 error[E0271]: expected `Empty` to be an iterator that yields `i32`, but it yields `u32`
-  --> $DIR/duplicate-bound-err.rs:119:16
+  --> $DIR/duplicate-bound-err.rs:115:16
    |
 LL |     uncallable(iter::empty::());
    |     ---------- ^^^^^^^^^^^^^^^^^^^^ expected `i32`, found `u32`
@@ -179,13 +179,13 @@ LL |     uncallable(iter::empty::());
    |     required by a bound introduced by this call
    |
 note: required by a bound in `uncallable`
-  --> $DIR/duplicate-bound-err.rs:79:32
+  --> $DIR/duplicate-bound-err.rs:76:32
    |
 LL | fn uncallable(_: impl Iterator) {}
    |                                ^^^^^^^^^^ required by this bound in `uncallable`
 
 error[E0271]: expected `Empty` to be an iterator that yields `u32`, but it yields `i32`
-  --> $DIR/duplicate-bound-err.rs:120:16
+  --> $DIR/duplicate-bound-err.rs:116:16
    |
 LL |     uncallable(iter::empty::());
    |     ---------- ^^^^^^^^^^^^^^^^^^^^ expected `u32`, found `i32`
@@ -193,13 +193,13 @@ LL |     uncallable(iter::empty::());
    |     required by a bound introduced by this call
    |
 note: required by a bound in `uncallable`
-  --> $DIR/duplicate-bound-err.rs:79:44
+  --> $DIR/duplicate-bound-err.rs:76:44
    |
 LL | fn uncallable(_: impl Iterator) {}
    |                                            ^^^^^^^^^^ required by this bound in `uncallable`
 
 error[E0271]: type mismatch resolving `<() as Trait>::ASSOC == 4`
-  --> $DIR/duplicate-bound-err.rs:121:22
+  --> $DIR/duplicate-bound-err.rs:117:22
    |
 LL |     uncallable_const(());
    |     ---------------- ^^ expected `4`, found `3`
@@ -209,13 +209,13 @@ LL |     uncallable_const(());
    = note: expected constant `4`
               found constant `3`
 note: required by a bound in `uncallable_const`
-  --> $DIR/duplicate-bound-err.rs:81:46
+  --> $DIR/duplicate-bound-err.rs:78:46
    |
 LL | fn uncallable_const(_: impl Trait) {}
    |                                              ^^^^^^^^^ required by this bound in `uncallable_const`
 
 error[E0271]: type mismatch resolving `::ASSOC == 3`
-  --> $DIR/duplicate-bound-err.rs:122:22
+  --> $DIR/duplicate-bound-err.rs:118:22
    |
 LL |     uncallable_const(4u32);
    |     ---------------- ^^^^ expected `3`, found `4`
@@ -225,13 +225,13 @@ LL |     uncallable_const(4u32);
    = note: expected constant `3`
               found constant `4`
 note: required by a bound in `uncallable_const`
-  --> $DIR/duplicate-bound-err.rs:81:35
+  --> $DIR/duplicate-bound-err.rs:78:35
    |
 LL | fn uncallable_const(_: impl Trait) {}
    |                                   ^^^^^^^^^ required by this bound in `uncallable_const`
 
 error[E0271]: type mismatch resolving `<() as Trait>::ASSOC == 4`
-  --> $DIR/duplicate-bound-err.rs:123:20
+  --> $DIR/duplicate-bound-err.rs:119:20
    |
 LL |     uncallable_rtn(());
    |     -------------- ^^ expected `4`, found `3`
@@ -241,7 +241,7 @@ LL |     uncallable_rtn(());
    = note: expected constant `4`
               found constant `3`
 note: required by a bound in `uncallable_rtn`
-  --> $DIR/duplicate-bound-err.rs:84:61
+  --> $DIR/duplicate-bound-err.rs:81:61
    |
 LL | fn uncallable_rtn(
    |    -------------- required by a bound in this function
@@ -249,7 +249,7 @@ LL |     _: impl Trait, foo(..): Trait>
    |                                                             ^^^^^^^^^ required by this bound in `uncallable_rtn`
 
 error[E0271]: type mismatch resolving `::ASSOC == 3`
-  --> $DIR/duplicate-bound-err.rs:124:20
+  --> $DIR/duplicate-bound-err.rs:120:20
    |
 LL |     uncallable_rtn(17u32);
    |     -------------- ^^^^^ expected `3`, found `4`
@@ -259,7 +259,7 @@ LL |     uncallable_rtn(17u32);
    = note: expected constant `3`
               found constant `4`
 note: required by a bound in `uncallable_rtn`
-  --> $DIR/duplicate-bound-err.rs:84:34
+  --> $DIR/duplicate-bound-err.rs:81:34
    |
 LL | fn uncallable_rtn(
    |    -------------- required by a bound in this function
diff --git a/tests/ui/associated-type-bounds/duplicate-bound.rs b/tests/ui/associated-type-bounds/duplicate-bound.rs
index 1aeb0022a04f..39cfa9db072c 100644
--- a/tests/ui/associated-type-bounds/duplicate-bound.rs
+++ b/tests/ui/associated-type-bounds/duplicate-bound.rs
@@ -189,8 +189,7 @@ trait Tra3 {
 trait Trait {
     type Gat;
 
-    #[type_const]
-    const ASSOC: i32;
+    type const ASSOC: i32;
 
     fn foo() -> impl Sized;
 }
@@ -198,8 +197,7 @@ trait Trait {
 impl Trait for () {
     type Gat = ();
 
-    #[type_const]
-    const ASSOC: i32 = 3;
+    type const ASSOC: i32 = 3;
 
     fn foo() {}
 }
diff --git a/tests/ui/associated-types/type-const-inherent-impl-normalize.rs b/tests/ui/associated-types/type-const-inherent-impl-normalize.rs
index 70eea70db4c6..a9c7373b4f44 100644
--- a/tests/ui/associated-types/type-const-inherent-impl-normalize.rs
+++ b/tests/ui/associated-types/type-const-inherent-impl-normalize.rs
@@ -1,8 +1,8 @@
 struct S;
 impl S {
-    #[type_const]
-    //~^ ERROR: the `#[type_const]` attribute is an experimental feature
-    const LEN: usize = 1;
+    type const LEN: usize = 1;
+    //~^ ERROR: associated `type const` are unstable [E0658]
+    //~| ERROR: `type const` syntax is experimental [E0658]
     fn arr() {
         [8; Self::LEN]
         //~^ WARN: cannot use constants which depend on generic parameters in types
diff --git a/tests/ui/associated-types/type-const-inherent-impl-normalize.stderr b/tests/ui/associated-types/type-const-inherent-impl-normalize.stderr
index 958d38e855ea..b86859f4a993 100644
--- a/tests/ui/associated-types/type-const-inherent-impl-normalize.stderr
+++ b/tests/ui/associated-types/type-const-inherent-impl-normalize.stderr
@@ -1,8 +1,18 @@
-error[E0658]: the `#[type_const]` attribute is an experimental feature
+error[E0658]: `type const` syntax is experimental
   --> $DIR/type-const-inherent-impl-normalize.rs:3:5
    |
-LL |     #[type_const]
-   |     ^^^^^^^^^^^^^
+LL |     type const LEN: usize = 1;
+   |     ^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: associated `type const` are unstable
+  --> $DIR/type-const-inherent-impl-normalize.rs:3:5
+   |
+LL |     type const LEN: usize = 1;
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: see issue #132980  for more information
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
@@ -36,7 +46,7 @@ LL |     fn arr() {
 LL |         [8; Self::LEN]
    |         ^^^^^^^^^^^^^^ expected `()`, found `[{integer}; 1]`
 
-error: aborting due to 2 previous errors; 2 warnings emitted
+error: aborting due to 3 previous errors; 2 warnings emitted
 
 Some errors have detailed explanations: E0308, E0658.
 For more information about an error, try `rustc --explain E0308`.
diff --git a/tests/ui/attributes/malformed-attrs.rs b/tests/ui/attributes/malformed-attrs.rs
index 6dc3086b63e1..489c8bf9f495 100644
--- a/tests/ui/attributes/malformed-attrs.rs
+++ b/tests/ui/attributes/malformed-attrs.rs
@@ -145,8 +145,6 @@ struct Test;
 #[diagnostic::on_unimplemented = 1]
 //~^ WARN malformed
 trait Hey {
-    #[type_const = 1]
-    //~^ ERROR malformed
     const HEY: usize = 5;
 }
 
diff --git a/tests/ui/attributes/malformed-attrs.stderr b/tests/ui/attributes/malformed-attrs.stderr
index 22e222efa435..009da1d12a16 100644
--- a/tests/ui/attributes/malformed-attrs.stderr
+++ b/tests/ui/attributes/malformed-attrs.stderr
@@ -21,13 +21,13 @@ LL | #[cfg_attr]
    = note: for more information, visit 
 
 error[E0463]: can't find crate for `wloop`
-  --> $DIR/malformed-attrs.rs:217:1
+  --> $DIR/malformed-attrs.rs:215:1
    |
 LL | extern crate wloop;
    | ^^^^^^^^^^^^^^^^^^^ can't find crate
 
 error: malformed `allow` attribute input
-  --> $DIR/malformed-attrs.rs:183:1
+  --> $DIR/malformed-attrs.rs:181:1
    |
 LL | #[allow]
    | ^^^^^^^^
@@ -43,7 +43,7 @@ LL | #[allow(lint1, lint2, lint3, reason = "...")]
    |        +++++++++++++++++++++++++++++++++++++
 
 error: malformed `expect` attribute input
-  --> $DIR/malformed-attrs.rs:185:1
+  --> $DIR/malformed-attrs.rs:183:1
    |
 LL | #[expect]
    | ^^^^^^^^^
@@ -59,7 +59,7 @@ LL | #[expect(lint1, lint2, lint3, reason = "...")]
    |         +++++++++++++++++++++++++++++++++++++
 
 error: malformed `warn` attribute input
-  --> $DIR/malformed-attrs.rs:187:1
+  --> $DIR/malformed-attrs.rs:185:1
    |
 LL | #[warn]
    | ^^^^^^^
@@ -75,7 +75,7 @@ LL | #[warn(lint1, lint2, lint3, reason = "...")]
    |       +++++++++++++++++++++++++++++++++++++
 
 error: malformed `deny` attribute input
-  --> $DIR/malformed-attrs.rs:189:1
+  --> $DIR/malformed-attrs.rs:187:1
    |
 LL | #[deny]
    | ^^^^^^^
@@ -91,7 +91,7 @@ LL | #[deny(lint1, lint2, lint3, reason = "...")]
    |       +++++++++++++++++++++++++++++++++++++
 
 error: malformed `forbid` attribute input
-  --> $DIR/malformed-attrs.rs:191:1
+  --> $DIR/malformed-attrs.rs:189:1
    |
 LL | #[forbid]
    | ^^^^^^^^^
@@ -125,7 +125,7 @@ LL | #[proc_macro_derive]
    | ^^^^^^^^^^^^^^^^^^^^
 
 error[E0658]: allow_internal_unsafe side-steps the unsafe_code lint
-  --> $DIR/malformed-attrs.rs:222:1
+  --> $DIR/malformed-attrs.rs:220:1
    |
 LL | #[allow_internal_unsafe = 1]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -541,7 +541,7 @@ LL | #[cfi_encoding = ""]
    | help: must be of the form: `#[cfi_encoding = "encoding"]`
 
 error[E0565]: malformed `marker` attribute input
-  --> $DIR/malformed-attrs.rs:160:1
+  --> $DIR/malformed-attrs.rs:158:1
    |
 LL | #[marker = 3]
    | ^^^^^^^^^---^
@@ -550,7 +550,7 @@ LL | #[marker = 3]
    | help: must be of the form: `#[marker]`
 
 error[E0565]: malformed `fundamental` attribute input
-  --> $DIR/malformed-attrs.rs:162:1
+  --> $DIR/malformed-attrs.rs:160:1
    |
 LL | #[fundamental()]
    | ^^^^^^^^^^^^^--^
@@ -559,7 +559,7 @@ LL | #[fundamental()]
    | help: must be of the form: `#[fundamental]`
 
 error[E0565]: malformed `ffi_pure` attribute input
-  --> $DIR/malformed-attrs.rs:170:5
+  --> $DIR/malformed-attrs.rs:168:5
    |
 LL |     #[unsafe(ffi_pure = 1)]
    |     ^^^^^^^^^^^^^^^^^^---^^
@@ -568,7 +568,7 @@ LL |     #[unsafe(ffi_pure = 1)]
    |     help: must be of the form: `#[ffi_pure]`
 
 error[E0539]: malformed `link_ordinal` attribute input
-  --> $DIR/malformed-attrs.rs:172:5
+  --> $DIR/malformed-attrs.rs:170:5
    |
 LL |     #[link_ordinal]
    |     ^^^^^^^^^^^^^^^
@@ -579,7 +579,7 @@ LL |     #[link_ordinal]
    = note: for more information, visit 
 
 error[E0565]: malformed `ffi_const` attribute input
-  --> $DIR/malformed-attrs.rs:176:5
+  --> $DIR/malformed-attrs.rs:174:5
    |
 LL |     #[unsafe(ffi_const = 1)]
    |     ^^^^^^^^^^^^^^^^^^^---^^
@@ -588,13 +588,13 @@ LL |     #[unsafe(ffi_const = 1)]
    |     help: must be of the form: `#[ffi_const]`
 
 error[E0539]: malformed `linkage` attribute input
-  --> $DIR/malformed-attrs.rs:178:5
+  --> $DIR/malformed-attrs.rs:176:5
    |
 LL |     #[linkage]
    |     ^^^^^^^^^^ expected this to be of the form `linkage = "..."`
 
 error[E0539]: malformed `debugger_visualizer` attribute input
-  --> $DIR/malformed-attrs.rs:193:1
+  --> $DIR/malformed-attrs.rs:191:1
    |
 LL | #[debugger_visualizer]
    | ^^^^^^^^^^^^^^^^^^^^^^
@@ -605,7 +605,7 @@ LL | #[debugger_visualizer]
    = note: for more information, visit 
 
 error[E0565]: malformed `automatically_derived` attribute input
-  --> $DIR/malformed-attrs.rs:195:1
+  --> $DIR/malformed-attrs.rs:193:1
    |
 LL | #[automatically_derived = 18]
    | ^^^^^^^^^^^^^^^^^^^^^^^^----^
@@ -614,7 +614,7 @@ LL | #[automatically_derived = 18]
    | help: must be of the form: `#[automatically_derived]`
 
 error[E0565]: malformed `non_exhaustive` attribute input
-  --> $DIR/malformed-attrs.rs:203:1
+  --> $DIR/malformed-attrs.rs:201:1
    |
 LL | #[non_exhaustive = 1]
    | ^^^^^^^^^^^^^^^^^---^
@@ -623,7 +623,7 @@ LL | #[non_exhaustive = 1]
    | help: must be of the form: `#[non_exhaustive]`
 
 error[E0565]: malformed `thread_local` attribute input
-  --> $DIR/malformed-attrs.rs:209:1
+  --> $DIR/malformed-attrs.rs:207:1
    |
 LL | #[thread_local()]
    | ^^^^^^^^^^^^^^--^
@@ -632,7 +632,7 @@ LL | #[thread_local()]
    | help: must be of the form: `#[thread_local]`
 
 error[E0565]: malformed `no_link` attribute input
-  --> $DIR/malformed-attrs.rs:213:1
+  --> $DIR/malformed-attrs.rs:211:1
    |
 LL | #[no_link()]
    | ^^^^^^^^^--^
@@ -641,7 +641,7 @@ LL | #[no_link()]
    | help: must be of the form: `#[no_link]`
 
 error[E0539]: malformed `macro_use` attribute input
-  --> $DIR/malformed-attrs.rs:215:1
+  --> $DIR/malformed-attrs.rs:213:1
    |
 LL | #[macro_use = 1]
    | ^^^^^^^^^^^^---^
@@ -659,7 +659,7 @@ LL + #[macro_use]
    |
 
 error[E0539]: malformed `macro_export` attribute input
-  --> $DIR/malformed-attrs.rs:220:1
+  --> $DIR/malformed-attrs.rs:218:1
    |
 LL | #[macro_export = 18]
    | ^^^^^^^^^^^^^^^----^
@@ -676,7 +676,7 @@ LL + #[macro_export]
    |
 
 error[E0565]: malformed `allow_internal_unsafe` attribute input
-  --> $DIR/malformed-attrs.rs:222:1
+  --> $DIR/malformed-attrs.rs:220:1
    |
 LL | #[allow_internal_unsafe = 1]
    | ^^^^^^^^^^^^^^^^^^^^^^^^---^
@@ -684,15 +684,6 @@ LL | #[allow_internal_unsafe = 1]
    | |                       didn't expect any arguments here
    | help: must be of the form: `#[allow_internal_unsafe]`
 
-error[E0565]: malformed `type_const` attribute input
-  --> $DIR/malformed-attrs.rs:148:5
-   |
-LL |     #[type_const = 1]
-   |     ^^^^^^^^^^^^^---^
-   |     |            |
-   |     |            didn't expect any arguments here
-   |     help: must be of the form: `#[type_const]`
-
 error: attribute should be applied to `const fn`
   --> $DIR/malformed-attrs.rs:32:1
    |
@@ -820,13 +811,13 @@ LL | #[no_implicit_prelude = 23]
    = help: `#[no_implicit_prelude]` can be applied to crates and modules
 
 warning: `#[diagnostic::do_not_recommend]` does not expect any arguments
-  --> $DIR/malformed-attrs.rs:154:1
+  --> $DIR/malformed-attrs.rs:152:1
    |
 LL | #[diagnostic::do_not_recommend()]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 warning: `#[automatically_derived]` attribute cannot be used on modules
-  --> $DIR/malformed-attrs.rs:195:1
+  --> $DIR/malformed-attrs.rs:193:1
    |
 LL | #[automatically_derived = 18]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -835,7 +826,7 @@ LL | #[automatically_derived = 18]
    = help: `#[automatically_derived]` can only be applied to trait impl blocks
 
 error: valid forms for the attribute are `#[ignore = "reason"]` and `#[ignore]`
-  --> $DIR/malformed-attrs.rs:229:1
+  --> $DIR/malformed-attrs.rs:227:1
    |
 LL | #[ignore = 1]
    | ^^^^^^^^^^^^^
@@ -854,7 +845,7 @@ LL |     #[coroutine = 63] || {}
    = note: expected unit type `()`
               found coroutine `{coroutine@$DIR/malformed-attrs.rs:116:23: 116:25}`
 
-error: aborting due to 76 previous errors; 8 warnings emitted
+error: aborting due to 75 previous errors; 8 warnings emitted
 
 Some errors have detailed explanations: E0308, E0463, E0539, E0565, E0658, E0805.
 For more information about an error, try `rustc --explain E0308`.
@@ -882,7 +873,7 @@ LL | #[ignore()]
 
 Future breakage diagnostic:
 error: valid forms for the attribute are `#[ignore = "reason"]` and `#[ignore]`
-  --> $DIR/malformed-attrs.rs:229:1
+  --> $DIR/malformed-attrs.rs:227:1
    |
 LL | #[ignore = 1]
    | ^^^^^^^^^^^^^
diff --git a/tests/ui/const-generics/associated-const-bindings/ambiguity.rs b/tests/ui/const-generics/associated-const-bindings/ambiguity.rs
index 785d43e51b3d..6871a028de5f 100644
--- a/tests/ui/const-generics/associated-const-bindings/ambiguity.rs
+++ b/tests/ui/const-generics/associated-const-bindings/ambiguity.rs
@@ -6,8 +6,7 @@
 
 trait Trait0: Parent0 + Parent0 {}
 trait Parent0 {
-    #[type_const]
-    const K: ();
+    type const K: ();
 }
 
 fn take0(_: impl Trait0) {}
@@ -15,12 +14,10 @@ fn take0(_: impl Trait0) {}
 
 trait Trait1: Parent1 + Parent2 {}
 trait Parent1 {
-    #[type_const]
-    const C: i32;
+    type const C: i32;
 }
 trait Parent2 {
-    #[type_const]
-    const C: &'static str;
+    type const C: &'static str;
 }
 
 fn take1(_: impl Trait1) {}
diff --git a/tests/ui/const-generics/associated-const-bindings/ambiguity.stderr b/tests/ui/const-generics/associated-const-bindings/ambiguity.stderr
index 806708f18d65..9dcb30d6b737 100644
--- a/tests/ui/const-generics/associated-const-bindings/ambiguity.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/ambiguity.stderr
@@ -1,8 +1,8 @@
 error[E0222]: ambiguous associated constant `K` in bounds of `Trait0`
-  --> $DIR/ambiguity.rs:13:25
+  --> $DIR/ambiguity.rs:12:25
    |
-LL |     const K: ();
-   |     -----------
+LL |     type const K: ();
+   |     ----------------
    |     |
    |     ambiguous `K` from `Parent0`
    |     ambiguous `K` from `Parent0`
@@ -17,13 +17,13 @@ LL | fn take0(_: impl Trait0) {}
                    T: Parent0::K = const { }
 
 error[E0222]: ambiguous associated constant `C` in bounds of `Trait1`
-  --> $DIR/ambiguity.rs:26:25
+  --> $DIR/ambiguity.rs:23:25
    |
-LL |     const C: i32;
-   |     ------------ ambiguous `C` from `Parent1`
+LL |     type const C: i32;
+   |     ----------------- ambiguous `C` from `Parent1`
 ...
-LL |     const C: &'static str;
-   |     --------------------- ambiguous `C` from `Parent2`
+LL |     type const C: &'static str;
+   |     -------------------------- ambiguous `C` from `Parent2`
 ...
 LL | fn take1(_: impl Trait1) {}
    |                         ^^^^^^^ ambiguous associated constant `C`
diff --git a/tests/ui/const-generics/associated-const-bindings/assoc-const.rs b/tests/ui/const-generics/associated-const-bindings/assoc-const.rs
index ac9f7e53b3cb..3f8353b6914d 100644
--- a/tests/ui/const-generics/associated-const-bindings/assoc-const.rs
+++ b/tests/ui/const-generics/associated-const-bindings/assoc-const.rs
@@ -3,15 +3,13 @@
 #![allow(unused, incomplete_features)]
 
 pub trait Foo {
-  #[type_const]
-  const N: usize;
+    type const N: usize;
 }
 
 pub struct Bar;
 
 impl Foo for Bar {
-  #[type_const]
-  const N: usize = 3;
+    type const N: usize = 3;
 }
 
 const TEST: usize = 3;
diff --git a/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.rs b/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.rs
index 803cc59cc93d..53f65319db9d 100644
--- a/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.rs
+++ b/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.rs
@@ -11,8 +11,7 @@
 use std::marker::ConstParamTy_;
 
 trait Trait {
-    #[type_const]
-    const K: T;
+    type const K: T;
 }
 
 fn take(
diff --git a/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.stderr b/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.stderr
index a4f97525b515..f2f69aad4ee6 100644
--- a/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty-not-wf.stderr
@@ -1,11 +1,11 @@
 error: higher-ranked subtype error
-  --> $DIR/bound-var-in-ty-not-wf.rs:21:13
+  --> $DIR/bound-var-in-ty-not-wf.rs:20:13
    |
 LL |         K = const { () }
    |             ^^^^^^^^^^^^
 
 error: higher-ranked subtype error
-  --> $DIR/bound-var-in-ty-not-wf.rs:21:13
+  --> $DIR/bound-var-in-ty-not-wf.rs:20:13
    |
 LL |         K = const { () }
    |             ^^^^^^^^^^^^
diff --git a/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty.rs b/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty.rs
index e6fd5bdad002..a509fe0d52e3 100644
--- a/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty.rs
+++ b/tests/ui/const-generics/associated-const-bindings/bound-var-in-ty.rs
@@ -14,8 +14,7 @@
 use std::marker::ConstParamTy_;
 
 trait Trait {
-    #[type_const]
-    const K: T;
+    type const K: T;
 }
 
 fn take(
diff --git a/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs b/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs
index 0af91bfe7da8..149755bad4ca 100644
--- a/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs
+++ b/tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs
@@ -12,13 +12,11 @@ trait Trait: SuperTrait {
     type N;
     type Q;
 
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 trait SuperTrait {
-    #[type_const]
-    const Q: &'static str;
+    type const Q: &'static str;
 }
 
 fn take0(_: impl Trait) {}
diff --git a/tests/ui/const-generics/associated-const-bindings/coherence.rs b/tests/ui/const-generics/associated-const-bindings/coherence.rs
index f4081fae6144..e9296f3a8df0 100644
--- a/tests/ui/const-generics/associated-const-bindings/coherence.rs
+++ b/tests/ui/const-generics/associated-const-bindings/coherence.rs
@@ -2,12 +2,10 @@
 #![expect(incomplete_features)]
 
 pub trait IsVoid {
-    #[type_const]
-    const IS_VOID: bool;
+    type const IS_VOID: bool;
 }
 impl IsVoid for () {
-    #[type_const]
-    const IS_VOID: bool = true;
+    type const IS_VOID: bool = true;
 }
 
 pub trait Maybe {}
diff --git a/tests/ui/const-generics/associated-const-bindings/coherence.stderr b/tests/ui/const-generics/associated-const-bindings/coherence.stderr
index 23d6495a16a4..df6781f2f9a2 100644
--- a/tests/ui/const-generics/associated-const-bindings/coherence.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/coherence.stderr
@@ -1,5 +1,5 @@
 error[E0119]: conflicting implementations of trait `Maybe` for type `()`
-  --> $DIR/coherence.rs:15:1
+  --> $DIR/coherence.rs:13:1
    |
 LL | impl Maybe for () {}
    | ----------------- first implementation here
diff --git a/tests/ui/const-generics/associated-const-bindings/const-projection-err.rs b/tests/ui/const-generics/associated-const-bindings/const-projection-err.rs
index d485316ce371..8e871ddf90ce 100644
--- a/tests/ui/const-generics/associated-const-bindings/const-projection-err.rs
+++ b/tests/ui/const-generics/associated-const-bindings/const-projection-err.rs
@@ -2,8 +2,7 @@
 #![allow(incomplete_features)]
 
 trait TraitWAssocConst {
-    #[type_const]
-    const A: usize;
+    type const A: usize;
 }
 
 fn foo>() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/const-projection-err.stderr b/tests/ui/const-generics/associated-const-bindings/const-projection-err.stderr
index 552b1579e618..c533a7d65b90 100644
--- a/tests/ui/const-generics/associated-const-bindings/const-projection-err.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/const-projection-err.stderr
@@ -1,5 +1,5 @@
 error[E0271]: type mismatch resolving `::A == 1`
-  --> $DIR/const-projection-err.rs:12:11
+  --> $DIR/const-projection-err.rs:11:11
    |
 LL |     foo::();
    |           ^ expected `1`, found `0`
@@ -7,7 +7,7 @@ LL |     foo::();
    = note: expected constant `1`
               found constant `0`
 note: required by a bound in `foo`
-  --> $DIR/const-projection-err.rs:9:28
+  --> $DIR/const-projection-err.rs:8:28
    |
 LL | fn foo>() {}
    |                            ^^^^^ required by this bound in `foo`
diff --git a/tests/ui/const-generics/associated-const-bindings/const_evaluatable_unchecked.rs b/tests/ui/const-generics/associated-const-bindings/const_evaluatable_unchecked.rs
index 161eef63d36f..578904ea1891 100644
--- a/tests/ui/const-generics/associated-const-bindings/const_evaluatable_unchecked.rs
+++ b/tests/ui/const-generics/associated-const-bindings/const_evaluatable_unchecked.rs
@@ -8,8 +8,7 @@
 #![allow(incomplete_features)]
 
 pub trait TraitA {
-    #[type_const]
-    const K: u8 = 0;
+    type const K: u8 = 0;
 }
 pub trait TraitB {}
 
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs
index bd26acce681f..771f48cba068 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.rs
@@ -14,14 +14,12 @@ trait Trait {
     // NOTE: The `ConstParamTy_` bound is intentionally on the assoc const and not on the trait as
     //       doing the latter would already render the trait dyn incompatible due to it being
     //       bounded by `PartialEq` and supertrait bounds cannot mention `Self` like this.
-    #[type_const]
-    const K: Self where Self: std::marker::ConstParamTy_;
+    type const K: Self where Self: std::marker::ConstParamTy_;
     //~^ NOTE it contains associated const `K` whose type references the `Self` type
 
     // This is not a "`Self` projection" in our sense (which would be allowed)
     // since the trait is not the principal trait or a supertrait thereof.
-    #[type_const]
-    const Q: ::Output;
+    type const Q: ::Output;
     //~^ NOTE it contains associated const `Q` whose type references the `Self` type
 }
 
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr
index dedbdd7f82bb..19cd8bf5af99 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-assoc-const-ty-mentions-self.stderr
@@ -1,21 +1,21 @@
 error[E0038]: the trait `Trait` is not dyn compatible
-  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:38:16
+  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:36:16
    |
 LL |     let _: dyn Trait;
    |                ^^^^^ `Trait` is not dyn compatible
    |
 note: for a trait to be dyn compatible it needs to allow building a vtable
       for more information, visit 
-  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:18:11
+  --> $DIR/dyn-compat-assoc-const-ty-mentions-self.rs:17:16
    |
 LL | trait Trait {
    |       ----- this trait is not dyn compatible...
 ...
-LL |     const K: Self where Self: std::marker::ConstParamTy_;
-   |           ^ ...because it contains associated const `K` whose type references the `Self` type
+LL |     type const K: Self where Self: std::marker::ConstParamTy_;
+   |                ^ ...because it contains associated const `K` whose type references the `Self` type
 ...
-LL |     const Q: ::Output;
-   |           ^ ...because it contains associated const `Q` whose type references the `Self` type
+LL |     type const Q: ::Output;
+   |                ^ ...because it contains associated const `Q` whose type references the `Self` type
    = help: consider moving `K` to another trait
    = help: consider moving `Q` to another trait
 
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-basic.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-basic.rs
index 6e35d56dc6e7..8de8cb1a60db 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-basic.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-basic.rs
@@ -7,25 +7,20 @@
 #![expect(incomplete_features)]
 
 trait Trait: SuperTrait {
-    #[type_const]
-    const K: usize;
+    type const K: usize;
 }
 
 trait SuperTrait {
-    #[type_const]
-    const Q: usize;
-    #[type_const]
-    const C: usize;
+    type const Q: usize;
+    type const C: usize;
 }
 
 trait Bound {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 impl Bound for () {
-    #[type_const]
-    const N: usize = 10;
+    type const N: usize = 10;
 }
 
 fn main() {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.rs
index be41906a4f2f..ad5c4b679117 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.rs
@@ -4,13 +4,11 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 impl Trait for () {
-    #[type_const]
-    const N: usize = 1;
+    type const N: usize = 1;
 }
 
 fn main() {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.stderr
index 2ab02068eb34..282504e3bc35 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-mismatch.stderr
@@ -1,5 +1,5 @@
 error[E0271]: type mismatch resolving `<() as Trait>::N == 0`
-  --> $DIR/dyn-compat-const-mismatch.rs:17:32
+  --> $DIR/dyn-compat-const-mismatch.rs:15:32
    |
 LL |     let _: &dyn Trait = &();
    |                                ^^^ expected `0`, found `1`
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.rs
index 30e7a78c2a85..8130960195d9 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.rs
@@ -7,13 +7,11 @@
 trait X::N }> {}
 
 trait Y {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 impl Y for T {
-    #[type_const]
-    const N: usize = 1;
+    type const N: usize = 1;
 }
 
 fn main() {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.stderr
index d92fd620f0ea..a22545fcd8d2 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-param-default-mentions-self.stderr
@@ -1,5 +1,5 @@
 error[E0393]: the const parameter `N` must be explicitly specified
-  --> $DIR/dyn-compat-const-param-default-mentions-self.rs:20:16
+  --> $DIR/dyn-compat-const-param-default-mentions-self.rs:18:16
    |
 LL | trait X::N }> {}
    | -------------------------------------------- const parameter `N` must be specified for this
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs
index 93036fbc01a8..415d7bda347b 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs
@@ -9,8 +9,7 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const Y: i32;
+    type const Y: i32;
 }
 
 struct Hold(T);
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr
index 109cb7602dcd..576d4c3230ee 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr
@@ -1,5 +1,5 @@
 error: associated constant binding in trait object type mentions `Self`
-  --> $DIR/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs:21:12
+  --> $DIR/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs:20:12
    |
 LL | trait Bound = Trait }>;
    |                     -------------------- this binding mentions `Self`
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.rs
index aa93edd02531..560a1b7f3f7a 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.rs
@@ -5,13 +5,11 @@
 #![expect(incomplete_features)]
 
 trait X: Y {
-    #[type_const]
-    const Q: usize;
+    type const Q: usize;
 }
 
 trait Y {
-    #[type_const]
-    const K: usize;
+    type const K: usize;
 }
 
 fn main() {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.stderr
index 373c4f0e6611..dae9e3d8c22b 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-from-supertrait-mentions-self.stderr
@@ -1,8 +1,8 @@
 error[E0191]: the value of the associated constant `K` in `Y` must be specified
-  --> $DIR/dyn-compat-const-projection-from-supertrait-mentions-self.rs:18:16
+  --> $DIR/dyn-compat-const-projection-from-supertrait-mentions-self.rs:16:16
    |
-LL |     const K: usize;
-   |     -------------- `K` defined here
+LL |     type const K: usize;
+   |     ------------------- `K` defined here
 ...
 LL |     let _: dyn X;
    |                ^^^^^^^^^ help: specify the associated constant: `X`
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.rs
index 67650bce8c89..38d593984724 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.rs
@@ -7,7 +7,7 @@
 
 trait Trait {
     const K: usize;
-    //~^ NOTE it contains associated const `K` that's not marked `#[type_const]`
+    //~^ NOTE it contains associated const `K` that's not defined as `type const`
 }
 
 fn main() {
@@ -16,5 +16,5 @@ fn main() {
     // Check that specifying the non-type assoc const doesn't "magically make it work".
     let _: dyn Trait;
     //~^ ERROR the trait `Trait` is not dyn compatible
-    //~| ERROR use of trait associated const without `#[type_const]`
+    //~| ERROR use of trait associated const not defined as `type const`
 }
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.stderr
index c579cd312f1f..5bc072e98c0f 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-non-type-assoc-const.stderr
@@ -11,16 +11,16 @@ note: for a trait to be dyn compatible it needs to allow building a vtable
 LL | trait Trait {
    |       ----- this trait is not dyn compatible...
 LL |     const K: usize;
-   |           ^ ...because it contains associated const `K` that's not marked `#[type_const]`
+   |           ^ ...because it contains associated const `K` that's not defined as `type const`
    = help: consider moving `K` to another trait
 
-error: use of trait associated const without `#[type_const]`
+error: use of trait associated const not defined as `type const`
   --> $DIR/dyn-compat-non-type-assoc-const.rs:17:22
    |
 LL |     let _: dyn Trait;
    |                      ^^^^^
    |
-   = note: the declaration in the trait must be marked with `#[type_const]`
+   = note: the declaration in the trait must begin with `type const` not just `const` alone
 
 error[E0038]: the trait `Trait` is not dyn compatible
   --> $DIR/dyn-compat-non-type-assoc-const.rs:17:16
@@ -35,7 +35,7 @@ note: for a trait to be dyn compatible it needs to allow building a vtable
 LL | trait Trait {
    |       ----- this trait is not dyn compatible...
 LL |     const K: usize;
-   |           ^ ...because it contains associated const `K` that's not marked `#[type_const]`
+   |           ^ ...because it contains associated const `K` that's not defined as `type const`
    = help: consider moving `K` to another trait
 
 error: aborting due to 3 previous errors
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs
index 07ce629ab7be..03f1a526b73a 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs
@@ -6,13 +6,11 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const N: i32 where Self: Bound;
+    type const N: i32 where Self: Bound;
 }
 
 impl Trait for () {
-    #[type_const]
-    const N: i32 = 0;
+    type const N: i32 = 0;
 }
 
 trait Bound {}
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.stderr
index 8eeb60d55c38..21e407f8a861 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.stderr
@@ -1,19 +1,19 @@
 error[E0277]: the trait bound `(): Bound` is not satisfied
-  --> $DIR/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs:23:32
+  --> $DIR/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs:21:32
    |
 LL |     let _: &dyn Trait = &();
    |                                ^^^ the trait `Bound` is not implemented for `()`
    |
 help: this trait has no implementations, consider adding one
-  --> $DIR/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs:18:1
+  --> $DIR/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs:16:1
    |
 LL | trait Bound {}
    | ^^^^^^^^^^^
 note: required by a bound in `Trait::N`
-  --> $DIR/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs:10:30
+  --> $DIR/dyn-compat-self-bound-on-assoc-const-allowed-and-enforced.rs:9:35
    |
-LL |     const N: i32 where Self: Bound;
-   |                              ^^^^^ required by this bound in `Trait::N`
+LL |     type const N: i32 where Self: Bound;
+   |                                   ^^^^^ required by this bound in `Trait::N`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs
index fc3bd1d2e753..623769f5d12f 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.rs
@@ -14,14 +14,12 @@
 
 trait A {
     type Ty: std::marker::ConstParamTy_;
-    #[type_const]
-    const CT: Self::Ty;
+    type const CT: Self::Ty;
 }
 
 impl A for () {
     type Ty = i32;
-    #[type_const]
-    const CT: i32 = 0;
+    type const CT: i32 = 0;
 }
 
 fn main() {
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr
index 0b8dae1aac43..cc08c25906b6 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-assoc-const-ty.stderr
@@ -1,23 +1,23 @@
 error[E0277]: the trait bound `FreshTy(0): A` is not satisfied
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:34:33
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:32:33
    |
 LL |     let _: dyn A;
    |                                 ^ the trait `A` is not implemented for `FreshTy(0)`
    |
 help: the trait `A` is implemented for `()`
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:21:1
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:20:1
    |
 LL | impl A for () {
    | ^^^^^^^^^^^^^
 
 error[E0277]: the trait bound `FreshTy(0): A` is not satisfied
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:36:34
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:34:34
    |
 LL |     let _: &dyn A = &();
    |                                  ^ the trait `A` is not implemented for `FreshTy(0)`
    |
 help: the trait `A` is implemented for `()`
-  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:21:1
+  --> $DIR/dyn-compat-self-const-projections-in-assoc-const-ty.rs:20:1
    |
 LL | impl A for () {
    | ^^^^^^^^^^^^^
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-methods.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-methods.rs
index f5a5fbc8188c..9886213743c8 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-methods.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-methods.rs
@@ -15,15 +15,13 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 
     fn process(&self, _: [u8; Self::N]) -> [u8; Self::N];
 }
 
 impl Trait for u8 {
-    #[type_const]
-    const N: usize = 2;
+    type const N: usize = 2;
 
     fn process(&self, [x, y]: [u8; Self::N]) -> [u8; Self::N] {
         [self * x, self + y]
@@ -31,8 +29,7 @@ impl Trait for u8 {
 }
 
 impl Trait for [u8; N] {
-    #[type_const]
-    const N: usize = N;
+    type const N: usize = N;
 
     fn process(&self, other: [u8; Self::N]) -> [u8; Self::N] {
         let mut result = [0; _];
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.rs
index c8929750f61c..f2ae98accda5 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.rs
@@ -9,8 +9,7 @@
 
 trait Trait: SuperTrait<{ Self::N }> {
 //~^ NOTE it uses `Self` as a type parameter
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 trait SuperTrait {}
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.stderr
index 33edc96117d5..38c928fd58d7 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-supertrait-bounds.stderr
@@ -1,5 +1,5 @@
 error[E0038]: the trait `Trait` is not dyn compatible
-  --> $DIR/dyn-compat-self-const-projections-in-supertrait-bounds.rs:19:16
+  --> $DIR/dyn-compat-self-const-projections-in-supertrait-bounds.rs:18:16
    |
 LL |     let _: dyn Trait;
    |                ^^^^^ `Trait` is not dyn compatible
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.rs
index 18be0fccc051..52ecd42b191d 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.rs
@@ -15,8 +15,7 @@
 #![crate_name = "sym"]
 
 trait Trait {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 #[rustc_symbol_name]
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.v0.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.v0.stderr
index 424251acc3e9..8ca0f73494d3 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.v0.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-symbol-mangling.v0.stderr
@@ -1,17 +1,17 @@
 error: symbol-name(_RMCsCRATE_HASH_3symDNtB_5Traitp1NKj0_EL_)
-  --> $DIR/dyn-compat-symbol-mangling.rs:22:1
+  --> $DIR/dyn-compat-symbol-mangling.rs:21:1
    |
 LL | #[rustc_symbol_name]
    | ^^^^^^^^^^^^^^^^^^^^
 
 error: demangling(>)
-  --> $DIR/dyn-compat-symbol-mangling.rs:22:1
+  --> $DIR/dyn-compat-symbol-mangling.rs:21:1
    |
 LL | #[rustc_symbol_name]
    | ^^^^^^^^^^^^^^^^^^^^
 
 error: demangling-alt(>)
-  --> $DIR/dyn-compat-symbol-mangling.rs:22:1
+  --> $DIR/dyn-compat-symbol-mangling.rs:21:1
    |
 LL | #[rustc_symbol_name]
    | ^^^^^^^^^^^^^^^^^^^^
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.rs
index 3525cbceb87b..5bd8aa609420 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.rs
@@ -6,8 +6,7 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const K: usize;
+    type const K: usize;
 }
 
 // fn ctxt / body
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.stderr
index 68658a5711cb..35fdd7f2751e 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-unspecified-assoc-consts.stderr
@@ -1,26 +1,26 @@
 error[E0191]: the value of the associated constant `K` in `Trait` must be specified
-  --> $DIR/dyn-compat-unspecified-assoc-consts.rs:20:18
+  --> $DIR/dyn-compat-unspecified-assoc-consts.rs:19:18
    |
-LL |     const K: usize;
-   |     -------------- `K` defined here
+LL |     type const K: usize;
+   |     ------------------- `K` defined here
 ...
 LL | struct Store(dyn Trait);
    |                  ^^^^^ help: specify the associated constant: `Trait`
 
 error[E0191]: the value of the associated constant `K` in `Trait` must be specified
-  --> $DIR/dyn-compat-unspecified-assoc-consts.rs:24:21
+  --> $DIR/dyn-compat-unspecified-assoc-consts.rs:23:21
    |
-LL |     const K: usize;
-   |     -------------- `K` defined here
+LL |     type const K: usize;
+   |     ------------------- `K` defined here
 ...
 LL | type DynTrait = dyn Trait;
    |                     ^^^^^ help: specify the associated constant: `Trait`
 
 error[E0191]: the value of the associated constant `K` in `Trait` must be specified
-  --> $DIR/dyn-compat-unspecified-assoc-consts.rs:15:16
+  --> $DIR/dyn-compat-unspecified-assoc-consts.rs:14:16
    |
-LL |     const K: usize;
-   |     -------------- `K` defined here
+LL |     type const K: usize;
+   |     ------------------- `K` defined here
 ...
 LL |     let _: dyn Trait;
    |                ^^^^^ help: specify the associated constant: `Trait`
diff --git a/tests/ui/const-generics/associated-const-bindings/equality-unused-issue-126729.rs b/tests/ui/const-generics/associated-const-bindings/equality-unused-issue-126729.rs
index 614ed8c803d4..f42589f91d15 100644
--- a/tests/ui/const-generics/associated-const-bindings/equality-unused-issue-126729.rs
+++ b/tests/ui/const-generics/associated-const-bindings/equality-unused-issue-126729.rs
@@ -5,42 +5,34 @@
 #![deny(dead_code)]
 
 trait Tr {
-    #[type_const]
-    const I: i32;
+    type const I: i32;
 }
 
 impl Tr for () {
-    #[type_const]
-    const I: i32 = 1;
+    type const I: i32 = 1;
 }
 
 fn foo() -> impl Tr {}
 
 trait Tr2 {
-    #[type_const]
-    const J: i32;
-    #[type_const]
-    const K: i32;
+    type const J: i32;
+    type const K: i32;
 }
 
 impl Tr2 for () {
-    #[type_const]
-    const J: i32 = 1;
-    #[type_const]
-    const K: i32 = 1;
+    type const J: i32 = 1;
+    type const K: i32 = 1;
 }
 
 fn foo2() -> impl Tr2 {}
 
 mod t {
     pub trait Tr3 {
-        #[type_const]
-        const L: i32;
+        type     const L: i32;
     }
 
     impl Tr3 for () {
-        #[type_const]
-        const L: i32 = 1;
+        type     const L: i32 = 1;
     }
 }
 
diff --git a/tests/ui/const-generics/associated-const-bindings/equality_bound_with_infer.rs b/tests/ui/const-generics/associated-const-bindings/equality_bound_with_infer.rs
index b7ed8ee39f87..e50c13b5d3b5 100644
--- a/tests/ui/const-generics/associated-const-bindings/equality_bound_with_infer.rs
+++ b/tests/ui/const-generics/associated-const-bindings/equality_bound_with_infer.rs
@@ -7,13 +7,11 @@
 // though it contained inference variables, which would cause ICEs.
 
 trait Foo {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 impl Foo for () {
-    #[type_const]
-    const ASSOC: u32 = N;
+    type const ASSOC: u32 = N;
 }
 
 fn bar = 10>>() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs
index 4d06cdc7620f..de888c5a7939 100644
--- a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs
+++ b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.rs
@@ -9,8 +9,7 @@
 #![allow(incomplete_features)]
 
 trait Trait<'a> {
-    #[type_const]
-    const K: &'a ();
+    type const K: &'a ();
 }
 
 fn take(_: impl for<'r> Trait<'r, K = const { &() }>) {}
diff --git a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr
index 3966483aa600..122893662933 100644
--- a/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/esc-bound-var-in-ty.stderr
@@ -1,5 +1,5 @@
 error: the type of the associated constant `K` cannot capture late-bound generic parameters
-  --> $DIR/esc-bound-var-in-ty.rs:16:35
+  --> $DIR/esc-bound-var-in-ty.rs:15:35
    |
 LL | fn take(_: impl for<'r> Trait<'r, K = const { &() }>) {}
    |                     --            ^ its type cannot capture the late-bound lifetime parameter `'r`
diff --git a/tests/ui/const-generics/associated-const-bindings/issue-102335-const.rs b/tests/ui/const-generics/associated-const-bindings/issue-102335-const.rs
index a88a3abf0a12..1663cad13c7c 100644
--- a/tests/ui/const-generics/associated-const-bindings/issue-102335-const.rs
+++ b/tests/ui/const-generics/associated-const-bindings/issue-102335-const.rs
@@ -8,8 +8,7 @@ trait T {
 }
 
 trait S {
-    #[type_const]
-    const C: i32;
+    type const C: i32;
 }
 
 fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.rs b/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.rs
index 77f0f06f8040..1c6e873b98c3 100644
--- a/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.rs
+++ b/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.rs
@@ -2,13 +2,11 @@
 #![expect(incomplete_features)]
 
 trait Foo {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 impl Foo for () {
-    #[type_const]
-    const ASSOC: u32 = N;
+    type const ASSOC: u32 = N;
 }
 
 fn bar = { N }>>() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.stderr b/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.stderr
index c2fb7faa3a3a..b447cd08a214 100644
--- a/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/mismatched-types-with-generic-in-ace.stderr
@@ -1,26 +1,26 @@
 error: the constant `N` is not of type `u32`
-  --> $DIR/mismatched-types-with-generic-in-ace.rs:14:29
+  --> $DIR/mismatched-types-with-generic-in-ace.rs:12:29
    |
 LL | fn bar = { N }>>() {}
    |                             ^^^^^^^^^^^^^^^^ expected `u32`, found `u64`
    |
 note: required by a const generic parameter in `Foo::ASSOC`
-  --> $DIR/mismatched-types-with-generic-in-ace.rs:6:17
+  --> $DIR/mismatched-types-with-generic-in-ace.rs:5:22
    |
-LL |     const ASSOC: u32;
-   |                 ^^^^^^^^^^^^ required by this const generic parameter in `Foo::ASSOC`
+LL |     type const ASSOC: u32;
+   |                      ^^^^^^^^^^^^ required by this const generic parameter in `Foo::ASSOC`
 
 error: the constant `10` is not of type `u32`
-  --> $DIR/mismatched-types-with-generic-in-ace.rs:18:5
+  --> $DIR/mismatched-types-with-generic-in-ace.rs:16:5
    |
 LL |     bar::<10_u64, ()>();
    |     ^^^^^^^^^^^^^^^^^^^ expected `u32`, found `u64`
    |
 note: required by a const generic parameter in `Foo::ASSOC`
-  --> $DIR/mismatched-types-with-generic-in-ace.rs:6:17
+  --> $DIR/mismatched-types-with-generic-in-ace.rs:5:22
    |
-LL |     const ASSOC: u32;
-   |                 ^^^^^^^^^^^^ required by this const generic parameter in `Foo::ASSOC`
+LL |     type const ASSOC: u32;
+   |                      ^^^^^^^^^^^^ required by this const generic parameter in `Foo::ASSOC`
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/associated-const-bindings/normalization-via-param-env.rs b/tests/ui/const-generics/associated-const-bindings/normalization-via-param-env.rs
index c7bb5bccedb3..a1efa74090bb 100644
--- a/tests/ui/const-generics/associated-const-bindings/normalization-via-param-env.rs
+++ b/tests/ui/const-generics/associated-const-bindings/normalization-via-param-env.rs
@@ -6,8 +6,7 @@
 // with associated const equality bounds.
 
 trait Trait {
-    #[type_const]
-    const C: usize;
+    type const C: usize;
 }
 
 fn f>() {
diff --git a/tests/ui/const-generics/associated-const-bindings/param-in-ty.rs b/tests/ui/const-generics/associated-const-bindings/param-in-ty.rs
index 44e7e3f19ef2..4d67186f71f9 100644
--- a/tests/ui/const-generics/associated-const-bindings/param-in-ty.rs
+++ b/tests/ui/const-generics/associated-const-bindings/param-in-ty.rs
@@ -11,8 +11,7 @@
 use std::marker::ConstParamTy_;
 
 trait Trait<'a, T: 'a + ConstParamTy_, const N: usize> {
-    #[type_const]
-    const K: &'a [T; N];
+    type const K: &'a [T; N];
 }
 
 fn take0<'r, A: 'r + ConstParamTy_, const Q: usize>(
@@ -32,8 +31,7 @@ fn take0<'r, A: 'r + ConstParamTy_, const Q: usize>(
 ) {}
 
 trait Project: ConstParamTy_ {
-    #[type_const]
-    const SELF: Self;
+    type const SELF: Self;
 }
 
 fn take1(_: impl Project) {}
diff --git a/tests/ui/const-generics/associated-const-bindings/param-in-ty.stderr b/tests/ui/const-generics/associated-const-bindings/param-in-ty.stderr
index 719dad816a6f..2ef3fab7e5ff 100644
--- a/tests/ui/const-generics/associated-const-bindings/param-in-ty.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/param-in-ty.stderr
@@ -1,5 +1,5 @@
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:22:29
+  --> $DIR/param-in-ty.rs:21:29
    |
 LL | fn take0<'r, A: 'r + ConstParamTy_, const Q: usize>(
    |          -- the lifetime parameter `'r` is defined here
@@ -10,7 +10,7 @@ LL |     _: impl Trait<'r, A, Q, K = const { loop {} }>
    = note: `K` has type `&'r [A; Q]`
 
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:22:29
+  --> $DIR/param-in-ty.rs:21:29
    |
 LL | fn take0<'r, A: 'r + ConstParamTy_, const Q: usize>(
    |              - the type parameter `A` is defined here
@@ -21,7 +21,7 @@ LL |     _: impl Trait<'r, A, Q, K = const { loop {} }>
    = note: `K` has type `&'r [A; Q]`
 
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:22:29
+  --> $DIR/param-in-ty.rs:21:29
    |
 LL | fn take0<'r, A: 'r + ConstParamTy_, const Q: usize>(
    |                                           - the const parameter `Q` is defined here
@@ -32,7 +32,7 @@ LL |     _: impl Trait<'r, A, Q, K = const { loop {} }>
    = note: `K` has type `&'r [A; Q]`
 
 error: the type of the associated constant `SELF` must not depend on `impl Trait`
-  --> $DIR/param-in-ty.rs:39:26
+  --> $DIR/param-in-ty.rs:37:26
    |
 LL | fn take1(_: impl Project) {}
    |             -------------^^^^------------
@@ -41,7 +41,7 @@ LL | fn take1(_: impl Project) {}
    |             the `impl Trait` is specified here
 
 error: the type of the associated constant `SELF` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:44:21
+  --> $DIR/param-in-ty.rs:42:21
    |
 LL | fn take2>(_: P) {}
    |          -          ^^^^ its type must not depend on the type parameter `P`
@@ -51,7 +51,7 @@ LL | fn take2>(_: P) {}
    = note: `SELF` has type `P`
 
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:53:52
+  --> $DIR/param-in-ty.rs:51:52
    |
 LL | trait Iface<'r>: ConstParamTy_ {
    |             -- the lifetime parameter `'r` is defined here
@@ -62,7 +62,7 @@ LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    = note: `K` has type `&'r [Self; Q]`
 
 error: the type of the associated constant `K` must not depend on `Self`
-  --> $DIR/param-in-ty.rs:53:52
+  --> $DIR/param-in-ty.rs:51:52
    |
 LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    |                                                    ^ its type must not depend on `Self`
@@ -70,7 +70,7 @@ LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    = note: `K` has type `&'r [Self; Q]`
 
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:53:52
+  --> $DIR/param-in-ty.rs:51:52
    |
 LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    |                      -                             ^ its type must not depend on the const parameter `Q`
@@ -80,7 +80,7 @@ LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    = note: `K` has type `&'r [Self; Q]`
 
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:53:52
+  --> $DIR/param-in-ty.rs:51:52
    |
 LL | trait Iface<'r>: ConstParamTy_ {
    |             -- the lifetime parameter `'r` is defined here
@@ -92,7 +92,7 @@ LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
 error: the type of the associated constant `K` must not depend on `Self`
-  --> $DIR/param-in-ty.rs:53:52
+  --> $DIR/param-in-ty.rs:51:52
    |
 LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    |                                                    ^ its type must not depend on `Self`
@@ -101,7 +101,7 @@ LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
 error: the type of the associated constant `K` must not depend on generic parameters
-  --> $DIR/param-in-ty.rs:53:52
+  --> $DIR/param-in-ty.rs:51:52
    |
 LL |     type Assoc: Trait<'r, Self, Q, K = const { loop {} }>
    |                      -                             ^ its type must not depend on the const parameter `Q`
diff --git a/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.rs b/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.rs
index 8a78b26dbc5d..dbfbba9b7cbd 100644
--- a/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.rs
+++ b/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.rs
@@ -4,8 +4,7 @@
 // Issue 110549
 
 pub trait TraitWAssocConst {
-    #[type_const]
-    const A: usize;
+    type const A: usize;
 }
 
 fn foo>() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.stderr b/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.stderr
index 232b15b7e981..11490a044091 100644
--- a/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/projection-unspecified-but-bounded.stderr
@@ -1,5 +1,5 @@
 error[E0271]: type mismatch resolving `::A == 32`
-  --> $DIR/projection-unspecified-but-bounded.rs:14:11
+  --> $DIR/projection-unspecified-but-bounded.rs:13:11
    |
 LL |     foo::();
    |           ^ expected `32`, found `::A`
@@ -7,7 +7,7 @@ LL |     foo::();
    = note: expected constant `32`
               found constant `::A`
 note: required by a bound in `foo`
-  --> $DIR/projection-unspecified-but-bounded.rs:11:28
+  --> $DIR/projection-unspecified-but-bounded.rs:10:28
    |
 LL | fn foo>() {}
    |                            ^^^^^^ required by this bound in `foo`
diff --git a/tests/ui/const-generics/associated-const-bindings/supertraits.rs b/tests/ui/const-generics/associated-const-bindings/supertraits.rs
index a5f8859c92bd..cdceb682a854 100644
--- a/tests/ui/const-generics/associated-const-bindings/supertraits.rs
+++ b/tests/ui/const-generics/associated-const-bindings/supertraits.rs
@@ -16,8 +16,7 @@ use std::marker::ConstParamTy_;
 trait Trait: SuperTrait {}
 trait SuperTrait: SuperSuperTrait {}
 trait SuperSuperTrait {
-    #[type_const]
-    const K: T;
+    type const K: T;
 }
 
 fn take(_: impl Trait) {}
diff --git a/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.rs b/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.rs
index 5c494031d4fa..95f81323acf4 100644
--- a/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.rs
+++ b/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.rs
@@ -4,8 +4,7 @@
 #![feature(min_generic_const_args)]
 
 trait Trait {
-    #[type_const]
-    const F: fn();
+    type const F: fn();
     //~^ ERROR using function pointers as const generic parameters is forbidden
 }
 
diff --git a/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.stderr b/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.stderr
index 09d1063081fe..333dd1b89e9b 100644
--- a/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/using-fnptr-as-type_const.stderr
@@ -1,8 +1,8 @@
 error[E0741]: using function pointers as const generic parameters is forbidden
-  --> $DIR/using-fnptr-as-type_const.rs:8:14
+  --> $DIR/using-fnptr-as-type_const.rs:7:19
    |
-LL |     const F: fn();
-   |              ^^^^
+LL |     type const F: fn();
+   |                   ^^^^
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/generic_const_exprs/auxiliary/non_local_type_const.rs b/tests/ui/const-generics/generic_const_exprs/auxiliary/non_local_type_const.rs
index 6ec1071461c5..8cd46783d4fd 100644
--- a/tests/ui/const-generics/generic_const_exprs/auxiliary/non_local_type_const.rs
+++ b/tests/ui/const-generics/generic_const_exprs/auxiliary/non_local_type_const.rs
@@ -1,5 +1,4 @@
 #![feature(min_generic_const_args)]
 #![allow(incomplete_features)]
 
-#[type_const]
-pub const NON_LOCAL_CONST: char = 'a';
+pub type const NON_LOCAL_CONST: char = 'a';
diff --git a/tests/ui/const-generics/mgca/adt_expr_arg_simple.rs b/tests/ui/const-generics/mgca/adt_expr_arg_simple.rs
index 16479ba3f64f..8470b933cadd 100644
--- a/tests/ui/const-generics/mgca/adt_expr_arg_simple.rs
+++ b/tests/ui/const-generics/mgca/adt_expr_arg_simple.rs
@@ -10,9 +10,9 @@ use Option::Some;
 
 fn foo>() {}
 
+
 trait Trait {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 fn bar() {
diff --git a/tests/ui/const-generics/mgca/adt_expr_arg_tuple_expr_fail.rs b/tests/ui/const-generics/mgca/adt_expr_arg_tuple_expr_fail.rs
index 5f0d6c924bd1..68f999fb76fa 100644
--- a/tests/ui/const-generics/mgca/adt_expr_arg_tuple_expr_fail.rs
+++ b/tests/ui/const-generics/mgca/adt_expr_arg_tuple_expr_fail.rs
@@ -1,9 +1,9 @@
 #![feature(min_generic_const_args, adt_const_params, unsized_const_params)]
 #![expect(incomplete_features)]
 
+
 trait Trait {
-    #[type_const]
-    const ASSOC: usize;
+    type const ASSOC: usize;
 }
 
 fn takes_tuple() {}
diff --git a/tests/ui/const-generics/mgca/adt_expr_infers_from_value.rs b/tests/ui/const-generics/mgca/adt_expr_infers_from_value.rs
index d3c9b655a9e5..5def3cbb829b 100644
--- a/tests/ui/const-generics/mgca/adt_expr_infers_from_value.rs
+++ b/tests/ui/const-generics/mgca/adt_expr_infers_from_value.rs
@@ -16,8 +16,7 @@ struct Foo {
     field: T,
 }
 
-#[type_const]
-const WRAP: Foo = { Foo:: {
+type const WRAP: Foo = { Foo:: {
     field: N,
 } };
 
diff --git a/tests/ui/const-generics/mgca/array-expr-with-assoc-const.rs b/tests/ui/const-generics/mgca/array-expr-with-assoc-const.rs
index 47ecbfa6b7e1..7401181962bb 100644
--- a/tests/ui/const-generics/mgca/array-expr-with-assoc-const.rs
+++ b/tests/ui/const-generics/mgca/array-expr-with-assoc-const.rs
@@ -6,8 +6,7 @@
 fn takes_array() {}
 
 trait Trait {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 fn generic_caller() {
diff --git a/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs b/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs
index 460e14359483..13b2e031b118 100644
--- a/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs
+++ b/tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs
@@ -7,13 +7,11 @@
 trait Abc {}
 
 trait A {
-    #[type_const]
-    const VALUE: usize;
+    type const VALUE: usize;
 }
 
 impl A for T {
-    #[type_const]
-    const VALUE: usize = 0;
+    type const VALUE: usize = 0;
 }
 
 trait S {}
diff --git a/tests/ui/const-generics/mgca/assoc-const-without-type_const.rs b/tests/ui/const-generics/mgca/assoc-const-without-type_const.rs
index a11314c11aae..ca7299e7690b 100644
--- a/tests/ui/const-generics/mgca/assoc-const-without-type_const.rs
+++ b/tests/ui/const-generics/mgca/assoc-const-without-type_const.rs
@@ -6,9 +6,9 @@ pub trait Tr {
 }
 
 fn mk_array(_x: T) -> [(); T::SIZE] {
-    //~^ ERROR type_const
+    //~^ ERROR: use of `const` in the type system not defined as `type const`
     [(); T::SIZE]
-    //~^ ERROR type_const
+    //~^ ERROR: use of `const` in the type system not defined as `type const`
 }
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/assoc-const-without-type_const.stderr b/tests/ui/const-generics/mgca/assoc-const-without-type_const.stderr
index e06850747ff3..759a40cc0675 100644
--- a/tests/ui/const-generics/mgca/assoc-const-without-type_const.stderr
+++ b/tests/ui/const-generics/mgca/assoc-const-without-type_const.stderr
@@ -1,26 +1,20 @@
-error: use of `const` in the type system without `#[type_const]`
+error: use of `const` in the type system not defined as `type const`
   --> $DIR/assoc-const-without-type_const.rs:8:35
    |
+LL |     const SIZE: usize;
+   |     - help: add `type` before `const` for `Tr::SIZE`: `type`
+...
 LL | fn mk_array(_x: T) -> [(); T::SIZE] {
    |                                   ^^^^^^^
-   |
-help: add `#[type_const]` attribute to `Tr::SIZE`
-   |
-LL +     #[type_const]
-LL |     const SIZE: usize;
-   |
 
-error: use of `const` in the type system without `#[type_const]`
+error: use of `const` in the type system not defined as `type const`
   --> $DIR/assoc-const-without-type_const.rs:10:10
    |
+LL |     const SIZE: usize;
+   |     - help: add `type` before `const` for `Tr::SIZE`: `type`
+...
 LL |     [(); T::SIZE]
    |          ^^^^^^^
-   |
-help: add `#[type_const]` attribute to `Tr::SIZE`
-   |
-LL +     #[type_const]
-LL |     const SIZE: usize;
-   |
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/mgca/assoc-const.rs b/tests/ui/const-generics/mgca/assoc-const.rs
index fb5b4308a7f8..c49b84edba10 100644
--- a/tests/ui/const-generics/mgca/assoc-const.rs
+++ b/tests/ui/const-generics/mgca/assoc-const.rs
@@ -4,8 +4,7 @@
 #![allow(incomplete_features)]
 
 pub trait Tr {
-    #[type_const]
-    const SIZE: usize;
+    type const SIZE: usize;
 }
 
 fn mk_array>(_x: T) -> [(); >::SIZE] {
diff --git a/tests/ui/const-generics/mgca/bad-type_const-syntax.rs b/tests/ui/const-generics/mgca/bad-type_const-syntax.rs
index bb5bdb8d7c4c..81be1ca4eb88 100644
--- a/tests/ui/const-generics/mgca/bad-type_const-syntax.rs
+++ b/tests/ui/const-generics/mgca/bad-type_const-syntax.rs
@@ -1,16 +1,16 @@
 trait Tr {
-    #[type_const()]
-    //~^ ERROR malformed
-    //~| ERROR experimental
-    const N: usize;
+    type const N: usize;
+    //~^ ERROR: `type const` syntax is experimental [E0658]
+    //~| ERROR: associated `type const` are unstable [E0658]
 }
 
 struct S;
 
 impl Tr for S {
-    #[type_const]
-    //~^ ERROR experimental
-    const N: usize = 0;
+
+    type const N: usize = 0;
+    //~^ ERROR: `type const` syntax is experimental [E0658]
+    //~| ERROR: associated `type const` are unstable [E0658]
 }
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/bad-type_const-syntax.stderr b/tests/ui/const-generics/mgca/bad-type_const-syntax.stderr
index df442c22241b..7bb2adf27199 100644
--- a/tests/ui/const-generics/mgca/bad-type_const-syntax.stderr
+++ b/tests/ui/const-generics/mgca/bad-type_const-syntax.stderr
@@ -1,33 +1,43 @@
-error[E0658]: the `#[type_const]` attribute is an experimental feature
+error[E0658]: `type const` syntax is experimental
   --> $DIR/bad-type_const-syntax.rs:2:5
    |
-LL |     #[type_const()]
-   |     ^^^^^^^^^^^^^^^
+LL |     type const N: usize;
+   |     ^^^^^^^^^^
    |
    = note: see issue #132980  for more information
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error[E0658]: the `#[type_const]` attribute is an experimental feature
+error[E0658]: `type const` syntax is experimental
   --> $DIR/bad-type_const-syntax.rs:11:5
    |
-LL |     #[type_const]
-   |     ^^^^^^^^^^^^^
+LL |     type const N: usize = 0;
+   |     ^^^^^^^^^^
    |
    = note: see issue #132980  for more information
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error[E0565]: malformed `type_const` attribute input
+error[E0658]: associated `type const` are unstable
   --> $DIR/bad-type_const-syntax.rs:2:5
    |
-LL |     #[type_const()]
-   |     ^^^^^^^^^^^^--^
-   |     |           |
-   |     |           didn't expect any arguments here
-   |     help: must be of the form: `#[type_const]`
+LL |     type const N: usize;
+   |     ^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error: aborting due to 3 previous errors
+error[E0658]: associated `type const` are unstable
+  --> $DIR/bad-type_const-syntax.rs:11:5
+   |
+LL |     type const N: usize = 0;
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-Some errors have detailed explanations: E0565, E0658.
-For more information about an error, try `rustc --explain E0565`.
+error: aborting due to 4 previous errors
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/const-generics/mgca/concrete-expr-with-generics-in-env.rs b/tests/ui/const-generics/mgca/concrete-expr-with-generics-in-env.rs
index 37f9c31feaa5..69d16993a7e0 100644
--- a/tests/ui/const-generics/mgca/concrete-expr-with-generics-in-env.rs
+++ b/tests/ui/const-generics/mgca/concrete-expr-with-generics-in-env.rs
@@ -4,23 +4,17 @@
 #![feature(min_generic_const_args, generic_const_items)]
 
 pub trait Tr {
-    #[type_const]
-    const N1: usize;
-    #[type_const]
-    const N2: usize;
-    #[type_const]
-    const N3: usize;
+    type const N1: usize;
+    type const N2: usize;
+    type const N3: usize;
 }
 
 pub struct S;
 
 impl Tr for S {
-    #[type_const]
-    const N1: usize = 0;
-    #[type_const]
-    const N2: usize = 1;
-    #[type_const]
-    const N3: usize = 2;
+    type const N1: usize = 0;
+    type const N2: usize = 1;
+    type const N3: usize = 2;
 }
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/const-arg-coherence-conflicting-methods.rs b/tests/ui/const-generics/mgca/const-arg-coherence-conflicting-methods.rs
index efab01489404..9d475f8224fa 100644
--- a/tests/ui/const-generics/mgca/const-arg-coherence-conflicting-methods.rs
+++ b/tests/ui/const-generics/mgca/const-arg-coherence-conflicting-methods.rs
@@ -3,8 +3,8 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args)]
 
-#[type_const]
-const C: usize = 0;
+
+type const C: usize = 0;
 pub struct A {}
 impl A {
     fn fun1() {}
diff --git a/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs b/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs
index 823a6d58bf47..a7e46ad877e7 100644
--- a/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs
+++ b/tests/ui/const-generics/mgca/cyclic-type-const-151251.rs
@@ -4,8 +4,7 @@
 #![feature(generic_const_exprs)]
 #![expect(incomplete_features)]
 
-#[type_const]
-const A: u8 = A;
+type const A: u8 = A;
 //~^ ERROR overflow normalizing the unevaluated constant `A`
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr b/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr
index 1ce2af817277..47653dd1896f 100644
--- a/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr
+++ b/tests/ui/const-generics/mgca/cyclic-type-const-151251.stderr
@@ -1,8 +1,8 @@
 error[E0275]: overflow normalizing the unevaluated constant `A`
-  --> $DIR/cyclic-type-const-151251.rs:8:1
+  --> $DIR/cyclic-type-const-151251.rs:7:1
    |
-LL | const A: u8 = A;
-   | ^^^^^^^^^^^
+LL | type const A: u8 = A;
+   | ^^^^^^^^^^^^^^^^
    |
    = note: in case this is a recursive type alias, consider using a struct, enum, or union instead
 
diff --git a/tests/ui/const-generics/mgca/explicit_anon_consts.rs b/tests/ui/const-generics/mgca/explicit_anon_consts.rs
index bf825a44b1c0..2b9909b43dfb 100644
--- a/tests/ui/const-generics/mgca/explicit_anon_consts.rs
+++ b/tests/ui/const-generics/mgca/explicit_anon_consts.rs
@@ -34,22 +34,22 @@ fn repeats() -> [(); N] {
     //~^ ERROR: generic parameters may not be used in const operations
 }
 
-#[type_const]
-const ITEM1: usize = N;
-#[type_const]
-const ITEM2: usize = { N };
-#[type_const]
-const ITEM3: usize = const { N };
+
+type const ITEM1: usize = N;
+
+type const ITEM2: usize = { N };
+
+type const ITEM3: usize = const { N };
 //~^ ERROR: generic parameters may not be used in const operations
-#[type_const]
-const ITEM4: usize = { 1 + 1 };
+
+type const ITEM4: usize = { 1 + 1 };
 //~^ ERROR: complex const arguments must be placed inside of a `const` block
-#[type_const]
-const ITEM5: usize = const { 1 + 1};
+
+type const ITEM5: usize = const { 1 + 1};
 
 trait Trait {
-    #[type_const]
-    const ASSOC: usize;
+
+    type const ASSOC: usize;
 }
 
 fn ace_bounds<
diff --git a/tests/ui/const-generics/mgca/explicit_anon_consts.stderr b/tests/ui/const-generics/mgca/explicit_anon_consts.stderr
index 1251f4415171..714d7a804d11 100644
--- a/tests/ui/const-generics/mgca/explicit_anon_consts.stderr
+++ b/tests/ui/const-generics/mgca/explicit_anon_consts.stderr
@@ -17,10 +17,10 @@ LL |     let _4 = [(); 1 + 1];
    |                   ^^^^^
 
 error: complex const arguments must be placed inside of a `const` block
-  --> $DIR/explicit_anon_consts.rs:45:38
+  --> $DIR/explicit_anon_consts.rs:45:43
    |
-LL | const ITEM4: usize = { 1 + 1 };
-   |                                      ^^^^^^^^^
+LL | type const ITEM4: usize = { 1 + 1 };
+   |                                           ^^^^^^^^^
 
 error: complex const arguments must be placed inside of a `const` block
   --> $DIR/explicit_anon_consts.rs:62:23
@@ -35,10 +35,10 @@ LL | struct Default4;
    |                                                  ^^^^^^^^^
 
 error: generic parameters may not be used in const operations
-  --> $DIR/explicit_anon_consts.rs:42:46
+  --> $DIR/explicit_anon_consts.rs:42:51
    |
-LL | const ITEM3: usize = const { N };
-   |                                              ^
+LL | type const ITEM3: usize = const { N };
+   |                                                   ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
diff --git a/tests/ui/const-generics/mgca/explicit_anon_consts_literals_hack.rs b/tests/ui/const-generics/mgca/explicit_anon_consts_literals_hack.rs
index 8131a5b72343..be853bb87a37 100644
--- a/tests/ui/const-generics/mgca/explicit_anon_consts_literals_hack.rs
+++ b/tests/ui/const-generics/mgca/explicit_anon_consts_literals_hack.rs
@@ -4,8 +4,7 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const ASSOC: isize;
+    type const ASSOC: isize;
 }
 
 fn ace>() {}
diff --git a/tests/ui/const-generics/mgca/multi_braced_direct_const_args.rs b/tests/ui/const-generics/mgca/multi_braced_direct_const_args.rs
index 31f54abf31ef..bbe624269672 100644
--- a/tests/ui/const-generics/mgca/multi_braced_direct_const_args.rs
+++ b/tests/ui/const-generics/mgca/multi_braced_direct_const_args.rs
@@ -6,8 +6,7 @@
 struct Foo;
 
 trait Trait {
-    #[type_const]
-    const ASSOC: usize;
+    type const ASSOC: usize;
 }
 
 type Arr = [(); {{{ N }}}];
diff --git a/tests/ui/const-generics/mgca/non-local-const-without-type_const.rs b/tests/ui/const-generics/mgca/non-local-const-without-type_const.rs
index e98cbdf8e617..6f1235269dfc 100644
--- a/tests/ui/const-generics/mgca/non-local-const-without-type_const.rs
+++ b/tests/ui/const-generics/mgca/non-local-const-without-type_const.rs
@@ -5,5 +5,5 @@
 extern crate non_local_const;
 fn main() {
     let x = [(); non_local_const::N];
-    //~^ ERROR use of `const` in the type system without `#[type_const]`
+    //~^ ERROR: use of `const` in the type system not defined as `type const`
 }
diff --git a/tests/ui/const-generics/mgca/non-local-const-without-type_const.stderr b/tests/ui/const-generics/mgca/non-local-const-without-type_const.stderr
index 671dfa5fbb9e..3c10b78eb3e1 100644
--- a/tests/ui/const-generics/mgca/non-local-const-without-type_const.stderr
+++ b/tests/ui/const-generics/mgca/non-local-const-without-type_const.stderr
@@ -1,10 +1,10 @@
-error: use of `const` in the type system without `#[type_const]`
+error: use of `const` in the type system not defined as `type const`
   --> $DIR/non-local-const-without-type_const.rs:7:18
    |
 LL |     let x = [(); non_local_const::N];
    |                  ^^^^^^^^^^^^^^^^^^
    |
-   = note: only consts marked with `#[type_const]` may be used in types
+   = note: only consts marked defined as `type const` may be used in types
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/printing_valtrees_supports_non_values.rs b/tests/ui/const-generics/mgca/printing_valtrees_supports_non_values.rs
index 819323d9cbec..16ed24a7b287 100644
--- a/tests/ui/const-generics/mgca/printing_valtrees_supports_non_values.rs
+++ b/tests/ui/const-generics/mgca/printing_valtrees_supports_non_values.rs
@@ -8,8 +8,8 @@
 struct Foo;
 
 trait Trait {
-    #[type_const]
-    const ASSOC: u32;
+
+    type const ASSOC: u32;
 }
 
 fn foo() {}
diff --git a/tests/ui/const-generics/mgca/projection-error.rs b/tests/ui/const-generics/mgca/projection-error.rs
index edb6db098084..d3bd520297e1 100644
--- a/tests/ui/const-generics/mgca/projection-error.rs
+++ b/tests/ui/const-generics/mgca/projection-error.rs
@@ -5,10 +5,8 @@
 // containing erroneous types normalizes to a const error instead of
 // a type error.
 
-
 pub trait Tr {
-    #[type_const]
-    const SIZE: usize;
+    type const SIZE: usize;
 }
 
 fn mk_array(_x: T) -> [(); >::SIZE] {}
diff --git a/tests/ui/const-generics/mgca/projection-error.stderr b/tests/ui/const-generics/mgca/projection-error.stderr
index 9357600a5e11..94e6bbcdd87f 100644
--- a/tests/ui/const-generics/mgca/projection-error.stderr
+++ b/tests/ui/const-generics/mgca/projection-error.stderr
@@ -1,5 +1,5 @@
 error[E0425]: cannot find type `T` in this scope
-  --> $DIR/projection-error.rs:14:17
+  --> $DIR/projection-error.rs:12:17
    |
 LL | pub trait Tr {
    | --------------- similarly named trait `Tr` defined here
@@ -17,7 +17,7 @@ LL | fn mk_array(_x: T) -> [(); >::SIZE] {}
    |            +++
 
 error[E0425]: cannot find type `T` in this scope
-  --> $DIR/projection-error.rs:14:29
+  --> $DIR/projection-error.rs:12:29
    |
 LL | pub trait Tr {
    | --------------- similarly named trait `Tr` defined here
diff --git a/tests/ui/const-generics/mgca/tuple_ctor_arg_simple.rs b/tests/ui/const-generics/mgca/tuple_ctor_arg_simple.rs
index a5b3d3d0d5b6..e946441453d8 100644
--- a/tests/ui/const-generics/mgca/tuple_ctor_arg_simple.rs
+++ b/tests/ui/const-generics/mgca/tuple_ctor_arg_simple.rs
@@ -15,8 +15,7 @@ enum MyEnum {
 }
 
 trait Trait {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 fn with_point() -> Point {
diff --git a/tests/ui/const-generics/mgca/tuple_ctor_erroneous.rs b/tests/ui/const-generics/mgca/tuple_ctor_erroneous.rs
index ebbe097c226e..95ee42d93d73 100644
--- a/tests/ui/const-generics/mgca/tuple_ctor_erroneous.rs
+++ b/tests/ui/const-generics/mgca/tuple_ctor_erroneous.rs
@@ -12,8 +12,8 @@ enum MyEnum {
     Unit,
 }
 
-#[type_const]
-const CONST_ITEM: u32 = 42;
+
+type const CONST_ITEM: u32 = 42;
 
 fn accepts_point() {}
 fn accepts_enum>() {}
diff --git a/tests/ui/const-generics/mgca/tuple_expr_arg_complex.rs b/tests/ui/const-generics/mgca/tuple_expr_arg_complex.rs
index e47052523fa3..5a40c1b14c2d 100644
--- a/tests/ui/const-generics/mgca/tuple_expr_arg_complex.rs
+++ b/tests/ui/const-generics/mgca/tuple_expr_arg_complex.rs
@@ -2,8 +2,8 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const ASSOC: usize;
+
+    type const ASSOC: usize;
 }
 
 fn takes_tuple() {}
diff --git a/tests/ui/const-generics/mgca/tuple_expr_arg_simple.rs b/tests/ui/const-generics/mgca/tuple_expr_arg_simple.rs
index 3fde431e27e2..4c040bcaa945 100644
--- a/tests/ui/const-generics/mgca/tuple_expr_arg_simple.rs
+++ b/tests/ui/const-generics/mgca/tuple_expr_arg_simple.rs
@@ -4,8 +4,7 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const ASSOC: u32;
+    type const ASSOC: u32;
 }
 
 fn takes_tuple() {}
diff --git a/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs
index 158a7addd10d..e98b1ad4fe37 100644
--- a/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs
+++ b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.rs
@@ -4,15 +4,13 @@
 #![expect(incomplete_features)]
 
 trait Tr {
-    #[type_const]
-    const SIZE: usize;
+    type const SIZE: usize;
 }
 
 struct T;
 
 impl Tr for T {
-    #[type_const]
-    const SIZE: usize;
+    type const SIZE: usize;
     //~^ ERROR associated constant in `impl` without body
 }
 
diff --git a/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr
index 2db677aa0ca6..ba01456ee040 100644
--- a/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr
+++ b/tests/ui/const-generics/mgca/type-const-assoc-const-without-body.stderr
@@ -1,10 +1,10 @@
 error: associated constant in `impl` without body
-  --> $DIR/type-const-assoc-const-without-body.rs:15:5
+  --> $DIR/type-const-assoc-const-without-body.rs:13:5
    |
-LL |     const SIZE: usize;
-   |     ^^^^^^^^^^^^^^^^^-
-   |                      |
-   |                      help: provide a definition for the constant: `= ;`
+LL |     type const SIZE: usize;
+   |     ^^^^^^^^^^^^^^^^^^^^^^-
+   |                           |
+   |                           help: provide a definition for the constant: `= ;`
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs
index 85b2327d3351..9fa417637295 100644
--- a/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs
+++ b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.rs
@@ -4,8 +4,7 @@
 #![expect(incomplete_features)]
 
 impl S { //~ ERROR cannot find type `S` in this scope
-    #[type_const]
-    const SIZE: usize;
+    type const SIZE: usize;
     //~^ ERROR associated constant in `impl` without body
 }
 
diff --git a/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr
index ac520a4e6946..b1e1edfa70d6 100644
--- a/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr
+++ b/tests/ui/const-generics/mgca/type-const-inherent-assoc-const-without-body.stderr
@@ -1,10 +1,10 @@
 error: associated constant in `impl` without body
-  --> $DIR/type-const-inherent-assoc-const-without-body.rs:8:5
+  --> $DIR/type-const-inherent-assoc-const-without-body.rs:7:5
    |
-LL |     const SIZE: usize;
-   |     ^^^^^^^^^^^^^^^^^-
-   |                      |
-   |                      help: provide a definition for the constant: `= ;`
+LL |     type const SIZE: usize;
+   |     ^^^^^^^^^^^^^^^^^^^^^^-
+   |                           |
+   |                           help: provide a definition for the constant: `= ;`
 
 error[E0425]: cannot find type `S` in this scope
   --> $DIR/type-const-inherent-assoc-const-without-body.rs:6:6
diff --git a/tests/ui/const-generics/mgca/type-const-used-in-trait.rs b/tests/ui/const-generics/mgca/type-const-used-in-trait.rs
index c98c14775a0f..1efc65bd7018 100644
--- a/tests/ui/const-generics/mgca/type-const-used-in-trait.rs
+++ b/tests/ui/const-generics/mgca/type-const-used-in-trait.rs
@@ -3,8 +3,7 @@
 #![feature(min_generic_const_args)]
 #![expect(incomplete_features)]
 
-#[type_const]
-const N: usize = 2;
+type const N: usize = 2;
 
 trait CollectArray {
     fn inner_array(&mut self) -> [A; N];
diff --git a/tests/ui/const-generics/mgca/type_const-array-return.rs b/tests/ui/const-generics/mgca/type_const-array-return.rs
index 5375e4fded6d..43db35966a45 100644
--- a/tests/ui/const-generics/mgca/type_const-array-return.rs
+++ b/tests/ui/const-generics/mgca/type_const-array-return.rs
@@ -6,14 +6,12 @@
 pub struct A;
 
 pub trait Array {
-    #[type_const]
-    const LEN: usize;
+    type const LEN: usize;
     fn arr() -> [u8; Self::LEN];
 }
 
 impl Array for A {
-    #[type_const]
-    const LEN: usize = 4;
+    type const LEN: usize = 4;
 
     #[allow(unused_braces)]
     fn arr() -> [u8; const { Self::LEN }] {
diff --git a/tests/ui/const-generics/mgca/type_const-generic-param-in-type.gate.stderr b/tests/ui/const-generics/mgca/type_const-generic-param-in-type.gate.stderr
index 8a64af285da5..095f42355c77 100644
--- a/tests/ui/const-generics/mgca/type_const-generic-param-in-type.gate.stderr
+++ b/tests/ui/const-generics/mgca/type_const-generic-param-in-type.gate.stderr
@@ -1,38 +1,38 @@
 error: anonymous constants referencing generics are not yet supported
-  --> $DIR/type_const-generic-param-in-type.rs:9:53
+  --> $DIR/type_const-generic-param-in-type.rs:8:58
    |
-LL | const FOO: [T; 0] = const { [] };
-   |                                                     ^^^^^^^^^^^^
+LL | type const FOO: [T; 0] = const { [] };
+   |                                                          ^^^^^^^^^^^^
 
 error: anonymous constants referencing generics are not yet supported
-  --> $DIR/type_const-generic-param-in-type.rs:14:38
+  --> $DIR/type_const-generic-param-in-type.rs:12:43
    |
-LL | const BAR: [(); N] = const { [] };
-   |                                      ^^^^^^^^^^^^
+LL | type const BAR: [(); N] = const { [] };
+   |                                           ^^^^^^^^^^^^
 
 error: anonymous constants with lifetimes in their type are not yet supported
-  --> $DIR/type_const-generic-param-in-type.rs:19:30
+  --> $DIR/type_const-generic-param-in-type.rs:16:35
    |
-LL | const BAZ<'a>: [&'a (); 0] = const { [] };
-   |                              ^^^^^^^^^^^^
+LL | type const BAZ<'a>: [&'a (); 0] = const { [] };
+   |                                   ^^^^^^^^^^^^
 
 error: anonymous constants referencing generics are not yet supported
-  --> $DIR/type_const-generic-param-in-type.rs:39:59
+  --> $DIR/type_const-generic-param-in-type.rs:32:64
    |
-LL |     const ASSOC: [T; 0] = const { [] };
-   |                                                           ^^^^^^^^^^^^
+LL |     type const ASSOC: [T; 0] = const { [] };
+   |                                                                ^^^^^^^^^^^^
 
 error: anonymous constants referencing generics are not yet supported
-  --> $DIR/type_const-generic-param-in-type.rs:44:50
+  --> $DIR/type_const-generic-param-in-type.rs:36:55
    |
-LL |     const ASSOC_CONST: [(); N] = const { [] };
-   |                                                  ^^^^^^^^^^^^
+LL |     type const ASSOC_CONST: [(); N] = const { [] };
+   |                                                       ^^^^^^^^^^^^
 
 error: anonymous constants with lifetimes in their type are not yet supported
-  --> $DIR/type_const-generic-param-in-type.rs:49:39
+  --> $DIR/type_const-generic-param-in-type.rs:40:44
    |
-LL |     const ASSOC_LT<'a>: [&'a (); 0] = const { [] };
-   |                                       ^^^^^^^^^^^^
+LL |     type const ASSOC_LT<'a>: [&'a (); 0] = const { [] };
+   |                                            ^^^^^^^^^^^^
 
 error: aborting due to 6 previous errors
 
diff --git a/tests/ui/const-generics/mgca/type_const-generic-param-in-type.nogate.stderr b/tests/ui/const-generics/mgca/type_const-generic-param-in-type.nogate.stderr
index 14ae276324ad..b18bd678973a 100644
--- a/tests/ui/const-generics/mgca/type_const-generic-param-in-type.nogate.stderr
+++ b/tests/ui/const-generics/mgca/type_const-generic-param-in-type.nogate.stderr
@@ -1,56 +1,56 @@
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:9:45
+  --> $DIR/type_const-generic-param-in-type.rs:8:50
    |
-LL | const FOO: [T; 0] = const { [] };
-   |                                             ^ the type must not depend on the parameter `T`
+LL | type const FOO: [T; 0] = const { [] };
+   |                                                  ^ the type must not depend on the parameter `T`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:14:33
+  --> $DIR/type_const-generic-param-in-type.rs:12:38
    |
-LL | const BAR: [(); N] = const { [] };
-   |                                 ^ the type must not depend on the parameter `N`
+LL | type const BAR: [(); N] = const { [] };
+   |                                      ^ the type must not depend on the parameter `N`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:19:18
+  --> $DIR/type_const-generic-param-in-type.rs:16:23
    |
-LL | const BAZ<'a>: [&'a (); 0] = const { [] };
-   |                  ^^ the type must not depend on the parameter `'a`
+LL | type const BAZ<'a>: [&'a (); 0] = const { [] };
+   |                       ^^ the type must not depend on the parameter `'a`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:25:51
+  --> $DIR/type_const-generic-param-in-type.rs:21:56
    |
-LL |     const ASSOC: [T; 0];
-   |                                                   ^ the type must not depend on the parameter `T`
+LL |     type const ASSOC: [T; 0];
+   |                                                        ^ the type must not depend on the parameter `T`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:29:45
+  --> $DIR/type_const-generic-param-in-type.rs:24:50
    |
-LL |     const ASSOC_CONST: [(); N];
-   |                                             ^ the type must not depend on the parameter `N`
+LL |     type const ASSOC_CONST: [(); N];
+   |                                                  ^ the type must not depend on the parameter `N`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:33:27
+  --> $DIR/type_const-generic-param-in-type.rs:27:32
    |
-LL |     const ASSOC_LT<'a>: [&'a (); 0];
-   |                           ^^ the type must not depend on the parameter `'a`
+LL |     type const ASSOC_LT<'a>: [&'a (); 0];
+   |                                ^^ the type must not depend on the parameter `'a`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:39:51
+  --> $DIR/type_const-generic-param-in-type.rs:32:56
    |
-LL |     const ASSOC: [T; 0] = const { [] };
-   |                                                   ^ the type must not depend on the parameter `T`
+LL |     type const ASSOC: [T; 0] = const { [] };
+   |                                                        ^ the type must not depend on the parameter `T`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:44:45
+  --> $DIR/type_const-generic-param-in-type.rs:36:50
    |
-LL |     const ASSOC_CONST: [(); N] = const { [] };
-   |                                             ^ the type must not depend on the parameter `N`
+LL |     type const ASSOC_CONST: [(); N] = const { [] };
+   |                                                  ^ the type must not depend on the parameter `N`
 
 error[E0770]: the type of const parameters must not depend on other generic parameters
-  --> $DIR/type_const-generic-param-in-type.rs:49:27
+  --> $DIR/type_const-generic-param-in-type.rs:40:32
    |
-LL |     const ASSOC_LT<'a>: [&'a (); 0] = const { [] };
-   |                           ^^ the type must not depend on the parameter `'a`
+LL |     type const ASSOC_LT<'a>: [&'a (); 0] = const { [] };
+   |                                ^^ the type must not depend on the parameter `'a`
 
 error: aborting due to 9 previous errors
 
diff --git a/tests/ui/const-generics/mgca/type_const-generic-param-in-type.rs b/tests/ui/const-generics/mgca/type_const-generic-param-in-type.rs
index f1a4aba9bcee..72ae464822e4 100644
--- a/tests/ui/const-generics/mgca/type_const-generic-param-in-type.rs
+++ b/tests/ui/const-generics/mgca/type_const-generic-param-in-type.rs
@@ -5,48 +5,39 @@
 #![feature(adt_const_params, unsized_const_params, min_generic_const_args, generic_const_items)]
 #![cfg_attr(gate, feature(generic_const_parameter_types))]
 
-#[type_const]
-const FOO: [T; 0] = const { [] };
+type const FOO: [T; 0] = const { [] };
 //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
 //[gate]~^^ ERROR anonymous constants referencing generics are not yet supported
 
-#[type_const]
-const BAR: [(); N] = const { [] };
+type const BAR: [(); N] = const { [] };
 //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
 //[gate]~^^ ERROR anonymous constants referencing generics are not yet supported
 
-#[type_const]
-const BAZ<'a>: [&'a (); 0] = const { [] };
+type const BAZ<'a>: [&'a (); 0] = const { [] };
 //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
 //[gate]~^^ ERROR anonymous constants with lifetimes in their type are not yet supported
 
 trait Tr {
-    #[type_const]
-    const ASSOC: [T; 0];
+    type const ASSOC: [T; 0];
     //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
 
-    #[type_const]
-    const ASSOC_CONST: [(); N];
+    type const ASSOC_CONST: [(); N];
     //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
 
-    #[type_const]
-    const ASSOC_LT<'a>: [&'a (); 0];
+    type const ASSOC_LT<'a>: [&'a (); 0];
     //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
 }
 
 impl Tr for () {
-    #[type_const]
-    const ASSOC: [T; 0] = const { [] };
+    type const ASSOC: [T; 0] = const { [] };
     //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
     //[gate]~^^ ERROR anonymous constants referencing generics are not yet supported
 
-    #[type_const]
-    const ASSOC_CONST: [(); N] = const { [] };
+    type const ASSOC_CONST: [(); N] = const { [] };
     //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
     //[gate]~^^ ERROR anonymous constants referencing generics are not yet supported
 
-    #[type_const]
-    const ASSOC_LT<'a>: [&'a (); 0] = const { [] };
+    type const ASSOC_LT<'a>: [&'a (); 0] = const { [] };
     //[nogate]~^ ERROR the type of const parameters must not depend on other generic parameters
     //[gate]~^^ ERROR anonymous constants with lifetimes in their type are not yet supported
 }
diff --git a/tests/ui/const-generics/mgca/type_const-incemental-compile.rs b/tests/ui/const-generics/mgca/type_const-incemental-compile.rs
index 60e6968363db..7094d89d5062 100644
--- a/tests/ui/const-generics/mgca/type_const-incemental-compile.rs
+++ b/tests/ui/const-generics/mgca/type_const-incemental-compile.rs
@@ -6,6 +6,5 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args)]
 
-#[type_const]
-const TYPE_CONST: usize = 0;
+type const TYPE_CONST: usize = 0;
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.rs b/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.rs
index b2c734098009..3262e79478bd 100644
--- a/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.rs
+++ b/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.rs
@@ -4,8 +4,7 @@
 struct A;
 
 impl A {
-    #[type_const]
-    const B = 4;
+    type const B = 4;
     //~^ ERROR: missing type for `const` item
 }
 
diff --git a/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.stderr b/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.stderr
index b44e47cd7e61..77e54ab2f2e9 100644
--- a/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.stderr
+++ b/tests/ui/const-generics/mgca/type_const-inherent-const-omitted-type.stderr
@@ -1,13 +1,13 @@
 error: missing type for `const` item
-  --> $DIR/type_const-inherent-const-omitted-type.rs:8:12
+  --> $DIR/type_const-inherent-const-omitted-type.rs:7:17
    |
-LL |     const B = 4;
-   |            ^
+LL |     type const B = 4;
+   |                 ^
    |
 help: provide a type for the item
    |
-LL |     const B:  = 4;
-   |            ++++++++
+LL |     type const B:  = 4;
+   |                 ++++++++
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/type_const-mismatched-types.rs b/tests/ui/const-generics/mgca/type_const-mismatched-types.rs
index 8d2eae71d330..460c5d7b2199 100644
--- a/tests/ui/const-generics/mgca/type_const-mismatched-types.rs
+++ b/tests/ui/const-generics/mgca/type_const-mismatched-types.rs
@@ -1,22 +1,18 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args)]
 
-#[type_const]
-const FREE: u32 = 5_usize;
+type const FREE: u32 = 5_usize;
 //~^ ERROR mismatched types
 
-#[type_const]
-const FREE2: isize = FREE;
+type const FREE2: isize = FREE;
 //~^ ERROR the constant `5` is not of type `isize`
 
 trait Tr {
-    #[type_const]
-    const N: usize;
+    type const N: usize;
 }
 
 impl Tr for () {
-    #[type_const]
-    const N: usize = false;
+    type const N: usize = false;
     //~^ ERROR mismatched types
 }
 
diff --git a/tests/ui/const-generics/mgca/type_const-mismatched-types.stderr b/tests/ui/const-generics/mgca/type_const-mismatched-types.stderr
index 4029bb7b6bff..152dd9ec0cca 100644
--- a/tests/ui/const-generics/mgca/type_const-mismatched-types.stderr
+++ b/tests/ui/const-generics/mgca/type_const-mismatched-types.stderr
@@ -1,26 +1,26 @@
 error: the constant `5` is not of type `isize`
-  --> $DIR/type_const-mismatched-types.rs:9:1
+  --> $DIR/type_const-mismatched-types.rs:7:1
    |
-LL | const FREE2: isize = FREE;
-   | ^^^^^^^^^^^^^^^^^^ expected `isize`, found `u32`
+LL | type const FREE2: isize = FREE;
+   | ^^^^^^^^^^^^^^^^^^^^^^^ expected `isize`, found `u32`
 
 error[E0308]: mismatched types
-  --> $DIR/type_const-mismatched-types.rs:5:19
+  --> $DIR/type_const-mismatched-types.rs:4:24
    |
-LL | const FREE: u32 = 5_usize;
-   |                   ^^^^^^^ expected `u32`, found `usize`
+LL | type const FREE: u32 = 5_usize;
+   |                        ^^^^^^^ expected `u32`, found `usize`
    |
 help: change the type of the numeric literal from `usize` to `u32`
    |
-LL - const FREE: u32 = 5_usize;
-LL + const FREE: u32 = 5_u32;
+LL - type const FREE: u32 = 5_usize;
+LL + type const FREE: u32 = 5_u32;
    |
 
 error[E0308]: mismatched types
-  --> $DIR/type_const-mismatched-types.rs:19:22
+  --> $DIR/type_const-mismatched-types.rs:15:27
    |
-LL |     const N: usize = false;
-   |                      ^^^^^ expected `usize`, found `bool`
+LL |     type const N: usize = false;
+   |                           ^^^^^ expected `usize`, found `bool`
 
 error: aborting due to 3 previous errors
 
diff --git a/tests/ui/const-generics/mgca/type_const-not-constparamty.rs b/tests/ui/const-generics/mgca/type_const-not-constparamty.rs
index 11db82187b84..b78bb4ca599d 100644
--- a/tests/ui/const-generics/mgca/type_const-not-constparamty.rs
+++ b/tests/ui/const-generics/mgca/type_const-not-constparamty.rs
@@ -5,21 +5,18 @@ struct S;
 
 // FIXME(mgca): need support for ctors without anon const
 // (we use a const-block to trigger an anon const here)
-#[type_const]
-const FREE: S = const { S };
+type const FREE: S = const { S };
 //~^ ERROR `S` must implement `ConstParamTy` to be used as the type of a const generic parameter
 
 trait Tr {
-    #[type_const]
-    const N: S;
+    type const N: S;
     //~^ ERROR `S` must implement `ConstParamTy` to be used as the type of a const generic parameter
 }
 
 impl Tr for S {
     // FIXME(mgca): need support for ctors without anon const
     // (we use a const-block to trigger an anon const here)
-    #[type_const]
-    const N: S = const { S };
+    type const N: S = const { S };
     //~^ ERROR `S` must implement `ConstParamTy` to be used as the type of a const generic parameter
 }
 
diff --git a/tests/ui/const-generics/mgca/type_const-not-constparamty.stderr b/tests/ui/const-generics/mgca/type_const-not-constparamty.stderr
index d07bbde1e62e..2cbb644f2c71 100644
--- a/tests/ui/const-generics/mgca/type_const-not-constparamty.stderr
+++ b/tests/ui/const-generics/mgca/type_const-not-constparamty.stderr
@@ -1,8 +1,8 @@
 error[E0741]: `S` must implement `ConstParamTy` to be used as the type of a const generic parameter
-  --> $DIR/type_const-not-constparamty.rs:9:13
+  --> $DIR/type_const-not-constparamty.rs:8:18
    |
-LL | const FREE: S = const { S };
-   |             ^
+LL | type const FREE: S = const { S };
+   |                  ^
    |
 help: add `#[derive(ConstParamTy, PartialEq, Eq)]` to the struct
    |
@@ -11,10 +11,10 @@ LL | struct S;
    |
 
 error[E0741]: `S` must implement `ConstParamTy` to be used as the type of a const generic parameter
-  --> $DIR/type_const-not-constparamty.rs:22:14
+  --> $DIR/type_const-not-constparamty.rs:19:19
    |
-LL |     const N: S = const { S };
-   |              ^
+LL |     type const N: S = const { S };
+   |                   ^
    |
 help: add `#[derive(ConstParamTy, PartialEq, Eq)]` to the struct
    |
@@ -23,10 +23,10 @@ LL | struct S;
    |
 
 error[E0741]: `S` must implement `ConstParamTy` to be used as the type of a const generic parameter
-  --> $DIR/type_const-not-constparamty.rs:14:14
+  --> $DIR/type_const-not-constparamty.rs:12:19
    |
-LL |     const N: S;
-   |              ^
+LL |     type const N: S;
+   |                   ^
    |
 help: add `#[derive(ConstParamTy, PartialEq, Eq)]` to the struct
    |
diff --git a/tests/ui/const-generics/mgca/type_const-on-generic-expr.rs b/tests/ui/const-generics/mgca/type_const-on-generic-expr.rs
index ac4c4fb59419..f4cf3a4c5ce9 100644
--- a/tests/ui/const-generics/mgca/type_const-on-generic-expr.rs
+++ b/tests/ui/const-generics/mgca/type_const-on-generic-expr.rs
@@ -1,11 +1,11 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args, generic_const_items)]
 
-#[type_const]
-const FREE1: usize = const { std::mem::size_of::() };
+
+type const FREE1: usize = const { std::mem::size_of::() };
 //~^ ERROR generic parameters may not be used in const operations
-#[type_const]
-const FREE2: usize = const { I + 1 };
+
+type const FREE2: usize = const { I + 1 };
 //~^ ERROR generic parameters may not be used in const operations
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr b/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr
index 8d43f2177562..475d2cf312d6 100644
--- a/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr
+++ b/tests/ui/const-generics/mgca/type_const-on-generic-expr.stderr
@@ -1,16 +1,16 @@
 error: generic parameters may not be used in const operations
-  --> $DIR/type_const-on-generic-expr.rs:5:53
+  --> $DIR/type_const-on-generic-expr.rs:5:58
    |
-LL | const FREE1: usize = const { std::mem::size_of::() };
-   |                                                     ^
+LL | type const FREE1: usize = const { std::mem::size_of::() };
+   |                                                          ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
-  --> $DIR/type_const-on-generic-expr.rs:8:46
+  --> $DIR/type_const-on-generic-expr.rs:8:51
    |
-LL | const FREE2: usize = const { I + 1 };
-   |                                              ^
+LL | type const FREE2: usize = const { I + 1 };
+   |                                                   ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
diff --git a/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.rs b/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.rs
index 37de4d4a4abb..2a26138d373f 100644
--- a/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.rs
+++ b/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.rs
@@ -2,25 +2,19 @@
 #![feature(min_generic_const_args, generic_const_items)]
 
 pub trait Tr {
-    #[type_const]
-    const N1: usize;
-    #[type_const]
-    const N2: usize;
-    #[type_const]
-    const N3: usize;
+    type const N1: usize;
+    type const N2: usize;
+    type const N3: usize;
 }
 
 pub struct S;
 
 impl Tr for S {
-    #[type_const]
-    const N1: usize = const { std::mem::size_of::() };
+    type const N1: usize = const { std::mem::size_of::() };
     //~^ ERROR generic parameters may not be used in const operations
-    #[type_const]
-    const N2: usize = const { I + 1 };
+    type const N2: usize = const { I + 1 };
     //~^ ERROR generic parameters may not be used in const operations
-    #[type_const]
-    const N3: usize = const { 2 & X };
+    type const N3: usize = const { 2 & X };
     //~^ ERROR generic parameters may not be used in const operations
 }
 
diff --git a/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr b/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr
index 1a6097752e33..e13d6fbcdd82 100644
--- a/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr
+++ b/tests/ui/const-generics/mgca/type_const-on-generic_expr-2.stderr
@@ -1,24 +1,24 @@
 error: generic parameters may not be used in const operations
-  --> $DIR/type_const-on-generic_expr-2.rs:17:54
+  --> $DIR/type_const-on-generic_expr-2.rs:13:59
    |
-LL |     const N1: usize = const { std::mem::size_of::() };
-   |                                                      ^
+LL |     type const N1: usize = const { std::mem::size_of::() };
+   |                                                           ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
-  --> $DIR/type_const-on-generic_expr-2.rs:20:47
+  --> $DIR/type_const-on-generic_expr-2.rs:15:52
    |
-LL |     const N2: usize = const { I + 1 };
-   |                                               ^
+LL |     type const N2: usize = const { I + 1 };
+   |                                                    ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
 error: generic parameters may not be used in const operations
-  --> $DIR/type_const-on-generic_expr-2.rs:23:35
+  --> $DIR/type_const-on-generic_expr-2.rs:17:40
    |
-LL |     const N3: usize = const { 2 & X };
-   |                                   ^
+LL |     type const N3: usize = const { 2 & X };
+   |                                        ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
diff --git a/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.rs b/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.rs
index ab613859aa5c..a7ff9f0ce03f 100644
--- a/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.rs
+++ b/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.rs
@@ -8,14 +8,13 @@ trait BadTr {
 struct GoodS;
 
 impl BadTr for GoodS {
-    #[type_const]
-    const NUM: = 84;
+    type const NUM: = 84;
     //~^ ERROR: missing type for `const` item
 
 }
 
 fn accept_bad_tr>(_x: &T) {}
-//~^ ERROR use of trait associated const without `#[type_const]`
+//~^ ERROR use of trait associated const not defined as `type const`
 
 fn main() {
     accept_bad_tr::<84, _>(&GoodS);
diff --git a/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.stderr b/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.stderr
index 16f312454ed2..11a60246f6a6 100644
--- a/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.stderr
+++ b/tests/ui/const-generics/mgca/type_const-only-in-impl-omitted-type.stderr
@@ -1,16 +1,16 @@
 error: missing type for `const` item
-  --> $DIR/type_const-only-in-impl-omitted-type.rs:12:15
+  --> $DIR/type_const-only-in-impl-omitted-type.rs:11:20
    |
-LL |     const NUM: = 84;
-   |               ^ help: provide a type for the associated constant: `usize`
+LL |     type const NUM: = 84;
+   |                    ^ help: provide a type for the associated constant: `usize`
 
-error: use of trait associated const without `#[type_const]`
-  --> $DIR/type_const-only-in-impl-omitted-type.rs:17:43
+error: use of trait associated const not defined as `type const`
+  --> $DIR/type_const-only-in-impl-omitted-type.rs:16:43
    |
 LL | fn accept_bad_tr>(_x: &T) {}
    |                                           ^^^^^^^^^^^
    |
-   = note: the declaration in the trait must be marked with `#[type_const]`
+   = note: the declaration in the trait must begin with `type const` not just `const` alone
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/mgca/type_const-only-in-impl.rs b/tests/ui/const-generics/mgca/type_const-only-in-impl.rs
index 1887f511bd62..e016908b3cc3 100644
--- a/tests/ui/const-generics/mgca/type_const-only-in-impl.rs
+++ b/tests/ui/const-generics/mgca/type_const-only-in-impl.rs
@@ -8,12 +8,11 @@ trait BadTr {
 struct GoodS;
 
 impl BadTr for GoodS {
-    #[type_const]
-    const NUM: usize = 84;
+    type const NUM: usize = 84;
 }
 
 fn accept_bad_tr>(_x: &T) {}
-//~^ ERROR use of trait associated const without `#[type_const]`
+//~^ ERROR use of trait associated const not defined as `type const`
 
 fn main() {
     accept_bad_tr::<84, _>(&GoodS);
diff --git a/tests/ui/const-generics/mgca/type_const-only-in-impl.stderr b/tests/ui/const-generics/mgca/type_const-only-in-impl.stderr
index 44632e46fb07..55d5cca6ba69 100644
--- a/tests/ui/const-generics/mgca/type_const-only-in-impl.stderr
+++ b/tests/ui/const-generics/mgca/type_const-only-in-impl.stderr
@@ -1,10 +1,10 @@
-error: use of trait associated const without `#[type_const]`
-  --> $DIR/type_const-only-in-impl.rs:15:43
+error: use of trait associated const not defined as `type const`
+  --> $DIR/type_const-only-in-impl.rs:14:43
    |
 LL | fn accept_bad_tr>(_x: &T) {}
    |                                           ^^^^^^^^^^^
    |
-   = note: the declaration in the trait must be marked with `#[type_const]`
+   = note: the declaration in the trait must begin with `type const` not just `const` alone
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/type_const-only-in-trait.rs b/tests/ui/const-generics/mgca/type_const-only-in-trait.rs
index 2ff38648206a..1def66a1ba68 100644
--- a/tests/ui/const-generics/mgca/type_const-only-in-trait.rs
+++ b/tests/ui/const-generics/mgca/type_const-only-in-trait.rs
@@ -2,15 +2,14 @@
 #![feature(min_generic_const_args)]
 
 trait GoodTr {
-    #[type_const]
-    const NUM: usize;
+    type const NUM: usize;
 }
 
 struct BadS;
 
 impl GoodTr for BadS {
     const NUM: usize = 42;
-    //~^ ERROR implementation of `#[type_const]` const must be marked with `#[type_const]`
+    //~^ ERROR implementation of a `type const` must also be marked as `type const`
 }
 
 fn accept_good_tr>(_x: &T) {}
diff --git a/tests/ui/const-generics/mgca/type_const-only-in-trait.stderr b/tests/ui/const-generics/mgca/type_const-only-in-trait.stderr
index 29f1b724960a..f98b4d9cbbfd 100644
--- a/tests/ui/const-generics/mgca/type_const-only-in-trait.stderr
+++ b/tests/ui/const-generics/mgca/type_const-only-in-trait.stderr
@@ -1,16 +1,14 @@
-error: implementation of `#[type_const]` const must be marked with `#[type_const]`
-  --> $DIR/type_const-only-in-trait.rs:12:5
+error: implementation of a `type const` must also be marked as `type const`
+  --> $DIR/type_const-only-in-trait.rs:11:5
    |
 LL |     const NUM: usize = 42;
    |     ^^^^^^^^^^^^^^^^
    |
-note: trait declaration of const is marked with `#[type_const]`
+note: trait declaration of const is marked as `type const`
   --> $DIR/type_const-only-in-trait.rs:5:5
    |
-LL |     #[type_const]
-   |     ^^^^^^^^^^^^^
-LL |     const NUM: usize;
-   |     ^^^^^^^^^^^^^^^^
+LL |     type const NUM: usize;
+   |     ^^^^^^^^^^^^^^^^^^^^^
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/type_const-pub.rs b/tests/ui/const-generics/mgca/type_const-pub.rs
index f443562394df..70fab75901b9 100644
--- a/tests/ui/const-generics/mgca/type_const-pub.rs
+++ b/tests/ui/const-generics/mgca/type_const-pub.rs
@@ -5,8 +5,7 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args)]
 
-#[type_const]
-pub const TYPE_CONST : usize = 1;
+pub type const TYPE_CONST : usize = 1;
 fn main() {
     print!("{}", TYPE_CONST)
 }
diff --git a/tests/ui/const-generics/mgca/type_const-recursive.rs b/tests/ui/const-generics/mgca/type_const-recursive.rs
index 15e49f747ed6..ebaab51bbc3b 100644
--- a/tests/ui/const-generics/mgca/type_const-recursive.rs
+++ b/tests/ui/const-generics/mgca/type_const-recursive.rs
@@ -1,8 +1,8 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args)]
 
-#[type_const]
-const A: u8 = A;
+
+type const A: u8 = A;
 //~^ ERROR: overflow normalizing the unevaluated constant `A` [E0275]
 
 fn main() {}
diff --git a/tests/ui/const-generics/mgca/type_const-recursive.stderr b/tests/ui/const-generics/mgca/type_const-recursive.stderr
index 947319ec7eda..d21ccb22bc90 100644
--- a/tests/ui/const-generics/mgca/type_const-recursive.stderr
+++ b/tests/ui/const-generics/mgca/type_const-recursive.stderr
@@ -1,8 +1,8 @@
 error[E0275]: overflow normalizing the unevaluated constant `A`
   --> $DIR/type_const-recursive.rs:5:1
    |
-LL | const A: u8 = A;
-   | ^^^^^^^^^^^
+LL | type const A: u8 = A;
+   | ^^^^^^^^^^^^^^^^
    |
    = note: in case this is a recursive type alias, consider using a struct, enum, or union instead
 
diff --git a/tests/ui/const-generics/mgca/type_const-use.rs b/tests/ui/const-generics/mgca/type_const-use.rs
index 04362cd28538..f295bf465e30 100644
--- a/tests/ui/const-generics/mgca/type_const-use.rs
+++ b/tests/ui/const-generics/mgca/type_const-use.rs
@@ -3,8 +3,7 @@
 #![expect(incomplete_features)]
 #![feature(min_generic_const_args)]
 
-#[type_const]
-const CONST: usize = 1;
+type const CONST: usize = 1;
 
 fn uses_const() {
     CONST;
diff --git a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs
index c01fb47002b2..cf75b45b9ff0 100644
--- a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs
+++ b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.rs
@@ -32,9 +32,10 @@ fn generic() {
 
 const NON_TYPE_CONST: usize = const { 1 };
 
-#[type_const]
-//~^ ERROR: the `#[type_const]` attribute is an experimental feature
-const TYPE_CONST: usize = const { 1 };
+
+type const TYPE_CONST: usize = const { 1 };
+//~^ ERROR: `type const` syntax is experimental [E0658]
+//~| ERROR: top-level `type const` are unstable [E0658]
 
 static STATIC: usize = const { 1 };
 
diff --git a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr
index bbcd9f9897a6..dbcb8c56ac5b 100644
--- a/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr
+++ b/tests/ui/const-generics/mgca/unbraced_const_block_const_arg_gated.stderr
@@ -48,16 +48,26 @@ LL |         generic::();
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error[E0658]: the `#[type_const]` attribute is an experimental feature
-  --> $DIR/unbraced_const_block_const_arg_gated.rs:35:1
+error[E0658]: `type const` syntax is experimental
+  --> $DIR/unbraced_const_block_const_arg_gated.rs:36:1
    |
-LL | #[type_const]
-   | ^^^^^^^^^^^^^
+LL | type const TYPE_CONST: usize = const { 1 };
+   | ^^^^^^^^^^
    |
    = note: see issue #132980  for more information
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error: aborting due to 6 previous errors
+error[E0658]: top-level `type const` are unstable
+  --> $DIR/unbraced_const_block_const_arg_gated.rs:36:1
+   |
+LL | type const TYPE_CONST: usize = const { 1 };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 7 previous errors
 
 For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/const-generics/mgca/unmarked-free-const.rs b/tests/ui/const-generics/mgca/unmarked-free-const.rs
index a517231facb7..d6d9b936d95b 100644
--- a/tests/ui/const-generics/mgca/unmarked-free-const.rs
+++ b/tests/ui/const-generics/mgca/unmarked-free-const.rs
@@ -7,5 +7,5 @@ const N: usize = 4;
 
 fn main() {
     let x = [(); N];
-    //~^ ERROR use of `const` in the type system without `#[type_const]`
+    //~^ ERROR use of `const` in the type system not defined as `type const`
 }
diff --git a/tests/ui/const-generics/mgca/unmarked-free-const.stderr b/tests/ui/const-generics/mgca/unmarked-free-const.stderr
index 052ae39fdf67..76875d8db0ef 100644
--- a/tests/ui/const-generics/mgca/unmarked-free-const.stderr
+++ b/tests/ui/const-generics/mgca/unmarked-free-const.stderr
@@ -1,14 +1,11 @@
-error: use of `const` in the type system without `#[type_const]`
+error: use of `const` in the type system not defined as `type const`
   --> $DIR/unmarked-free-const.rs:9:18
    |
+LL | const N: usize = 4;
+   | - help: add `type` before `const` for `N`: `type`
+...
 LL |     let x = [(); N];
    |                  ^
-   |
-help: add `#[type_const]` attribute to `N`
-   |
-LL + #[type_const]
-LL | const N: usize = 4;
-   |
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/mgca/wrong_type_const_arr_diag_trait.rs b/tests/ui/const-generics/mgca/wrong_type_const_arr_diag_trait.rs
index d290619a04ff..909189ae4875 100644
--- a/tests/ui/const-generics/mgca/wrong_type_const_arr_diag_trait.rs
+++ b/tests/ui/const-generics/mgca/wrong_type_const_arr_diag_trait.rs
@@ -4,8 +4,8 @@
 #![allow(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const ASSOC: u8;
+
+    type const ASSOC: u8;
 }
 
 struct TakesArr;
diff --git a/tests/ui/const-generics/ogca/basic-fail.rs b/tests/ui/const-generics/ogca/basic-fail.rs
index 87176c7b067e..e3db3dea3735 100644
--- a/tests/ui/const-generics/ogca/basic-fail.rs
+++ b/tests/ui/const-generics/ogca/basic-fail.rs
@@ -3,17 +3,13 @@
 #![feature(opaque_generic_const_args)]
 #![expect(incomplete_features)]
 
-#[type_const]
-const ADD1: usize = const { N + 1 };
+type const ADD1: usize = const { N + 1 };
 
-#[type_const]
-const INC: usize = const { N + 1 };
+type const INC: usize = const { N + 1 };
 
-#[type_const]
-const ONE: usize = ADD1::<0>;
+type const ONE: usize = ADD1::<0>;
 
-#[type_const]
-const OTHER_ONE: usize = INC::<0>;
+type const OTHER_ONE: usize = INC::<0>;
 
 // Not definitionally equal.
 const ARR: [(); ADD1::<0>] = [(); INC::<0>];
diff --git a/tests/ui/const-generics/ogca/basic-fail.stderr b/tests/ui/const-generics/ogca/basic-fail.stderr
index 05de01b132c6..ce4e8eb1471c 100644
--- a/tests/ui/const-generics/ogca/basic-fail.stderr
+++ b/tests/ui/const-generics/ogca/basic-fail.stderr
@@ -1,5 +1,5 @@
 error[E0308]: mismatched types
-  --> $DIR/basic-fail.rs:19:30
+  --> $DIR/basic-fail.rs:15:30
    |
 LL | const ARR: [(); ADD1::<0>] = [(); INC::<0>];
    |                              ^^^^^^^^^^^^^^ expected an array with a size of const { N + 1 }, found one with a size of const { N + 1 }
diff --git a/tests/ui/const-generics/ogca/basic.rs b/tests/ui/const-generics/ogca/basic.rs
index c8aec9ee24ce..e736484b5c3c 100644
--- a/tests/ui/const-generics/ogca/basic.rs
+++ b/tests/ui/const-generics/ogca/basic.rs
@@ -5,17 +5,13 @@
 #![feature(opaque_generic_const_args)]
 #![expect(incomplete_features)]
 
-#[type_const]
-const ADD1: usize = const { N + 1 };
+type const ADD1: usize = const { N + 1 };
 
-#[type_const]
-const INC: usize = ADD1::;
+type const INC: usize = ADD1::;
 
-#[type_const]
-const ONE: usize = ADD1::<0>;
+type const ONE: usize = ADD1::<0>;
 
-#[type_const]
-const OTHER_ONE: usize = INC::<0>;
+type const OTHER_ONE: usize = INC::<0>;
 
 const ARR: [(); ADD1::<0>] = [(); INC::<0>];
 
diff --git a/tests/ui/const-generics/ogca/coherence-ambiguous.rs b/tests/ui/const-generics/ogca/coherence-ambiguous.rs
index c8c088b1e8fc..21efe4cfe981 100644
--- a/tests/ui/const-generics/ogca/coherence-ambiguous.rs
+++ b/tests/ui/const-generics/ogca/coherence-ambiguous.rs
@@ -4,11 +4,9 @@
 #![feature(generic_const_items, min_generic_const_args, opaque_generic_const_args)]
 #![expect(incomplete_features)]
 
-#[type_const]
-const FOO: usize = const { N + 1 };
+type const FOO: usize = const { N + 1 };
 
-#[type_const]
-const BAR: usize = const { N + 1 };
+type const BAR: usize = const { N + 1 };
 
 trait Trait {}
 
diff --git a/tests/ui/const-generics/ogca/rhs-but-not-root.rs b/tests/ui/const-generics/ogca/rhs-but-not-root.rs
index 4ed136f04ced..2897b16d493f 100644
--- a/tests/ui/const-generics/ogca/rhs-but-not-root.rs
+++ b/tests/ui/const-generics/ogca/rhs-but-not-root.rs
@@ -4,11 +4,9 @@
 #![expect(incomplete_features)]
 
 // Anon consts must be the root of the RHS to be OGCA.
-#[type_const]
-const FOO: usize = ID::;
+type const FOO: usize = ID::;
 //~^ ERROR generic parameters may not be used in const operations
 
-#[type_const]
-const ID: usize = N;
+type const ID: usize = N;
 
 fn main() {}
diff --git a/tests/ui/const-generics/ogca/rhs-but-not-root.stderr b/tests/ui/const-generics/ogca/rhs-but-not-root.stderr
index f4fdf5fb981b..c720b58b9bde 100644
--- a/tests/ui/const-generics/ogca/rhs-but-not-root.stderr
+++ b/tests/ui/const-generics/ogca/rhs-but-not-root.stderr
@@ -1,8 +1,8 @@
 error: generic parameters may not be used in const operations
-  --> $DIR/rhs-but-not-root.rs:8:49
+  --> $DIR/rhs-but-not-root.rs:7:54
    |
-LL | const FOO: usize = ID::;
-   |                                                 ^
+LL | type const FOO: usize = ID::;
+   |                                                      ^
 
 error: aborting due to 1 previous error
 
diff --git a/tests/ui/const-generics/type-relative-path-144547.rs b/tests/ui/const-generics/type-relative-path-144547.rs
index b268055e1f7d..e3532fe635c2 100644
--- a/tests/ui/const-generics/type-relative-path-144547.rs
+++ b/tests/ui/const-generics/type-relative-path-144547.rs
@@ -2,20 +2,21 @@
 
 //@ revisions: min mgca
 //@[mgca] check-pass
-
+#![allow(incomplete_features)]
+#![feature(mgca_type_const_syntax)]
 #![cfg_attr(mgca, feature(min_generic_const_args))]
-#![cfg_attr(mgca, expect(incomplete_features))]
+// FIXME(mgca) syntax is it's own feature flag before
+// expansion and is also an incomplete feature.
+//#![cfg_attr(mgca, expect(incomplete_features))]
 
 trait UnderlyingImpl {
     type InfoType: LevelInfo;
     type SupportedArray;
 }
 
-// FIXME: cfg_attr(..., type_const) is broken (search for sym::type_const in compiler/)
 trait LevelInfo {
     #[cfg(mgca)]
-    #[type_const]
-    const SUPPORTED_SLOTS: usize;
+    type const SUPPORTED_SLOTS: usize;
 
     #[cfg(not(mgca))]
     const SUPPORTED_SLOTS: usize;
@@ -25,8 +26,7 @@ struct Info;
 
 impl LevelInfo for Info {
     #[cfg(mgca)]
-    #[type_const]
-    const SUPPORTED_SLOTS: usize = 1;
+    type const SUPPORTED_SLOTS: usize = 1;
 
     #[cfg(not(mgca))]
     const SUPPORTED_SLOTS: usize = 1;
diff --git a/tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.rs b/tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.rs
new file mode 100644
index 000000000000..eea1c798fd20
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.rs
@@ -0,0 +1,17 @@
+type const FOO: u8 = 10;
+//~^ ERROR `type const` syntax is experimental [E0658]
+//~| ERROR top-level `type const` are unstable [E0658]
+
+trait Bar {
+    type const BAR: bool;
+    //~^ ERROR `type const` syntax is experimental [E0658]
+    //~| ERROR associated `type const` are unstable [E0658]
+}
+
+impl Bar for bool {
+    type const BAR: bool = false;
+    //~^ ERROR `type const` syntax is experimental [E0658]
+    //~| ERROR associated `type const` are unstable [E0658]
+}
+
+fn main() { }
diff --git a/tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.stderr b/tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.stderr
new file mode 100644
index 000000000000..15db8b87d661
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-mgca-type-const-syntax.stderr
@@ -0,0 +1,63 @@
+error[E0658]: `type const` syntax is experimental
+  --> $DIR/feature-gate-mgca-type-const-syntax.rs:1:1
+   |
+LL | type const FOO: u8 = 10;
+   | ^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: `type const` syntax is experimental
+  --> $DIR/feature-gate-mgca-type-const-syntax.rs:6:5
+   |
+LL |     type const BAR: bool;
+   |     ^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: `type const` syntax is experimental
+  --> $DIR/feature-gate-mgca-type-const-syntax.rs:12:5
+   |
+LL |     type const BAR: bool = false;
+   |     ^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: top-level `type const` are unstable
+  --> $DIR/feature-gate-mgca-type-const-syntax.rs:1:1
+   |
+LL | type const FOO: u8 = 10;
+   | ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: associated `type const` are unstable
+  --> $DIR/feature-gate-mgca-type-const-syntax.rs:6:5
+   |
+LL |     type const BAR: bool;
+   |     ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: associated `type const` are unstable
+  --> $DIR/feature-gate-mgca-type-const-syntax.rs:12:5
+   |
+LL |     type const BAR: bool = false;
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 6 previous errors
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/feature-gates/feature-gate-min-generic-const-args.rs b/tests/ui/feature-gates/feature-gate-min-generic-const-args.rs
index 1cf755b2c565..49916736d46e 100644
--- a/tests/ui/feature-gates/feature-gate-min-generic-const-args.rs
+++ b/tests/ui/feature-gates/feature-gate-min-generic-const-args.rs
@@ -1,7 +1,7 @@
 trait Trait {
-    #[type_const]
-    //~^ ERROR experimental
-    const ASSOC: usize;
+    type const ASSOC: usize;
+    //~^ ERROR: associated `type const` are unstable [E0658]
+    //~| ERROR: `type const` syntax is experimental [E0658]
 }
 
 // FIXME(mgca): add suggestion for mgca to this error
diff --git a/tests/ui/feature-gates/feature-gate-min-generic-const-args.stderr b/tests/ui/feature-gates/feature-gate-min-generic-const-args.stderr
index af528a3c1b79..05166b4857fb 100644
--- a/tests/ui/feature-gates/feature-gate-min-generic-const-args.stderr
+++ b/tests/ui/feature-gates/feature-gate-min-generic-const-args.stderr
@@ -7,16 +7,26 @@ LL | fn foo() -> [u8; ::ASSOC] {
    = note: type parameters may not be used in const expressions
    = help: add `#![feature(generic_const_exprs)]` to allow generic const expressions
 
-error[E0658]: the `#[type_const]` attribute is an experimental feature
+error[E0658]: `type const` syntax is experimental
   --> $DIR/feature-gate-min-generic-const-args.rs:2:5
    |
-LL |     #[type_const]
-   |     ^^^^^^^^^^^^^
+LL |     type const ASSOC: usize;
+   |     ^^^^^^^^^^
    |
    = note: see issue #132980  for more information
    = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error: aborting due to 2 previous errors
+error[E0658]: associated `type const` are unstable
+  --> $DIR/feature-gate-min-generic-const-args.rs:2:5
+   |
+LL |     type const ASSOC: usize;
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #132980  for more information
+   = help: add `#![feature(min_generic_const_args)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs
index 0b78911158ba..f18f59cf60bd 100644
--- a/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs
+++ b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.rs
@@ -1,8 +1,7 @@
 #![feature(generic_const_items, min_generic_const_args)]
 #![expect(incomplete_features)]
 
-#[type_const]
-const INC: usize = const { N + 1 };
+type const INC: usize = const { N + 1 };
 //~^ ERROR generic parameters may not be used in const operations
 //~| HELP add `#![feature(opaque_generic_const_args)]`
 
diff --git a/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr
index 1b87443aa3cb..ef771e77f692 100644
--- a/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr
+++ b/tests/ui/feature-gates/feature-gate-opaque-generic-const-args.stderr
@@ -1,8 +1,8 @@
 error: generic parameters may not be used in const operations
-  --> $DIR/feature-gate-opaque-generic-const-args.rs:5:44
+  --> $DIR/feature-gate-opaque-generic-const-args.rs:4:49
    |
-LL | const INC: usize = const { N + 1 };
-   |                                            ^
+LL | type const INC: usize = const { N + 1 };
+   |                                                 ^
    |
    = help: add `#![feature(opaque_generic_const_args)]` to allow generic expressions as the RHS of const items
 
diff --git a/tests/ui/generic-const-items/assoc-const-bindings.rs b/tests/ui/generic-const-items/assoc-const-bindings.rs
index d0301b920e27..2bf4bea16deb 100644
--- a/tests/ui/generic-const-items/assoc-const-bindings.rs
+++ b/tests/ui/generic-const-items/assoc-const-bindings.rs
@@ -5,19 +5,15 @@
 #![allow(incomplete_features)]
 
 trait Owner {
-    #[type_const]
-    const C: u32;
-    #[type_const]
-    const K: u32;
+    type const C: u32;
+    type const K: u32;
     // #[type_const]
     // const Q: Maybe;
 }
 
 impl Owner for () {
-    #[type_const]
-    const C: u32 = N;
-    #[type_const]
-    const K: u32 = const { 99 + 1 };
+    type const C: u32 = N;
+    type const K: u32 = const { 99 + 1 };
     // FIXME(mgca): re-enable once we properly support ctors and generics on paths
     // #[type_const]
     // const Q: Maybe = Maybe::Nothing;
diff --git a/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs b/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs
index 518ed6452369..df680f30fed9 100644
--- a/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs
+++ b/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.rs
@@ -8,8 +8,7 @@ pub struct NoPin;
 impl Pins for NoPin {}
 
 pub trait PinA {
-    #[type_const]
-    const A: &'static () = const { &() };
+    type const A: &'static () = const { &() };
 }
 
 pub trait Pins {}
diff --git a/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.stderr b/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.stderr
index f57fd74ad99d..b719ebe0664f 100644
--- a/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.stderr
+++ b/tests/ui/generic-const-items/assoc-const-no-infer-ice-115806.stderr
@@ -1,5 +1,5 @@
 error[E0119]: conflicting implementations of trait `Pins<_>` for type `NoPin`
-  --> $DIR/assoc-const-no-infer-ice-115806.rs:17:1
+  --> $DIR/assoc-const-no-infer-ice-115806.rs:16:1
    |
 LL | impl Pins for NoPin {}
    | --------------------------- first implementation here
diff --git a/tests/ui/sanitizer/cfi/assoc-const-projection-issue-151878.rs b/tests/ui/sanitizer/cfi/assoc-const-projection-issue-151878.rs
index 50530e4f0db0..6bd995449fc6 100644
--- a/tests/ui/sanitizer/cfi/assoc-const-projection-issue-151878.rs
+++ b/tests/ui/sanitizer/cfi/assoc-const-projection-issue-151878.rs
@@ -8,8 +8,7 @@
 #![expect(incomplete_features)]
 
 trait Trait {
-    #[type_const]
-    const N: usize = 0;
+    type const N: usize = 0;
     fn process(&self, _: [u8; Self::N]) {}
 }
 
diff --git a/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.rs b/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.rs
index 538b0c2b1d46..578a46c67400 100644
--- a/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.rs
+++ b/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.rs
@@ -5,12 +5,11 @@
 //~| WARN the feature `min_generic_const_args` is incomplete
 
 pub trait IsVoid {
-    #[type_const]
-    const IS_VOID: bool;
+
+    type const IS_VOID: bool;
 }
 impl IsVoid for T {
-    #[type_const]
-    default const IS_VOID: bool = false;
+    default type const IS_VOID: bool = false;
 }
 
 pub trait NotVoid {}
diff --git a/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.stderr b/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.stderr
index 6159c2ed331a..9361f7d6222f 100644
--- a/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.stderr
+++ b/tests/ui/specialization/overlap-due-to-unsatisfied-const-bound.stderr
@@ -17,7 +17,7 @@ LL | #![feature(min_generic_const_args, specialization)]
    = help: consider using `min_specialization` instead, which is more stable and complete
 
 error[E0119]: conflicting implementations of trait `Maybe<()>` for type `()`
-  --> $DIR/overlap-due-to-unsatisfied-const-bound.rs:21:1
+  --> $DIR/overlap-due-to-unsatisfied-const-bound.rs:20:1
    |
 LL | impl Maybe for T {}
    | ---------------------- first implementation here

From 9bf9a12381bfab41c5d7c7a1d3f5c1457064575d Mon Sep 17 00:00:00 2001
From: Keith-Cancel 
Date: Thu, 5 Feb 2026 22:01:57 -0800
Subject: [PATCH 1509/1843] Allow provisional mgca syntax of type const 
 =  to be reconized.

Revert, but without type const.

Update symbol for feature err, then update suggestion output, and lastly update tests that change because of those.

Update these new tests with the correct syntax, and few existing tests with the new outputs the merge with main added.

Fix for tidyfmt and some errors when manually resolving a merge conflicts.

Update these tests to use update error messages and type const syntax.

Update comments and error message to use new syntax instead of old type_const attribute.

Remove the type_const attribute

update some more tests to use the new syntax.

Update these test cases.

update feature gate test

Change gate logic for `mgca_type_const_syntax` to work also if `min_generic_const_args` is enabled.

Create a new feature gate that checks for the feature before expansion.

Make rustfmt handle the `type const` syntax correctly.

Add a convience method to check if a RhsKind is type const.

Rename `Const` discriminant to `Body` for `ConstItemRhsKind`

Give the `TraitItemKind` flag an enum instead of a simple bool to better describe what the flag is for.

Update formatting for these match statements.

Update clippy test to use type const syntax.

Update test to use type const syntax.

update rustfmt to match ast items.

Update clippy to match ast and hir items.

Few more test cases that used old attribute, instead of 'type const'

Update to match the output from the feature gate checks.

tidyfmt adjustments.

Update the is_type_const, so I can constrain record!(..) in encoder.rs

Update conditional compilation test.

Move the feature gate to after expansion to allow for cfg(...) to work.

Update some more tests to use the new syntax.

Update type const tests in associated-const-bindings to use new syntax.

Don't check based off the attribute, but the item here.

Update some tests outside of the const_generics folder that were using #[type_const]

update the tests in associated consts that use #[type_const] to use type const

Update these mgca tests with the type const syntax.

Add a flag to TraitItemKind for detecting type const for now. Maybe later change ItemConstRhs to have optional consts but that touches a lot more lines of code.

Don't need into for these now that it's a query.

Add is_type_const query to handle foreign def ids.

update this test to use type const syntax.

Fix logic here, we only want to lower if there is expression in this case.

Update built-in macros to use ConstItemRhsKind

Update more instance of the old ConstItemRhs.

Rename ConstItemKind to ConstItemRhsKind, I noticed there is a typed called ConstantItemKind, so add the Rhs to the name to avoid confusion.

Update lower to use ConstItemKind

Add an other helper method to check if the rhs kinda has an expr.

Update item parse to use ConstItemKind enum.

Felt the field name could a be little clear when editing a few other things.

Change the ConstItem struct see know if we have a type const or regular const.

Make sure this syntax is properly feature gated.
---
 clippy_lints/src/non_copy_const.rs            |  4 +--
 clippy_lints/src/types/mod.rs                 |  2 +-
 clippy_utils/src/ast_utils/mod.rs             | 28 +++++++++++--------
 ...duplication_in_bounds_assoc_const_eq.fixed |  4 +--
 ...it_duplication_in_bounds_assoc_const_eq.rs |  4 +--
 5 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/clippy_lints/src/non_copy_const.rs b/clippy_lints/src/non_copy_const.rs
index f99748127a8f..e09fc0a76366 100644
--- a/clippy_lints/src/non_copy_const.rs
+++ b/clippy_lints/src/non_copy_const.rs
@@ -739,7 +739,7 @@ impl<'tcx> LateLintPass<'tcx> for NonCopyConst<'tcx> {
     }
 
     fn check_trait_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx TraitItem<'_>) {
-        if let TraitItemKind::Const(_, ct_rhs_opt) = item.kind
+        if let TraitItemKind::Const(_, ct_rhs_opt, _) = item.kind
             && let ty = cx.tcx.type_of(item.owner_id).instantiate_identity()
             && match self.is_ty_freeze(cx.tcx, cx.typing_env(), ty) {
                 IsFreeze::No => true,
@@ -931,7 +931,7 @@ fn get_const_hir_value<'tcx>(
         {
             match tcx.hir_node(tcx.local_def_id_to_hir_id(did)) {
                 Node::ImplItem(item) if let ImplItemKind::Const(.., ct_rhs) = item.kind => (did, ct_rhs),
-                Node::TraitItem(item) if let TraitItemKind::Const(.., Some(ct_rhs)) = item.kind => (did, ct_rhs),
+                Node::TraitItem(item) if let TraitItemKind::Const(_, Some(ct_rhs), _) = item.kind => (did, ct_rhs),
                 _ => return None,
             }
         },
diff --git a/clippy_lints/src/types/mod.rs b/clippy_lints/src/types/mod.rs
index 7018146f184b..7d97ce97f487 100644
--- a/clippy_lints/src/types/mod.rs
+++ b/clippy_lints/src/types/mod.rs
@@ -514,7 +514,7 @@ impl<'tcx> LateLintPass<'tcx> for Types {
         };
 
         match item.kind {
-            TraitItemKind::Const(ty, _) | TraitItemKind::Type(_, Some(ty)) => {
+            TraitItemKind::Const(ty, _, _) | TraitItemKind::Type(_, Some(ty)) => {
                 self.check_ty(cx, ty, context);
             },
             TraitItemKind::Fn(ref sig, trait_method) => {
diff --git a/clippy_utils/src/ast_utils/mod.rs b/clippy_utils/src/ast_utils/mod.rs
index cf8716398efb..3b043f7565ef 100644
--- a/clippy_utils/src/ast_utils/mod.rs
+++ b/clippy_utils/src/ast_utils/mod.rs
@@ -355,7 +355,7 @@ pub fn eq_item_kind(l: &ItemKind, r: &ItemKind) -> bool {
                 ident: li,
                 generics: lg,
                 ty: lt,
-                rhs: lb,
+                rhs_kind: lb,
                 define_opaque: _,
             }),
             Const(box ConstItem {
@@ -363,7 +363,7 @@ pub fn eq_item_kind(l: &ItemKind, r: &ItemKind) -> bool {
                 ident: ri,
                 generics: rg,
                 ty: rt,
-                rhs: rb,
+                rhs_kind: rb,
                 define_opaque: _,
             }),
         ) => {
@@ -371,7 +371,7 @@ pub fn eq_item_kind(l: &ItemKind, r: &ItemKind) -> bool {
                 && eq_id(*li, *ri)
                 && eq_generics(lg, rg)
                 && eq_ty(lt, rt)
-                && both(lb.as_ref(), rb.as_ref(), eq_const_item_rhs)
+                && both(Some(lb), Some(rb), eq_const_item_rhs)
         },
         (
             Fn(box ast::Fn {
@@ -615,7 +615,7 @@ pub fn eq_assoc_item_kind(l: &AssocItemKind, r: &AssocItemKind) -> bool {
                 ident: li,
                 generics: lg,
                 ty: lt,
-                rhs: lb,
+                rhs_kind: lb,
                 define_opaque: _,
             }),
             Const(box ConstItem {
@@ -623,7 +623,7 @@ pub fn eq_assoc_item_kind(l: &AssocItemKind, r: &AssocItemKind) -> bool {
                 ident: ri,
                 generics: rg,
                 ty: rt,
-                rhs: rb,
+                rhs_kind: rb,
                 define_opaque: _,
             }),
         ) => {
@@ -631,7 +631,7 @@ pub fn eq_assoc_item_kind(l: &AssocItemKind, r: &AssocItemKind) -> bool {
                 && eq_id(*li, *ri)
                 && eq_generics(lg, rg)
                 && eq_ty(lt, rt)
-                && both(lb.as_ref(), rb.as_ref(), eq_const_item_rhs)
+                && both(Some(lb), Some(rb), eq_const_item_rhs)
         },
         (
             Fn(box ast::Fn {
@@ -791,12 +791,18 @@ pub fn eq_anon_const(l: &AnonConst, r: &AnonConst) -> bool {
     eq_expr(&l.value, &r.value)
 }
 
-pub fn eq_const_item_rhs(l: &ConstItemRhs, r: &ConstItemRhs) -> bool {
-    use ConstItemRhs::*;
+pub fn eq_const_item_rhs(l: &ConstItemRhsKind, r: &ConstItemRhsKind) -> bool {
+    use ConstItemRhsKind::*;
     match (l, r) {
-        (TypeConst(l), TypeConst(r)) => eq_anon_const(l, r),
-        (Body(l), Body(r)) => eq_expr(l, r),
-        (TypeConst(..), Body(..)) | (Body(..), TypeConst(..)) => false,
+        (TypeConst { rhs: Some(l) }, TypeConst { rhs: Some(r) }) => eq_anon_const(l, r),
+        (TypeConst { rhs: None }, TypeConst { rhs: None }) => true,
+        (TypeConst { rhs: Some(..) }, TypeConst { rhs: None }) => false,
+        (TypeConst { rhs: None }, TypeConst { rhs: Some(..) }) => false,
+        (Body { rhs: Some(l) }, Body { rhs: Some(r) }) => eq_expr(l, r),
+        (Body { rhs: None }, Body { rhs: None }) => true,
+        (Body { rhs: None }, Body { rhs: Some(..) }) => false,
+        (Body { rhs: Some(..) }, Body { rhs: None }) => false,
+        (TypeConst {..}, Body { .. }) | ( Body { .. }, TypeConst { .. }) => false,
     }
 }
 
diff --git a/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed b/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed
index f8be3331317c..8d63fc44e7f8 100644
--- a/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed
+++ b/tests/ui/trait_duplication_in_bounds_assoc_const_eq.fixed
@@ -3,8 +3,8 @@
 #![feature(min_generic_const_args)]
 
 trait AssocConstTrait {
-    #[type_const]
-    const ASSOC: usize;
+
+    type const ASSOC: usize;
 }
 fn assoc_const_args()
 where
diff --git a/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs b/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs
index a0d7a653993f..36a83619c0f9 100644
--- a/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs
+++ b/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs
@@ -3,8 +3,8 @@
 #![feature(min_generic_const_args)]
 
 trait AssocConstTrait {
-    #[type_const]
-    const ASSOC: usize;
+
+    type const ASSOC: usize;
 }
 fn assoc_const_args()
 where

From 6546a98d90bf1a95a9d02432bab2d8550d4e980e Mon Sep 17 00:00:00 2001
From: rustbot <47979223+rustbot@users.noreply.github.com>
Date: Mon, 9 Feb 2026 18:00:54 +0100
Subject: [PATCH 1510/1843] Update books

---
 src/doc/book      | 2 +-
 src/doc/nomicon   | 2 +-
 src/doc/reference | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/doc/book b/src/doc/book
index 39aeceaa3aea..05d114287b7d 160000
--- a/src/doc/book
+++ b/src/doc/book
@@ -1 +1 @@
-Subproject commit 39aeceaa3aeab845bc4517e7a44e48727d3b9dbe
+Subproject commit 05d114287b7d6f6c9253d5242540f00fbd6172ab
diff --git a/src/doc/nomicon b/src/doc/nomicon
index 050c002a360f..b8f254a991b8 160000
--- a/src/doc/nomicon
+++ b/src/doc/nomicon
@@ -1 +1 @@
-Subproject commit 050c002a360fa45b701ea34feed7a860dc8a41bf
+Subproject commit b8f254a991b8b7e8f704527f0d4f343a4697dfa9
diff --git a/src/doc/reference b/src/doc/reference
index 990819b86c22..addd0602c819 160000
--- a/src/doc/reference
+++ b/src/doc/reference
@@ -1 +1 @@
-Subproject commit 990819b86c22bbf538c0526f0287670f3dc1a67a
+Subproject commit addd0602c819b6526b9cc97653b0fadca395528c

From 41e0e0690f74685f094d88ae2a0f05379e38873f Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Mon, 9 Feb 2026 19:12:13 +0100
Subject: [PATCH 1511/1843] Move `inline_fluent` to a proc macro

---
 compiler/rustc_macros/src/diagnostics/inline_fluent.rs | 10 ++++++++++
 compiler/rustc_macros/src/diagnostics/mod.rs           |  2 ++
 compiler/rustc_macros/src/lib.rs                       |  9 +++++++++
 3 files changed, 21 insertions(+)
 create mode 100644 compiler/rustc_macros/src/diagnostics/inline_fluent.rs

diff --git a/compiler/rustc_macros/src/diagnostics/inline_fluent.rs b/compiler/rustc_macros/src/diagnostics/inline_fluent.rs
new file mode 100644
index 000000000000..ab0ed6aa6e0e
--- /dev/null
+++ b/compiler/rustc_macros/src/diagnostics/inline_fluent.rs
@@ -0,0 +1,10 @@
+use syn::{LitStr, parse_macro_input};
+
+use crate::diagnostics::message::Message;
+
+pub(crate) fn inline_fluent(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let inline = parse_macro_input!(input as LitStr);
+    let message =
+        Message { attr_span: inline.span(), message_span: inline.span(), value: inline.value() };
+    message.diag_message(None).into()
+}
diff --git a/compiler/rustc_macros/src/diagnostics/mod.rs b/compiler/rustc_macros/src/diagnostics/mod.rs
index 09f05ce972f1..d9168939dd48 100644
--- a/compiler/rustc_macros/src/diagnostics/mod.rs
+++ b/compiler/rustc_macros/src/diagnostics/mod.rs
@@ -1,11 +1,13 @@
 mod diagnostic;
 mod diagnostic_builder;
 mod error;
+mod inline_fluent;
 mod message;
 mod subdiagnostic;
 mod utils;
 
 use diagnostic::{DiagnosticDerive, LintDiagnosticDerive};
+pub(super) use inline_fluent::inline_fluent;
 use proc_macro2::TokenStream;
 use subdiagnostic::SubdiagnosticDerive;
 use synstructure::Structure;
diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs
index 8cd6c0264448..3d2111b86881 100644
--- a/compiler/rustc_macros/src/lib.rs
+++ b/compiler/rustc_macros/src/lib.rs
@@ -241,6 +241,15 @@ decl_derive!(
         applicability)] => diagnostics::subdiagnostic_derive
 );
 
+/// This macro creates a translatable `DiagMessage` from a fluent format string.
+/// It should be used in places where a translatable message is needed, but struct diagnostics are undesired.
+///
+/// This macro statically checks that the message is valid Fluent, but not that variables in the Fluent message actually exist.
+#[proc_macro]
+pub fn inline_fluent(input: TokenStream) -> TokenStream {
+    diagnostics::inline_fluent(input)
+}
+
 decl_derive! {
     [PrintAttribute] =>
     /// Derives `PrintAttribute` for `AttributeKind`.

From 6eb2a8fa9ef2b35d34b021550942cc327fa93b3a Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Mon, 9 Feb 2026 19:12:22 +0100
Subject: [PATCH 1512/1843] Reformat existing error messages

---
 .../src/const_eval/eval_queries.rs            |   6 +-
 .../src/const_eval/machine.rs                 |  16 +-
 compiler/rustc_const_eval/src/errors.rs       | 212 +++++++++---------
 .../src/interpret/intrinsics.rs               |  10 +-
 .../rustc_const_eval/src/interpret/memory.rs  |  56 ++---
 compiler/rustc_errors/src/diagnostic_impls.rs |  12 +-
 compiler/rustc_errors/src/lib.rs              |   3 +-
 compiler/rustc_errors/src/translation.rs      |  11 -
 .../rustc_lint/src/impl_trait_overcaptures.rs |   2 +-
 compiler/rustc_lint/src/lints.rs              |  16 +-
 compiler/rustc_lint/src/non_fmt_panic.rs      |  10 +-
 compiler/rustc_middle/src/middle/stability.rs |  28 ++-
 .../src/lint_tail_expr_drop_order.rs          |  10 +-
 compiler/rustc_passes/src/errors.rs           |   6 +-
 .../src/error_reporting/infer/region.rs       |  10 +-
 compiler/rustc_trait_selection/src/errors.rs  | 104 +++++----
 .../src/errors/note_and_explain.rs            |  62 ++---
 17 files changed, 299 insertions(+), 275 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 25da90298791..131e17ac3173 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -471,9 +471,9 @@ fn report_eval_error<'tcx>(
                 span,
                 inline_fluent!(
                     "evaluation of `{$instance}` failed {$num_frames ->
-    [0] here
-    *[other] inside this call
-}"
+                        [0] here
+                        *[other] inside this call
+                    }"
                 ),
             );
             for frame in frames {
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index da4f97db1c59..8bc515a8b1b5 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -491,10 +491,10 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                     Err(err) => throw_ub_custom!(
                         inline_fluent!(
                             "invalid align passed to `{$name}`: {$align} is {$err_kind ->
-    [not_power_of_two] not a power of 2
-    [too_large] too large
-    *[other] {\"\"}
-}"
+                                [not_power_of_two] not a power of 2
+                                [too_large] too large
+                                *[other] {\"\"}
+                            }"
                         ),
                         name = "const_allocate",
                         err_kind = err.diag_ident(),
@@ -521,10 +521,10 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                     Err(err) => throw_ub_custom!(
                         inline_fluent!(
                             "invalid align passed to `{$name}`: {$align} is {$err_kind ->
-    [not_power_of_two] not a power of 2
-    [too_large] too large
-    *[other] {\"\"}
-}"
+                                [not_power_of_two] not a power of 2
+                                [too_large] too large
+                                *[other] {\"\"}
+                            }"
                         ),
                         name = "const_deallocate",
                         err_kind = err.diag_ident(),
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index a673e0cb1efb..cba2c462bf22 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -393,16 +393,17 @@ impl Subdiagnostic for FrameNote {
         }
         let msg = diag.eagerly_translate(inline_fluent!(
             r#"{$times ->
-    [0] {const_eval_frame_note_inner}
-    *[other] [... {$times} additional calls {const_eval_frame_note_inner} ...]
-}
-
-const_eval_frame_note_inner = inside {$where_ ->
-    [closure] closure
-    [instance] `{$instance}`
-    *[other] {""}
-}
-"#
+                [0] inside {$where_ ->
+                    [closure] closure
+                    [instance] `{$instance}`
+                    *[other] {""}
+                }
+                *[other] [... {$times} additional calls inside {$where_ ->
+                    [closure] closure
+                    [instance] `{$instance}`
+                    *[other] {""}
+                } ...]
+            }"#
         ));
         diag.remove_arg("times");
         diag.remove_arg("where_");
@@ -663,89 +664,96 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             InvalidMeta(InvalidMetaKind::SliceTooBig) => "invalid metadata in wide pointer: slice is bigger than largest supported object".into(),
             InvalidMeta(InvalidMetaKind::TooBig) => "invalid metadata in wide pointer: total size is bigger than largest supported object".into(),
             UnterminatedCString(_) => "reading a null-terminated string starting at {$pointer} with no null found before end of allocation".into(),
-            PointerUseAfterFree(_, _) => inline_fluent!("{$operation ->
-  [MemoryAccess] memory access failed
-  [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
-  *[Dereferenceable] pointer not dereferenceable
-}: {$alloc_id} has been freed, so this pointer is dangling"),
-            PointerOutOfBounds { .. } => inline_fluent!("{$operation ->
-  [MemoryAccess] memory access failed
-  [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
-  *[Dereferenceable] pointer not dereferenceable
-}: {$operation ->
-    [MemoryAccess] attempting to access {$inbounds_size ->
-        [1] 1 byte
-        *[x] {$inbounds_size} bytes
-    }
-    [InboundsPointerArithmetic] attempting to offset pointer by {$inbounds_size ->
-        [1] 1 byte
-        *[x] {$inbounds_size} bytes
-    }
-    *[Dereferenceable] pointer must {$inbounds_size ->
-        [0] point to some allocation
-        [1] be dereferenceable for 1 byte
-        *[x] be dereferenceable for {$inbounds_size} bytes
-    }
-}, but got {$pointer} which {$ptr_offset_is_neg ->
-    [true] points to before the beginning of the allocation
-    *[false] {$inbounds_size_is_neg ->
-        [false] {$alloc_size_minus_ptr_offset ->
-            [0] is at or beyond the end of the allocation of size {$alloc_size ->
-                [1] 1 byte
-                *[x] {$alloc_size} bytes
-            }
-            [1] is only 1 byte from the end of the allocation
-            *[x] is only {$alloc_size_minus_ptr_offset} bytes from the end of the allocation
-        }
-        *[true] {$ptr_offset_abs ->
-            [0] is at the beginning of the allocation
-            *[other] is only {$ptr_offset_abs} bytes from the beginning of the allocation
-        }
-    }
-}
-"),
-            DanglingIntPointer { addr: 0, .. } => inline_fluent!("{$operation ->
-  [MemoryAccess] memory access failed
-  [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
-  *[Dereferenceable] pointer not dereferenceable
-}: {$operation ->
-    [MemoryAccess] attempting to access {$inbounds_size ->
-        [1] 1 byte
-        *[x] {$inbounds_size} bytes
-    }
-    [InboundsPointerArithmetic] attempting to offset pointer by {$inbounds_size ->
-        [1] 1 byte
-        *[x] {$inbounds_size} bytes
-    }
-    *[Dereferenceable] pointer must {$inbounds_size ->
-        [0] point to some allocation
-        [1] be dereferenceable for 1 byte
-        *[x] be dereferenceable for {$inbounds_size} bytes
-    }
-}, but got null pointer"),
-            DanglingIntPointer { .. } => inline_fluent!("{$operation ->
-  [MemoryAccess] memory access failed
-  [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
-  *[Dereferenceable] pointer not dereferenceable
-}: {$operation ->
-    [MemoryAccess] attempting to access {$inbounds_size ->
-        [1] 1 byte
-        *[x] {$inbounds_size} bytes
-    }
-    [InboundsPointerArithmetic] attempting to offset pointer by {$inbounds_size ->
-        [1] 1 byte
-        *[x] {$inbounds_size} bytes
-    }
-    *[Dereferenceable] pointer must {$inbounds_size ->
-        [0] point to some allocation
-        [1] be dereferenceable for 1 byte
-        *[x] be dereferenceable for {$inbounds_size} bytes
-    }
-}, but got {$pointer} which is a dangling pointer (it has no provenance)"),
-            AlignmentCheckFailed { .. } => inline_fluent!("{$msg ->
-    [AccessedPtr] accessing memory
-    *[other] accessing memory based on pointer
-} with alignment {$has}, but alignment {$required} is required"),
+            PointerUseAfterFree(_, _) => inline_fluent!(
+                "{$operation ->
+                    [MemoryAccess] memory access failed
+                    [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
+                    *[Dereferenceable] pointer not dereferenceable
+                }: {$alloc_id} has been freed, so this pointer is dangling"
+            ),
+            PointerOutOfBounds { .. } => inline_fluent!(
+                "{$operation ->
+                    [MemoryAccess] memory access failed
+                    [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
+                    *[Dereferenceable] pointer not dereferenceable
+                }: {$operation ->
+                    [MemoryAccess] attempting to access {$inbounds_size ->
+                        [1] 1 byte
+                        *[x] {$inbounds_size} bytes
+                    }
+                    [InboundsPointerArithmetic] attempting to offset pointer by {$inbounds_size ->
+                        [1] 1 byte
+                        *[x] {$inbounds_size} bytes
+                    }
+                    *[Dereferenceable] pointer must {$inbounds_size ->
+                        [0] point to some allocation
+                        [1] be dereferenceable for 1 byte
+                        *[x] be dereferenceable for {$inbounds_size} bytes
+                    }
+                }, but got {$pointer} which {$ptr_offset_is_neg ->
+                    [true] points to before the beginning of the allocation
+                    *[false] {$inbounds_size_is_neg ->
+                        [false] {$alloc_size_minus_ptr_offset ->
+                            [0] is at or beyond the end of the allocation of size {$alloc_size ->
+                                [1] 1 byte
+                                *[x] {$alloc_size} bytes
+                            }
+                            [1] is only 1 byte from the end of the allocation
+                            *[x] is only {$alloc_size_minus_ptr_offset} bytes from the end of the allocation
+                        }
+                        *[true] {$ptr_offset_abs ->
+                            [0] is at the beginning of the allocation
+                            *[other] is only {$ptr_offset_abs} bytes from the beginning of the allocation
+                        }
+                    }
+                }"
+            ),
+            DanglingIntPointer { addr: 0, .. } => inline_fluent!(
+                "{$operation ->
+                    [MemoryAccess] memory access failed
+                    [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
+                    *[Dereferenceable] pointer not dereferenceable
+                }: {$operation ->
+                    [MemoryAccess] attempting to access {$inbounds_size ->
+                        [1] 1 byte
+                        *[x] {$inbounds_size} bytes
+                    }
+                    [InboundsPointerArithmetic] attempting to offset pointer by {$inbounds_size ->
+                        [1] 1 byte
+                        *[x] {$inbounds_size} bytes
+                    }
+                    *[Dereferenceable] pointer must {$inbounds_size ->
+                        [0] point to some allocation
+                        [1] be dereferenceable for 1 byte
+                        *[x] be dereferenceable for {$inbounds_size} bytes
+                    }
+                }, but got null pointer"),
+            DanglingIntPointer { .. } => inline_fluent!(
+                "{$operation ->
+                    [MemoryAccess] memory access failed
+                    [InboundsPointerArithmetic] in-bounds pointer arithmetic failed
+                    *[Dereferenceable] pointer not dereferenceable
+                }: {$operation ->
+                    [MemoryAccess] attempting to access {$inbounds_size ->
+                        [1] 1 byte
+                        *[x] {$inbounds_size} bytes
+                    }
+                    [InboundsPointerArithmetic] attempting to offset pointer by {$inbounds_size ->
+                        [1] 1 byte
+                        *[x] {$inbounds_size} bytes
+                    }
+                    *[Dereferenceable] pointer must {$inbounds_size ->
+                        [0] point to some allocation
+                        [1] be dereferenceable for 1 byte
+                        *[x] be dereferenceable for {$inbounds_size} bytes
+                    }
+                }, but got {$pointer} which is a dangling pointer (it has no provenance)"),
+            AlignmentCheckFailed { .. } => inline_fluent!(
+                "{$msg ->
+                    [AccessedPtr] accessing memory
+                    *[other] accessing memory based on pointer
+                } with alignment {$has}, but alignment {$required} is required"
+            ),
             WriteToReadOnly(_) => inline_fluent!("writing to {$allocation} which is read-only"),
             DerefFunctionPointer(_) => inline_fluent!("accessing {$allocation} which contains a function"),
             DerefVTablePointer(_) => inline_fluent!("accessing {$allocation} which contains a vtable"),
@@ -936,9 +944,9 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
             NullFnPtr { .. } => {
                 inline_fluent!(
                     "{$front_matter}: encountered a {$maybe ->
-    [true] maybe-null
-    *[false] null
-} function pointer"
+                        [true] maybe-null
+                        *[false] null
+                    } function pointer"
                 )
             }
             NeverVal => {
@@ -1021,17 +1029,17 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
             NullPtr { ptr_kind: PointerKind::Box, .. } => {
                 inline_fluent!(
                     "{$front_matter}: encountered a {$maybe ->
-    [true] maybe-null
-    *[false] null
-} box"
+                        [true] maybe-null
+                        *[false] null
+                    } box"
                 )
             }
             NullPtr { ptr_kind: PointerKind::Ref(_), .. } => {
                 inline_fluent!(
                     "{$front_matter}: encountered a {$maybe ->
-    [true] maybe-null
-    *[false] null
-} reference"
+                        [true] maybe-null
+                        *[false] null
+                    } reference"
                 )
             }
             DanglingPtrNoProvenance { ptr_kind: PointerKind::Box, .. } => {
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index 2ea5e4a25c11..7c49ba79a342 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -461,10 +461,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                         // a < b
                         if intrinsic_name == sym::ptr_offset_from_unsigned {
                             throw_ub_custom!(
-                                inline_fluent!("`ptr_offset_from_unsigned` called when first pointer has smaller {$is_addr ->
-    [true] address
-    *[false] offset
-} than second: {$a_offset} < {$b_offset}"),
+                                inline_fluent!(
+                                    "`ptr_offset_from_unsigned` called when first pointer has smaller {$is_addr ->
+                                        [true] address
+                                        *[false] offset
+                                    } than second: {$a_offset} < {$b_offset}"
+                                ),
                                 a_offset = a_offset,
                                 b_offset = b_offset,
                                 is_addr = is_addr,
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index 28dae2ef3b8b..200155ab010e 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -293,10 +293,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             throw_ub_custom!(
                 inline_fluent!(
                     "{$kind ->
-    [dealloc] deallocating
-    [realloc] reallocating
-    *[other] {\"\"}
-} {$ptr} which does not point to the beginning of an object"
+                        [dealloc] deallocating
+                        [realloc] reallocating
+                        *[other] {\"\"}
+                    } {$ptr} which does not point to the beginning of an object"
                 ),
                 ptr = format!("{ptr:?}"),
                 kind = "realloc"
@@ -379,10 +379,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             throw_ub_custom!(
                 inline_fluent!(
                     "{$kind ->
-    [dealloc] deallocating
-    [realloc] reallocating
-    *[other] {\"\"}
-} {$ptr} which does not point to the beginning of an object"
+                        [dealloc] deallocating
+                        [realloc] reallocating
+                        *[other] {\"\"}
+                    } {$ptr} which does not point to the beginning of an object"
                 ),
                 ptr = format!("{ptr:?}"),
                 kind = "dealloc",
@@ -396,11 +396,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     err_ub_custom!(
                         inline_fluent!(
                             "deallocating {$alloc_id}, which is {$kind ->
-    [fn] a function
-    [vtable] a vtable
-    [static_mem] static memory
-    *[other] {\"\"}
-}"
+                                [fn] a function
+                                [vtable] a vtable
+                                [static_mem] static memory
+                                *[other] {\"\"}
+                            }"
                         ),
                         alloc_id = alloc_id,
                         kind = "fn",
@@ -410,11 +410,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     err_ub_custom!(
                         inline_fluent!(
                             "deallocating {$alloc_id}, which is {$kind ->
-    [fn] a function
-    [vtable] a vtable
-    [static_mem] static memory
-    *[other] {\"\"}
-}"
+                                [fn] a function
+                                [vtable] a vtable
+                                [static_mem] static memory
+                                *[other] {\"\"}
+                            }"
                         ),
                         alloc_id = alloc_id,
                         kind = "vtable",
@@ -424,11 +424,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     err_ub_custom!(
                         inline_fluent!(
                             "deallocating {$alloc_id}, which is {$kind ->
-    [fn] a function
-    [vtable] a vtable
-    [static_mem] static memory
-    *[other] {\"\"}
-}"
+                                [fn] a function
+                                [vtable] a vtable
+                                [static_mem] static memory
+                                *[other] {\"\"}
+                            }"
                         ),
                         alloc_id = alloc_id,
                         kind = "typeid",
@@ -438,11 +438,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     err_ub_custom!(
                         inline_fluent!(
                             "deallocating {$alloc_id}, which is {$kind ->
-    [fn] a function
-    [vtable] a vtable
-    [static_mem] static memory
-    *[other] {\"\"}
-}"
+                                [fn] a function
+                                [vtable] a vtable
+                                [static_mem] static memory
+                                *[other] {\"\"}
+                            }"
                         ),
                         alloc_id = alloc_id,
                         kind = "static_mem"
diff --git a/compiler/rustc_errors/src/diagnostic_impls.rs b/compiler/rustc_errors/src/diagnostic_impls.rs
index c36afc6fc889..ae3e33498ea1 100644
--- a/compiler/rustc_errors/src/diagnostic_impls.rs
+++ b/compiler/rustc_errors/src/diagnostic_impls.rs
@@ -60,11 +60,13 @@ impl Diagnostic<'_, G> for TargetDataLayoutErrors<'_> {
                     .with_arg("cause", cause)
             }
             TargetDataLayoutErrors::InvalidAlignment { cause, err } => {
-                Diag::new(dcx, level, inline_fluent!("invalid alignment for `{$cause}` in \"data-layout\": `{$align}` is {$err_kind ->
-                    [not_power_of_two] not a power of 2
-                    [too_large] too large
-                    *[other] {\"\"}
-                }"))
+                Diag::new(dcx, level, inline_fluent!(
+                    "invalid alignment for `{$cause}` in \"data-layout\": `{$align}` is {$err_kind ->
+                        [not_power_of_two] not a power of 2
+                        [too_large] too large
+                        *[other] {\"\"}
+                    }"
+                ))
                 .with_arg("cause", cause)
                 .with_arg("err_kind", err.diag_ident())
                 .with_arg("align", err.align())
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 2e0afb8fc4c2..4eadb8249405 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -63,6 +63,7 @@ pub use rustc_error_messages::{
 use rustc_hashes::Hash128;
 use rustc_lint_defs::LintExpectationId;
 pub use rustc_lint_defs::{Applicability, listify, pluralize};
+pub use rustc_macros::inline_fluent;
 use rustc_macros::{Decodable, Encodable};
 pub use rustc_span::ErrorGuaranteed;
 pub use rustc_span::fatal_error::{FatalError, FatalErrorMarker, catch_fatal_errors};
@@ -1562,7 +1563,7 @@ impl DelayedDiagInner {
         let msg = match self.note.status() {
             BacktraceStatus::Captured => inline_fluent!(
                 "delayed at {$emitted_at}
-{$note}"
+                {$note}"
             ),
             // Avoid the needless newline when no backtrace has been captured,
             // the display impl should just be a single line.
diff --git a/compiler/rustc_errors/src/translation.rs b/compiler/rustc_errors/src/translation.rs
index b5de3f87e9f3..c6d30032f1af 100644
--- a/compiler/rustc_errors/src/translation.rs
+++ b/compiler/rustc_errors/src/translation.rs
@@ -90,14 +90,3 @@ impl Translator {
         }
     }
 }
-
-/// This macro creates a translatable `DiagMessage` from a literal string.
-/// It should be used in places where a translatable message is needed, but struct diagnostics are undesired.
-///
-/// This is a macro because in the future we may want to globally register these messages.
-#[macro_export]
-macro_rules! inline_fluent {
-    ($inline: literal) => {
-        rustc_errors::DiagMessage::Inline(std::borrow::Cow::Borrowed($inline))
-    };
-}
diff --git a/compiler/rustc_lint/src/impl_trait_overcaptures.rs b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
index 087403719968..aef0021d0660 100644
--- a/compiler/rustc_lint/src/impl_trait_overcaptures.rs
+++ b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
@@ -446,7 +446,7 @@ impl<'a> LintDiagnostic<'a, ()> for ImplTraitOvercapturesLint<'_> {
                     "specifically, {$num_captured ->
                         [one] this lifetime is
                         *[other] these lifetimes are
-                     } in scope but not mentioned in the type's bounds"
+                    } in scope but not mentioned in the type's bounds"
                 ),
             )
             .note(inline_fluent!(
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 0aa5199cffc6..4ec59a3cb98f 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -353,8 +353,10 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinTypeAliasBounds<'_> {
             self.label,
             inline_fluent!("will not be checked at usage sites of the type alias"),
         );
-        diag.note(inline_fluent!("this is a known limitation of the type checker that may be lifted in a future edition.
-            see issue #112792  for more information"));
+        diag.note(inline_fluent!(
+            "this is a known limitation of the type checker that may be lifted in a future edition.
+            see issue #112792  for more information"
+        ));
         if self.enable_feat_help {
             diag.help(inline_fluent!("add `#![feature(lazy_type_alias)]` to the crate attributes to enable the desired semantics"));
         }
@@ -1748,10 +1750,12 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 if doctest {
                     diag.help(inline_fluent!(r#"remove the `#[macro_export]` or make this doc-test a standalone test with its own `fn main() {"{"} ... {"}"}`"#));
                 } else {
-                    diag.help(inline_fluent!("remove the `#[macro_export]` or move this `macro_rules!` outside the of the current {$body_kind_descr} {$depth ->
-                        [one] `{$body_name}`
-                       *[other] `{$body_name}` and up {$depth} bodies
-                    }"));
+                    diag.help(inline_fluent!(
+                        "remove the `#[macro_export]` or move this `macro_rules!` outside the of the current {$body_kind_descr} {$depth ->
+                            [one] `{$body_name}`
+                            *[other] `{$body_name}` and up {$depth} bodies
+                        }"
+                    ));
                 }
 
                 diag.note(inline_fluent!("a `macro_rules!` definition is non-local if it is nested inside an item and has a `#[macro_export]` attribute"));
diff --git a/compiler/rustc_lint/src/non_fmt_panic.rs b/compiler/rustc_lint/src/non_fmt_panic.rs
index daf62277693a..10a37637400b 100644
--- a/compiler/rustc_lint/src/non_fmt_panic.rs
+++ b/compiler/rustc_lint/src/non_fmt_panic.rs
@@ -196,10 +196,12 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
                 if let Some((open, close, del)) = find_delimiters(cx, span) {
                     lint.arg("already_suggested", suggest_display || suggest_debug);
                     lint.multipart_suggestion(
-                        inline_fluent!("{$already_suggested ->
-                            [true] or use
-                            *[false] use
-                        } std::panic::panic_any instead"),
+                        inline_fluent!(
+                            "{$already_suggested ->
+                                [true] or use
+                                *[false] use
+                            } std::panic::panic_any instead"
+                        ),
                         if del == '(' {
                             vec![(span.until(open), "std::panic::panic_any".into())]
                         } else {
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index fce0158aa511..903ef645207d 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -128,19 +128,25 @@ pub struct Deprecated {
 impl<'a, G: EmissionGuarantee> rustc_errors::LintDiagnostic<'a, G> for Deprecated {
     fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, G>) {
         diag.primary_message(match &self.since_kind {
-            DeprecatedSinceKind::InEffect => inline_fluent!("use of deprecated {$kind} `{$path}`{$has_note ->
-                [true] : {$note}
-                *[other] {\"\"}
-            }"),
-            DeprecatedSinceKind::InFuture => inline_fluent!("use of {$kind} `{$path}` that will be deprecated in a future Rust version{$has_note ->
-                [true] : {$note}
-                *[other] {\"\"}
-            }"),
-            DeprecatedSinceKind::InVersion(_) => {
-                inline_fluent!("use of {$kind} `{$path}` that will be deprecated in future version {$version}{$has_note ->
+            DeprecatedSinceKind::InEffect => inline_fluent!(
+                "use of deprecated {$kind} `{$path}`{$has_note ->
                     [true] : {$note}
                     *[other] {\"\"}
-                }")
+                }"
+            ),
+            DeprecatedSinceKind::InFuture => inline_fluent!(
+                "use of {$kind} `{$path}` that will be deprecated in a future Rust version{$has_note ->
+                    [true] : {$note}
+                    *[other] {\"\"}
+                }"
+            ),
+            DeprecatedSinceKind::InVersion(_) => {
+                inline_fluent!(
+                    "use of {$kind} `{$path}` that will be deprecated in future version {$version}{$has_note ->
+                        [true] : {$note}
+                        *[other] {\"\"}
+                    }"
+                )
             }
         });
         diag.arg("kind", self.kind);
diff --git a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs
index 789398c58880..448ed78702b7 100644
--- a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs
+++ b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs
@@ -542,10 +542,12 @@ impl Subdiagnostic for LocalLabel<'_> {
             dtor.add_to_diag(diag);
         }
         let msg =
-            diag.eagerly_translate(inline_fluent!("{$is_dropped_first_edition_2024 ->
-                [true] up until Edition 2021 `{$name}` is dropped last but will be dropped earlier in Edition 2024
-                *[false] `{$name}` will be dropped later as of Edition 2024
-            }"));
+            diag.eagerly_translate(inline_fluent!(
+                "{$is_dropped_first_edition_2024 ->
+                    [true] up until Edition 2021 `{$name}` is dropped last but will be dropped earlier in Edition 2024
+                    *[false] `{$name}` will be dropped later as of Edition 2024
+                }"
+            ));
         diag.span_label(self.span, msg);
     }
 }
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index f420bba9b4e9..8a70302c4c4e 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -684,9 +684,9 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for NoMainErr {
             // There were some functions named `main` though. Try to give the user a hint.
             inline_fluent!(
                 "the main function must be defined at the crate level{$has_filename ->
-                [true] {\" \"}(in `{$filename}`)
-                *[false] {\"\"}
-            }"
+                    [true] {\" \"}(in `{$filename}`)
+                    *[false] {\"\"}
+                }"
             )
         } else if self.has_filename {
             inline_fluent!("consider adding a `main` function to `{$filename}`")
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
index d15fb40dd1cb..c99dde6d45b9 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
@@ -252,10 +252,12 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             SubregionOrigin::RelateParamBound(span, ty, opt_span) => {
                 RegionOriginNote::WithName {
                     span,
-                    msg: inline_fluent!("...so that the type `{$name}` will meet its required lifetime bounds{$continues ->
-[true] ...
-*[false] {\"\"}
-}"),
+                    msg: inline_fluent!(
+                        "...so that the type `{$name}` will meet its required lifetime bounds{$continues ->
+                            [true] ...
+                            *[false] {\"\"}
+                        }"
+                    ),
                     name: &self.ty_to_string(ty),
                     continues: opt_span.is_some(),
                 }
diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs
index 013e4b522fb5..529a7fcf5f1c 100644
--- a/compiler/rustc_trait_selection/src/errors.rs
+++ b/compiler/rustc_trait_selection/src/errors.rs
@@ -102,10 +102,16 @@ pub struct NegativePositiveConflict<'tcx> {
 impl Diagnostic<'_, G> for NegativePositiveConflict<'_> {
     #[track_caller]
     fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> {
-        let mut diag = Diag::new(dcx, level, inline_fluent!("found both positive and negative implementation of trait `{$trait_desc}`{$self_desc ->
-[none] {\"\"}
-*[default] {\" \"}for type `{$self_desc}`
-}:"));
+        let mut diag = Diag::new(
+            dcx,
+            level,
+            inline_fluent!(
+            "found both positive and negative implementation of trait `{$trait_desc}`{$self_desc ->
+                [none] {\"\"}
+                *[default] {\" \"}for type `{$self_desc}`
+            }:"
+        ),
+        );
         diag.arg("trait_desc", self.trait_desc.print_only_trait_path().to_string());
         diag.arg("self_desc", self.self_ty.map_or_else(|| "none".to_string(), |ty| ty.to_string()));
         diag.span(self.impl_span);
@@ -157,9 +163,9 @@ impl Subdiagnostic for AdjustSignatureBorrow {
                 diag.multipart_suggestion_verbose(
                     inline_fluent!(
                         "consider adjusting the signature so it borrows its {$borrow_len ->
-[one] argument
-*[other] arguments
-}"
+                            [one] argument
+                            *[other] arguments
+                        }"
                     ),
                     to_borrow,
                     Applicability::MaybeIncorrect,
@@ -170,9 +176,9 @@ impl Subdiagnostic for AdjustSignatureBorrow {
                 diag.multipart_suggestion_verbose(
                     inline_fluent!(
                         "consider adjusting the signature so it does not borrow its {$remove_borrow_len ->
-[one] argument
-*[other] arguments
-}"
+                            [one] argument
+                            *[other] arguments
+                        }"
                     ),
                     remove_borrow,
                     Applicability::MaybeIncorrect,
@@ -505,18 +511,18 @@ impl Subdiagnostic for RegionOriginNote<'_> {
                     span,
                     inline_fluent!(
                         "...so that the {$requirement ->
-[method_compat] method type is compatible with trait
-[type_compat] associated type is compatible with trait
-[const_compat] const is compatible with trait
-[expr_assignable] expression is assignable
-[if_else_different] `if` and `else` have incompatible types
-[no_else] `if` missing an `else` returns `()`
-[fn_main_correct_type] `main` function has the correct type
-[fn_lang_correct_type] lang item function has the correct type
-[intrinsic_correct_type] intrinsic has the correct type
-[method_correct_type] method receiver has the correct type
-*[other] types are compatible
-}"
+                            [method_compat] method type is compatible with trait
+                            [type_compat] associated type is compatible with trait
+                            [const_compat] const is compatible with trait
+                            [expr_assignable] expression is assignable
+                            [if_else_different] `if` and `else` have incompatible types
+                            [no_else] `if` missing an `else` returns `()`
+                            [fn_main_correct_type] `main` function has the correct type
+                            [fn_lang_correct_type] lang item function has the correct type
+                            [intrinsic_correct_type] intrinsic has the correct type
+                            [method_correct_type] method receiver has the correct type
+                            *[other] types are compatible
+                        }"
                     ),
                 );
                 diag.arg("requirement", requirement);
@@ -531,18 +537,18 @@ impl Subdiagnostic for RegionOriginNote<'_> {
                     span,
                     inline_fluent!(
                         "...so that {$requirement ->
-[method_compat] method type is compatible with trait
-[type_compat] associated type is compatible with trait
-[const_compat] const is compatible with trait
-[expr_assignable] expression is assignable
-[if_else_different] `if` and `else` have incompatible types
-[no_else] `if` missing an `else` returns `()`
-[fn_main_correct_type] `main` function has the correct type
-[fn_lang_correct_type] lang item function has the correct type
-[intrinsic_correct_type] intrinsic has the correct type
-[method_correct_type] method receiver has the correct type
-*[other] types are compatible
-}"
+                            [method_compat] method type is compatible with trait
+                            [type_compat] associated type is compatible with trait
+                            [const_compat] const is compatible with trait
+                            [expr_assignable] expression is assignable
+                            [if_else_different] `if` and `else` have incompatible types
+                            [no_else] `if` missing an `else` returns `()`
+                            [fn_main_correct_type] `main` function has the correct type
+                            [fn_lang_correct_type] lang item function has the correct type
+                            [intrinsic_correct_type] intrinsic has the correct type
+                            [method_correct_type] method receiver has the correct type
+                            *[other] types are compatible
+                        }"
                     ),
                 );
                 diag.arg("requirement", requirement);
@@ -578,9 +584,9 @@ impl Subdiagnostic for LifetimeMismatchLabels {
                     span,
                     inline_fluent!(
                         "...but data{$label_var1_exists ->
-[true] {\" \"}from `{$label_var1}`
-*[false] {\"\"}
-} is returned here"
+                            [true] {\" \"}from `{$label_var1}`
+                            *[false] {\"\"}
+                        } is returned here"
                     ),
                 );
                 diag.arg("label_var1_exists", label_var1.is_some());
@@ -614,12 +620,12 @@ impl Subdiagnostic for LifetimeMismatchLabels {
                         span,
                         inline_fluent!(
                             "...but data{$label_var1_exists ->
-[true] {\" \"}from `{$label_var1}`
-*[false] {\"\"}
-} flows{$label_var2_exists ->
-[true] {\" \"}into `{$label_var2}`
-*[false] {\"\"}
-} here"
+                                [true] {\" \"}from `{$label_var1}`
+                                *[false] {\"\"}
+                            } flows{$label_var2_exists ->
+                                [true] {\" \"}into `{$label_var2}`
+                                *[false] {\"\"}
+                            } here"
                         ),
                     );
                     diag.arg("label_var1_exists", label_var1.is_some());
@@ -789,12 +795,12 @@ impl Subdiagnostic for AddLifetimeParamsSuggestion<'_> {
             diag.multipart_suggestion_verbose(
                 inline_fluent!(
                     "consider {$is_reuse ->
-[true] reusing
-*[false] introducing
-} a named lifetime parameter{$is_impl ->
-[true] {\" \"}and update trait if needed
-*[false] {\"\"}
-}"
+                        [true] reusing
+                        *[false] introducing
+                    } a named lifetime parameter{$is_impl ->
+                        [true] {\" \"}and update trait if needed
+                        *[false] {\"\"}
+                    }"
                 ),
                 visitor.suggestions,
                 Applicability::MaybeIncorrect,
diff --git a/compiler/rustc_trait_selection/src/errors/note_and_explain.rs b/compiler/rustc_trait_selection/src/errors/note_and_explain.rs
index 8e86b8fae157..6a7e1d252806 100644
--- a/compiler/rustc_trait_selection/src/errors/note_and_explain.rs
+++ b/compiler/rustc_trait_selection/src/errors/note_and_explain.rs
@@ -171,37 +171,37 @@ impl Subdiagnostic for RegionExplanation<'_> {
 
         let msg = diag.eagerly_translate(inline_fluent!(
             "{$pref_kind ->
-*[should_not_happen] [{$pref_kind}]
-[ref_valid_for] ...the reference is valid for
-[content_valid_for] ...but the borrowed content is only valid for
-[type_obj_valid_for] object type is valid for
-[source_pointer_valid_for] source pointer is only valid for
-[type_satisfy] type must satisfy
-[type_outlive] type must outlive
-[lf_param_instantiated_with] lifetime parameter instantiated with
-[lf_param_must_outlive] but lifetime parameter must outlive
-[lf_instantiated_with] lifetime instantiated with
-[lf_must_outlive] but lifetime must outlive
-[pointer_valid_for] the pointer is valid for
-[data_valid_for] but the referenced data is only valid for
-[empty] {\"\"}
-}{$pref_kind ->
-[empty] {\"\"}
-*[other] {\" \"}
-}{$desc_kind ->
-*[should_not_happen] [{$desc_kind}]
-[restatic] the static lifetime
-[revar] lifetime {$desc_arg}
-[as_defined] the lifetime `{$desc_arg}` as defined here
-[as_defined_anon] the anonymous lifetime as defined here
-[defined_here] the anonymous lifetime defined here
-[defined_here_reg] the lifetime `{$desc_arg}` as defined here
-}{$suff_kind ->
-*[should_not_happen] [{$suff_kind}]
-[empty]{\"\"}
-[continues] ...
-[req_by_binding] {\" \"}as required by this binding
-}"
+                *[should_not_happen] [{$pref_kind}]
+                [ref_valid_for] ...the reference is valid for
+                [content_valid_for] ...but the borrowed content is only valid for
+                [type_obj_valid_for] object type is valid for
+                [source_pointer_valid_for] source pointer is only valid for
+                [type_satisfy] type must satisfy
+                [type_outlive] type must outlive
+                [lf_param_instantiated_with] lifetime parameter instantiated with
+                [lf_param_must_outlive] but lifetime parameter must outlive
+                [lf_instantiated_with] lifetime instantiated with
+                [lf_must_outlive] but lifetime must outlive
+                [pointer_valid_for] the pointer is valid for
+                [data_valid_for] but the referenced data is only valid for
+                [empty] {\"\"}
+            }{$pref_kind ->
+                [empty] {\"\"}
+                *[other] {\" \"}
+            }{$desc_kind ->
+                *[should_not_happen] [{$desc_kind}]
+                [restatic] the static lifetime
+                [revar] lifetime {$desc_arg}
+                [as_defined] the lifetime `{$desc_arg}` as defined here
+                [as_defined_anon] the anonymous lifetime as defined here
+                [defined_here] the anonymous lifetime defined here
+                [defined_here_reg] the lifetime `{$desc_arg}` as defined here
+                }{$suff_kind ->
+                *[should_not_happen] [{$suff_kind}]
+                [empty]{\"\"}
+                [continues] ...
+                [req_by_binding] {\" \"}as required by this binding
+            }"
         ));
         diag.restore_args();
         if let Some(span) = self.desc.span {

From 61b5382ad8972cf0c218b2eb2ebc88a78b741d03 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Mon, 9 Feb 2026 16:16:31 +0100
Subject: [PATCH 1513/1843] Correctly check if a macro call is actually a macro
 call in rustdoc highlighter

---
 src/librustdoc/html/highlight.rs | 39 ++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs
index f70b350de283..1c162a79c4c4 100644
--- a/src/librustdoc/html/highlight.rs
+++ b/src/librustdoc/html/highlight.rs
@@ -1246,7 +1246,7 @@ impl<'src> Classifier<'src> {
                 LiteralKind::Float { .. } | LiteralKind::Int { .. } => Class::Number,
             },
             TokenKind::GuardedStrPrefix => return no_highlight(sink),
-            TokenKind::RawIdent if let Some((TokenKind::Bang, _)) = self.peek_non_trivia() => {
+            TokenKind::RawIdent if self.check_if_macro_call("") => {
                 self.new_macro_span(text, sink, before, file_span);
                 return;
             }
@@ -1268,9 +1268,7 @@ impl<'src> Classifier<'src> {
                         // So if it's not a keyword which can be followed by a value (like `if` or
                         // `return`) and the next non-whitespace token is a `!`, then we consider
                         // it's a macro.
-                        if !NON_MACRO_KEYWORDS.contains(&text)
-                            && matches!(self.peek_non_trivia(), Some((TokenKind::Bang, _)))
-                        {
+                        if !NON_MACRO_KEYWORDS.contains(&text) && self.check_if_macro_call(text) {
                             self.new_macro_span(text, sink, before, file_span);
                             return;
                         }
@@ -1278,7 +1276,7 @@ impl<'src> Classifier<'src> {
                     }
                     // If it's not a keyword and the next non whitespace token is a `!`, then
                     // we consider it's a macro.
-                    _ if matches!(self.peek_non_trivia(), Some((TokenKind::Bang, _))) => {
+                    _ if self.check_if_macro_call(text) => {
                         self.new_macro_span(text, sink, before, file_span);
                         return;
                     }
@@ -1339,6 +1337,37 @@ impl<'src> Classifier<'src> {
         self.tokens.stop_peeking();
         None
     }
+
+    fn check_if_macro_call(&mut self, ident: &str) -> bool {
+        let mut has_bang = false;
+        let is_macro_rule_ident = ident == "macro_rules";
+
+        while let Some((kind, _)) = self.tokens.peek_next() {
+            if let TokenKind::Whitespace
+            | TokenKind::LineComment { doc_style: None }
+            | TokenKind::BlockComment { doc_style: None, .. } = kind
+            {
+                continue;
+            }
+            if !has_bang {
+                if kind != TokenKind::Bang {
+                    break;
+                }
+                has_bang = true;
+                continue;
+            }
+            self.tokens.stop_peeking();
+            if is_macro_rule_ident {
+                return matches!(kind, TokenKind::Ident | TokenKind::RawIdent);
+            }
+            return matches!(
+                kind,
+                TokenKind::OpenParen | TokenKind::OpenBracket | TokenKind::OpenBrace
+            );
+        }
+        self.tokens.stop_peeking();
+        false
+    }
 }
 
 fn is_keyword(symbol: Symbol) -> bool {

From 31c4ad33cc3c4b56a63657408842af4f628bfa59 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez 
Date: Mon, 9 Feb 2026 16:17:09 +0100
Subject: [PATCH 1514/1843] Add regression test for macro call highlighting

---
 .../source-code-pages/macro-call.rs           | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 tests/rustdoc-html/source-code-pages/macro-call.rs

diff --git a/tests/rustdoc-html/source-code-pages/macro-call.rs b/tests/rustdoc-html/source-code-pages/macro-call.rs
new file mode 100644
index 000000000000..df2d22aa9b60
--- /dev/null
+++ b/tests/rustdoc-html/source-code-pages/macro-call.rs
@@ -0,0 +1,29 @@
+// This is yet another test to ensure that only macro calls are considered as such
+// by the rustdoc highlighter.
+// This is a regression test for .
+
+#![crate_name = "foo"]
+
+//@ has src/foo/macro-call.rs.html
+//@ count - '//code/span[@class="macro"]' 2
+//@ has - '//code/span[@class="macro"]' 'panic!'
+//@ has - '//code/span[@class="macro"]' 'macro_rules!'
+
+pub struct Layout;
+
+impl Layout {
+    pub fn new() {}
+}
+
+pub fn bar() {
+    let layout = Layout::new::();
+    if layout != Layout::new::() {
+        panic!();
+    }
+    let macro_rules = 3;
+    if macro_rules != 3 {}
+}
+
+macro_rules! blob {
+    () => {}
+}

From 5a721a117ff8c94dd5083b28027dfa6b83a3aa72 Mon Sep 17 00:00:00 2001
From: A4-Tacks 
Date: Tue, 10 Feb 2026 04:13:05 +0800
Subject: [PATCH 1515/1843] Improve variable name

---
 src/tools/rust-analyzer/crates/ide-assists/src/utils.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs b/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs
index 1732d8c2018e..dd4bbd145537 100644
--- a/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs
+++ b/src/tools/rust-analyzer/crates/ide-assists/src/utils.rs
@@ -767,7 +767,7 @@ fn generate_impl_inner(
     });
     let generic_args =
         generic_params.as_ref().map(|params| params.to_generic_args().clone_for_update());
-    let trait_where_clause = trait_
+    let adt_assoc_bounds = trait_
         .as_ref()
         .zip(generic_params.as_ref())
         .and_then(|(trait_, params)| generic_param_associated_bounds(adt, trait_, params));
@@ -787,7 +787,7 @@ fn generate_impl_inner(
             false,
             trait_,
             ty,
-            trait_where_clause,
+            adt_assoc_bounds,
             adt.where_clause(),
             body,
         ),

From aefb9a9ae261dae8d84b801c9396dd571a75339a Mon Sep 17 00:00:00 2001
From: Augie Fackler 
Date: Mon, 9 Feb 2026 14:07:30 -0500
Subject: [PATCH 1516/1843] tests: adapt align-offset.rs for InstCombine
 improvements in LLVM 23

Upstream has improved InstCombine so that it can shrink added constants
using known zeroes, which caused a little bit of change in this test. As
far as I can tell either output is fine, so we just accept both.
---
 tests/assembly-llvm/align_offset.rs | 8 ++++----
 tests/codegen-llvm/align-offset.rs  | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/assembly-llvm/align_offset.rs b/tests/assembly-llvm/align_offset.rs
index d9902ce336b0..5663dca30b25 100644
--- a/tests/assembly-llvm/align_offset.rs
+++ b/tests/assembly-llvm/align_offset.rs
@@ -4,7 +4,7 @@
 #![crate_type = "rlib"]
 
 // CHECK-LABEL: align_offset_byte_ptr
-// CHECK: leaq 31
+// CHECK: leaq {{31|28}}
 // CHECK: andq $-32
 // CHECK: subq
 #[no_mangle]
@@ -13,7 +13,7 @@ pub fn align_offset_byte_ptr(ptr: *const u8) -> usize {
 }
 
 // CHECK-LABEL: align_offset_byte_slice
-// CHECK: leaq 31
+// CHECK: leaq {{31|28}}
 // CHECK: andq $-32
 // CHECK: subq
 #[no_mangle]
@@ -22,7 +22,7 @@ pub fn align_offset_byte_slice(slice: &[u8]) -> usize {
 }
 
 // CHECK-LABEL: align_offset_word_ptr
-// CHECK: leaq 31
+// CHECK: leaq {{31|28}}
 // CHECK: andq $-32
 // CHECK: subq
 // CHECK: shrq
@@ -35,7 +35,7 @@ pub fn align_offset_word_ptr(ptr: *const u32) -> usize {
 }
 
 // CHECK-LABEL: align_offset_word_slice
-// CHECK: leaq 31
+// CHECK: leaq {{31|28}}
 // CHECK: andq $-32
 // CHECK: subq
 // CHECK: shrq
diff --git a/tests/codegen-llvm/align-offset.rs b/tests/codegen-llvm/align-offset.rs
index 21062cc0a914..383e9dd45dab 100644
--- a/tests/codegen-llvm/align-offset.rs
+++ b/tests/codegen-llvm/align-offset.rs
@@ -24,7 +24,7 @@ pub fn align_to4(x: &[u8]) -> bool {
 #[no_mangle]
 pub fn align_offset_byte_ptr(ptr: *const u8) -> usize {
     // CHECK: %[[ADDR:.+]] = ptrtoint ptr %ptr to [[USIZE:i[0-9]+]]
-    // CHECK: %[[UP:.+]] = add [[USIZE]] %[[ADDR]], 31
+    // CHECK: %[[UP:.+]] = add [[USIZE]] %[[ADDR]], {{31|28}}
     // CHECK: %[[ALIGNED:.+]] = and [[USIZE]] %[[UP]], -32
     // CHECK: %[[OFFSET:.+]] = sub [[USIZE]] %[[ALIGNED]], %[[ADDR]]
 
@@ -41,7 +41,7 @@ pub fn align_offset_byte_ptr(ptr: *const u8) -> usize {
 #[no_mangle]
 pub fn align_offset_word_slice(slice: &[Align4]) -> usize {
     // CHECK: %[[ADDR:.+]] = ptrtoint ptr %slice.0 to [[USIZE]]
-    // CHECK: %[[UP:.+]] = add [[USIZE]] %[[ADDR]], 31
+    // CHECK: %[[UP:.+]] = add [[USIZE]] %[[ADDR]], {{31|28}}
     // CHECK: %[[ALIGNED:.+]] = and [[USIZE]] %[[UP]], -32
     // CHECK: %[[BOFFSET:.+]] = sub [[USIZE]] %[[ALIGNED]], %[[ADDR]]
     // CHECK: %[[OFFSET:.+]] = lshr exact [[USIZE]] %[[BOFFSET]], 2
@@ -57,7 +57,7 @@ pub fn align_offset_word_slice(slice: &[Align4]) -> usize {
 #[no_mangle]
 pub fn align_offset_word_ptr(ptr: *const Align4) -> usize {
     // CHECK: %[[ADDR:.+]] = ptrtoint ptr %ptr to [[USIZE]]
-    // CHECK: %[[UP:.+]] = add [[USIZE]] %[[ADDR]], 31
+    // CHECK: %[[UP:.+]] = add [[USIZE]] %[[ADDR]], {{31|28}}
     // CHECK: %[[ALIGNED:.+]] = and [[USIZE]] %[[UP]], -32
     // CHECK: %[[BOFFSET:.+]] = sub [[USIZE]] %[[ALIGNED]], %[[ADDR]]
 

From 34ed98677513c81efa3fa7cf5948a810ac4c9e48 Mon Sep 17 00:00:00 2001
From: Manuel Drehwald 
Date: Mon, 9 Feb 2026 16:31:39 -0500
Subject: [PATCH 1517/1843] improve build docs

---
 src/doc/rustc-dev-guide/src/offload/internals.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/doc/rustc-dev-guide/src/offload/internals.md b/src/doc/rustc-dev-guide/src/offload/internals.md
index 520c48d95896..836fd7ad2670 100644
--- a/src/doc/rustc-dev-guide/src/offload/internals.md
+++ b/src/doc/rustc-dev-guide/src/offload/internals.md
@@ -8,9 +8,15 @@ also offer more advanced, possibly unsafe, interfaces which allow a higher degre
 The implementation is based on LLVM's "offload" project, which is already used by OpenMP to run Fortran or C++ code on GPUs.
 While the project is under development, users will need to call other compilers like clang to finish the compilation process.
 
-## High-level design:
+## High-level compilation design:
 We use a single-source, two-pass compilation approach. 
 
 First we compile all functions that should be offloaded for the device (e.g nvptx64, amdgcn-amd-amdhsa, intel in the future). Currently we require cumbersome `#cfg(target_os="")` annotations, but we intend to recognize those in the future based on our offload intrinsic. 
+This first compilation currently does not leverage rustc's internal Query system, so it will always recompile your kernels at the moment. This should be easy to fix, but we prioritize features and runtime performance improvements at the moment. Please reach out if you want to implement it, though!
 
 We then compile the code for the host (e.g. x86-64), where most of the offloading logic happens. On the host side, we generate calls to the openmp offload runtime, to inform it about the layout of the types (a simplified version of the autodiff TypeTrees). We also use the type system to figure out whether kernel arguments have to be moved only to the device (e.g. `&[f32;1024]`), from the device, or both (e.g. `&mut [f64]`). We then launch the kernel, after which we inform the runtime to end this environment and move data back (as far as needed).
+
+The second pass for the host will load the kernel artifacts from the previous compilation. rustc in general may not "guess" or hardcode the build directory layout, and as such it must be told the path to the kernel artifacts in the second invocation. The logic for this could be integrated into cargo, but it also only requires a trivial cargo wrapper, which we could trivially provide via crates.io till we see larger adoption.
+
+It might seem tempting to think about a single-source, single pass compilation approach. However, a lot of the rustc frontend (e.g. AST) will drop any dead code (e.g. code behind an inactive `cfg`). Getting the frontend to expand and lower code for two targets naively will result in multiple definitions of the same symbol (and other issues). Trying to teach the whole rustc middle and backend to be aware that any symbol now might contain two implementations is a large undertaking, and it is questionable why we should make the whole compiler more complex, if the alternative is a ~5 line cargo wrapper. We still control the full compilation pipeline and have both host and device code available, therefore there shouldn't be a runtime performance difference between the two approaches. 
+

From 90fd76845cc73288f680f0d14d55f511bfe41fac Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sun, 4 Jan 2026 16:52:51 +1100
Subject: [PATCH 1518/1843] Make it clearer that `check_pat_lit` only handles
 literal patterns

---
 compiler/rustc_hir_typeck/src/pat.rs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs
index f054145dc7e9..13f87c092352 100644
--- a/compiler/rustc_hir_typeck/src/pat.rs
+++ b/compiler/rustc_hir_typeck/src/pat.rs
@@ -3,6 +3,7 @@ use std::collections::hash_map::Entry::{Occupied, Vacant};
 
 use rustc_abi::FieldIdx;
 use rustc_ast as ast;
+use rustc_data_structures::assert_matches;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::codes::*;
 use rustc_errors::{
@@ -24,7 +25,6 @@ use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
 use rustc_session::parse::feature_err;
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
-use rustc_span::source_map::Spanned;
 use rustc_span::{BytePos, DUMMY_SP, Ident, Span, kw, sym};
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode};
@@ -611,7 +611,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.write_ty(*hir_id, ty);
                 ty
             }
-            PatKind::Expr(lt) => self.check_pat_lit(pat.span, lt, expected, &pat_info.top_info),
+            PatKind::Expr(expr @ PatExpr { kind: PatExprKind::Lit { lit, .. }, .. }) => {
+                self.check_pat_lit(pat.span, expr, &lit.node, expected, &pat_info.top_info)
+            }
             PatKind::Range(lhs, rhs, _) => {
                 self.check_pat_range(pat.span, lhs, rhs, expected, &pat_info.top_info)
             }
@@ -938,23 +940,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn check_pat_lit(
         &self,
         span: Span,
-        lt: &hir::PatExpr<'tcx>,
+        expr: &hir::PatExpr<'tcx>,
+        lit_kind: &ast::LitKind,
         expected: Ty<'tcx>,
         ti: &TopInfo<'tcx>,
     ) -> Ty<'tcx> {
+        assert_matches!(expr.kind, hir::PatExprKind::Lit { .. });
+
         // We've already computed the type above (when checking for a non-ref pat),
         // so avoid computing it again.
-        let ty = self.node_ty(lt.hir_id);
+        let ty = self.node_ty(expr.hir_id);
 
         // Byte string patterns behave the same way as array patterns
         // They can denote both statically and dynamically-sized byte arrays.
         // Additionally, when `deref_patterns` is enabled, byte string literal patterns may have
         // types `[u8]` or `[u8; N]`, in order to type, e.g., `deref!(b"..."): Vec`.
         let mut pat_ty = ty;
-        if let hir::PatExprKind::Lit {
-            lit: Spanned { node: ast::LitKind::ByteStr(..), .. }, ..
-        } = lt.kind
-        {
+        if matches!(lit_kind, ast::LitKind::ByteStr(..)) {
             let tcx = self.tcx;
             let expected = self.structurally_resolve_type(span, expected);
             match *expected.kind() {
@@ -962,7 +964,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 ty::Ref(_, inner_ty, _)
                     if self.try_structurally_resolve_type(span, inner_ty).is_slice() =>
                 {
-                    trace!(?lt.hir_id.local_id, "polymorphic byte string lit");
+                    trace!(?expr.hir_id.local_id, "polymorphic byte string lit");
                     pat_ty = Ty::new_imm_ref(
                         tcx,
                         tcx.lifetimes.re_static,
@@ -988,9 +990,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // When `deref_patterns` is enabled, in order to allow `deref!("..."): String`, we allow
         // string literal patterns to have type `str`. This is accounted for when lowering to MIR.
         if self.tcx.features().deref_patterns()
-            && let hir::PatExprKind::Lit {
-                lit: Spanned { node: ast::LitKind::Str(..), .. }, ..
-            } = lt.kind
+            && matches!(lit_kind, ast::LitKind::Str(..))
             && self.try_structurally_resolve_type(span, expected).is_str()
         {
             pat_ty = self.tcx.types.str_;

From 91d47a5c0d82cce04c1e4aef2eacad4f8c9b7e8b Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sun, 8 Feb 2026 13:40:20 +1100
Subject: [PATCH 1519/1843] Don't import `TyCtxt` from `crate::ty::context`

---
 .../rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs   | 3 +--
 compiler/rustc_middle/src/ty/inhabitedness/mod.rs              | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
index d03e593e37b9..c935869b504c 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
@@ -2,8 +2,7 @@ use rustc_macros::HashStable;
 use smallvec::SmallVec;
 use tracing::instrument;
 
-use crate::ty::context::TyCtxt;
-use crate::ty::{self, DefId, OpaqueTypeKey, Ty, TypingEnv};
+use crate::ty::{self, DefId, OpaqueTypeKey, Ty, TyCtxt, TypingEnv};
 
 /// Represents whether some type is inhabited in a given context.
 /// Examples of uninhabited types are `!`, `enum Void {}`, or a struct
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 7eb74b52b44c..5c4c1733be29 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -47,8 +47,7 @@ use rustc_type_ir::TyKind::*;
 use tracing::instrument;
 
 use crate::query::Providers;
-use crate::ty::context::TyCtxt;
-use crate::ty::{self, DefId, Ty, TypeVisitableExt, VariantDef, Visibility};
+use crate::ty::{self, DefId, Ty, TyCtxt, TypeVisitableExt, VariantDef, Visibility};
 
 pub mod inhabited_predicate;
 

From 4234d2dd72b38069981a6ba0c4c6befb22d3276b Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sun, 8 Feb 2026 13:40:20 +1100
Subject: [PATCH 1520/1843] Move `impl Interner for TyCtxt` to its own
 submodule

---
 compiler/rustc_middle/src/ty/context.rs       | 808 +-----------------
 .../src/ty/context/impl_interner.rs           | 804 +++++++++++++++++
 2 files changed, 815 insertions(+), 797 deletions(-)
 create mode 100644 compiler/rustc_middle/src/ty/context/impl_interner.rs

diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 1bb83530487d..fbb959ce710d 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -2,6 +2,7 @@
 
 #![allow(rustc::usage_of_ty_tykind)]
 
+mod impl_interner;
 pub mod tls;
 
 use std::borrow::{Borrow, Cow};
@@ -16,6 +17,7 @@ use std::{fmt, iter, mem};
 
 use rustc_abi::{ExternAbi, FieldIdx, Layout, LayoutData, TargetDataLayout, VariantIdx};
 use rustc_ast as ast;
+use rustc_data_structures::defer;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::intern::Interned;
@@ -27,12 +29,9 @@ use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{
     self, DynSend, DynSync, FreezeReadGuard, Lock, RwLock, WorkerLocal,
 };
-use rustc_data_structures::{debug_assert_matches, defer};
-use rustc_errors::{
-    Applicability, Diag, DiagCtxtHandle, ErrorGuaranteed, LintDiagnostic, MultiSpan,
-};
+use rustc_errors::{Applicability, Diag, DiagCtxtHandle, LintDiagnostic, MultiSpan};
 use rustc_hir::attrs::AttributeKind;
-use rustc_hir::def::{CtorKind, CtorOf, DefKind};
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId};
 use rustc_hir::definitions::{DefPathData, Definitions, DisambiguatorState};
 use rustc_hir::intravisit::VisitorExt;
@@ -50,16 +49,13 @@ use rustc_session::lint::Lint;
 use rustc_span::def_id::{CRATE_DEF_ID, DefPathHash, StableCrateId};
 use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw};
 use rustc_type_ir::TyKind::*;
-use rustc_type_ir::lang_items::{SolverAdtLangItem, SolverLangItem, SolverTraitLangItem};
 pub use rustc_type_ir::lift::Lift;
-use rustc_type_ir::{
-    CollectAndApply, Interner, TypeFlags, TypeFoldable, WithCachedTypeInfo, elaborate, search_graph,
-};
+use rustc_type_ir::{CollectAndApply, TypeFlags, WithCachedTypeInfo, elaborate, search_graph};
 use tracing::{debug, instrument};
 
 use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindVTable};
-use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind, CanonicalVarKinds};
+use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind};
 use crate::lint::lint_level;
 use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature};
@@ -70,798 +66,16 @@ use crate::query::plumbing::QuerySystem;
 use crate::query::{IntoQueryParam, LocalCrate, Providers, TyCtxtAt};
 use crate::thir::Thir;
 use crate::traits;
-use crate::traits::cache::WithDepNode;
-use crate::traits::solve::{
-    self, CanonicalInput, ExternalConstraints, ExternalConstraintsData, PredefinedOpaques,
-    QueryResult, inspect,
-};
+use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData, PredefinedOpaques};
 use crate::ty::predicate::ExistentialPredicateStableCmpExt as _;
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Clauses, Const, GenericArg, GenericArgs,
-    GenericArgsRef, GenericParamDefKind, List, ListWithCachedTypeInfo, ParamConst, ParamTy,
-    Pattern, PatternKind, PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind,
-    PredicatePolarity, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid,
-    ValTree, ValTreeKind, Visibility,
+    GenericArgsRef, GenericParamDefKind, List, ListWithCachedTypeInfo, ParamConst, Pattern,
+    PatternKind, PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, PredicatePolarity,
+    Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid, ValTree, ValTreeKind,
+    Visibility,
 };
 
-#[allow(rustc::usage_of_ty_tykind)]
-impl<'tcx> Interner for TyCtxt<'tcx> {
-    fn next_trait_solver_globally(self) -> bool {
-        self.next_trait_solver_globally()
-    }
-
-    type DefId = DefId;
-    type LocalDefId = LocalDefId;
-    type TraitId = DefId;
-    type ForeignId = DefId;
-    type FunctionId = DefId;
-    type ClosureId = DefId;
-    type CoroutineClosureId = DefId;
-    type CoroutineId = DefId;
-    type AdtId = DefId;
-    type ImplId = DefId;
-    type UnevaluatedConstId = DefId;
-    type Span = Span;
-
-    type GenericArgs = ty::GenericArgsRef<'tcx>;
-
-    type GenericArgsSlice = &'tcx [ty::GenericArg<'tcx>];
-    type GenericArg = ty::GenericArg<'tcx>;
-    type Term = ty::Term<'tcx>;
-    type BoundVarKinds = &'tcx List>;
-
-    type PredefinedOpaques = solve::PredefinedOpaques<'tcx>;
-
-    fn mk_predefined_opaques_in_body(
-        self,
-        data: &[(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)],
-    ) -> Self::PredefinedOpaques {
-        self.mk_predefined_opaques_in_body(data)
-    }
-    type LocalDefIds = &'tcx ty::List;
-    type CanonicalVarKinds = CanonicalVarKinds<'tcx>;
-    fn mk_canonical_var_kinds(
-        self,
-        kinds: &[ty::CanonicalVarKind],
-    ) -> Self::CanonicalVarKinds {
-        self.mk_canonical_var_kinds(kinds)
-    }
-
-    type ExternalConstraints = ExternalConstraints<'tcx>;
-    fn mk_external_constraints(
-        self,
-        data: ExternalConstraintsData,
-    ) -> ExternalConstraints<'tcx> {
-        self.mk_external_constraints(data)
-    }
-    type DepNodeIndex = DepNodeIndex;
-    fn with_cached_task(self, task: impl FnOnce() -> T) -> (T, DepNodeIndex) {
-        self.dep_graph.with_anon_task(self, crate::dep_graph::dep_kinds::TraitSelect, task)
-    }
-    type Ty = Ty<'tcx>;
-    type Tys = &'tcx List>;
-
-    type FnInputTys = &'tcx [Ty<'tcx>];
-    type ParamTy = ParamTy;
-    type Symbol = Symbol;
-
-    type ErrorGuaranteed = ErrorGuaranteed;
-    type BoundExistentialPredicates = &'tcx List>;
-
-    type AllocId = crate::mir::interpret::AllocId;
-    type Pat = Pattern<'tcx>;
-    type PatList = &'tcx List>;
-    type Safety = hir::Safety;
-    type Abi = ExternAbi;
-    type Const = ty::Const<'tcx>;
-
-    type ParamConst = ty::ParamConst;
-    type ValueConst = ty::Value<'tcx>;
-    type ExprConst = ty::Expr<'tcx>;
-    type ValTree = ty::ValTree<'tcx>;
-    type ScalarInt = ty::ScalarInt;
-
-    type Region = Region<'tcx>;
-    type EarlyParamRegion = ty::EarlyParamRegion;
-    type LateParamRegion = ty::LateParamRegion;
-
-    type RegionAssumptions = &'tcx ty::List>;
-
-    type ParamEnv = ty::ParamEnv<'tcx>;
-    type Predicate = Predicate<'tcx>;
-
-    type Clause = Clause<'tcx>;
-    type Clauses = ty::Clauses<'tcx>;
-
-    type Tracked = WithDepNode;
-    fn mk_tracked(
-        self,
-        data: T,
-        dep_node: DepNodeIndex,
-    ) -> Self::Tracked {
-        WithDepNode::new(dep_node, data)
-    }
-    fn get_tracked(self, tracked: &Self::Tracked) -> T {
-        tracked.get(self)
-    }
-
-    fn with_global_cache(self, f: impl FnOnce(&mut search_graph::GlobalCache) -> R) -> R {
-        f(&mut *self.new_solver_evaluation_cache.lock())
-    }
-
-    fn canonical_param_env_cache_get_or_insert(
-        self,
-        param_env: ty::ParamEnv<'tcx>,
-        f: impl FnOnce() -> ty::CanonicalParamEnvCacheEntry,
-        from_entry: impl FnOnce(&ty::CanonicalParamEnvCacheEntry) -> R,
-    ) -> R {
-        let mut cache = self.new_solver_canonical_param_env_cache.lock();
-        let entry = cache.entry(param_env).or_insert_with(f);
-        from_entry(entry)
-    }
-
-    fn assert_evaluation_is_concurrent(&self) {
-        // Turns out, the assumption for this function isn't perfect.
-        // See trait-system-refactor-initiative#234.
-    }
-
-    fn expand_abstract_consts>>(self, t: T) -> T {
-        self.expand_abstract_consts(t)
-    }
-
-    type GenericsOf = &'tcx ty::Generics;
-
-    fn generics_of(self, def_id: DefId) -> &'tcx ty::Generics {
-        self.generics_of(def_id)
-    }
-
-    type VariancesOf = &'tcx [ty::Variance];
-
-    fn variances_of(self, def_id: DefId) -> Self::VariancesOf {
-        self.variances_of(def_id)
-    }
-
-    fn opt_alias_variances(
-        self,
-        kind: impl Into,
-        def_id: DefId,
-    ) -> Option<&'tcx [ty::Variance]> {
-        self.opt_alias_variances(kind, def_id)
-    }
-
-    fn type_of(self, def_id: DefId) -> ty::EarlyBinder<'tcx, Ty<'tcx>> {
-        self.type_of(def_id)
-    }
-    fn type_of_opaque_hir_typeck(self, def_id: LocalDefId) -> ty::EarlyBinder<'tcx, Ty<'tcx>> {
-        self.type_of_opaque_hir_typeck(def_id)
-    }
-    fn const_of_item(self, def_id: DefId) -> ty::EarlyBinder<'tcx, Const<'tcx>> {
-        self.const_of_item(def_id)
-    }
-    fn anon_const_kind(self, def_id: DefId) -> ty::AnonConstKind {
-        self.anon_const_kind(def_id)
-    }
-
-    type AdtDef = ty::AdtDef<'tcx>;
-    fn adt_def(self, adt_def_id: DefId) -> Self::AdtDef {
-        self.adt_def(adt_def_id)
-    }
-
-    fn alias_ty_kind(self, alias: ty::AliasTy<'tcx>) -> ty::AliasTyKind {
-        match self.def_kind(alias.def_id) {
-            DefKind::AssocTy => {
-                if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(alias.def_id))
-                {
-                    ty::Inherent
-                } else {
-                    ty::Projection
-                }
-            }
-            DefKind::OpaqueTy => ty::Opaque,
-            DefKind::TyAlias => ty::Free,
-            kind => bug!("unexpected DefKind in AliasTy: {kind:?}"),
-        }
-    }
-
-    fn alias_term_kind(self, alias: ty::AliasTerm<'tcx>) -> ty::AliasTermKind {
-        match self.def_kind(alias.def_id) {
-            DefKind::AssocTy => {
-                if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(alias.def_id))
-                {
-                    ty::AliasTermKind::InherentTy
-                } else {
-                    ty::AliasTermKind::ProjectionTy
-                }
-            }
-            DefKind::AssocConst => {
-                if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(alias.def_id))
-                {
-                    ty::AliasTermKind::InherentConst
-                } else {
-                    ty::AliasTermKind::ProjectionConst
-                }
-            }
-            DefKind::OpaqueTy => ty::AliasTermKind::OpaqueTy,
-            DefKind::TyAlias => ty::AliasTermKind::FreeTy,
-            DefKind::Const => ty::AliasTermKind::FreeConst,
-            DefKind::AnonConst | DefKind::Ctor(_, CtorKind::Const) => {
-                ty::AliasTermKind::UnevaluatedConst
-            }
-            kind => bug!("unexpected DefKind in AliasTy: {kind:?}"),
-        }
-    }
-
-    fn trait_ref_and_own_args_for_alias(
-        self,
-        def_id: DefId,
-        args: ty::GenericArgsRef<'tcx>,
-    ) -> (ty::TraitRef<'tcx>, &'tcx [ty::GenericArg<'tcx>]) {
-        debug_assert_matches!(self.def_kind(def_id), DefKind::AssocTy | DefKind::AssocConst);
-        let trait_def_id = self.parent(def_id);
-        debug_assert_matches!(self.def_kind(trait_def_id), DefKind::Trait);
-        let trait_ref = ty::TraitRef::from_assoc(self, trait_def_id, args);
-        (trait_ref, &args[trait_ref.args.len()..])
-    }
-
-    fn mk_args(self, args: &[Self::GenericArg]) -> ty::GenericArgsRef<'tcx> {
-        self.mk_args(args)
-    }
-
-    fn mk_args_from_iter(self, args: I) -> T::Output
-    where
-        I: Iterator,
-        T: CollectAndApply>,
-    {
-        self.mk_args_from_iter(args)
-    }
-
-    fn check_args_compatible(self, def_id: DefId, args: ty::GenericArgsRef<'tcx>) -> bool {
-        self.check_args_compatible(def_id, args)
-    }
-
-    fn debug_assert_args_compatible(self, def_id: DefId, args: ty::GenericArgsRef<'tcx>) {
-        self.debug_assert_args_compatible(def_id, args);
-    }
-
-    /// Assert that the args from an `ExistentialTraitRef` or `ExistentialProjection`
-    /// are compatible with the `DefId`. Since we're missing a `Self` type, stick on
-    /// a dummy self type and forward to `debug_assert_args_compatible`.
-    fn debug_assert_existential_args_compatible(
-        self,
-        def_id: Self::DefId,
-        args: Self::GenericArgs,
-    ) {
-        // FIXME: We could perhaps add a `skip: usize` to `debug_assert_args_compatible`
-        // to avoid needing to reintern the set of args...
-        if cfg!(debug_assertions) {
-            self.debug_assert_args_compatible(
-                def_id,
-                self.mk_args_from_iter(
-                    [self.types.trait_object_dummy_self.into()].into_iter().chain(args.iter()),
-                ),
-            );
-        }
-    }
-
-    fn mk_type_list_from_iter(self, args: I) -> T::Output
-    where
-        I: Iterator,
-        T: CollectAndApply, &'tcx List>>,
-    {
-        self.mk_type_list_from_iter(args)
-    }
-
-    fn parent(self, def_id: DefId) -> DefId {
-        self.parent(def_id)
-    }
-
-    fn recursion_limit(self) -> usize {
-        self.recursion_limit().0
-    }
-
-    type Features = &'tcx rustc_feature::Features;
-
-    fn features(self) -> Self::Features {
-        self.features()
-    }
-
-    fn coroutine_hidden_types(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes>>> {
-        self.coroutine_hidden_types(def_id)
-    }
-
-    fn fn_sig(self, def_id: DefId) -> ty::EarlyBinder<'tcx, ty::PolyFnSig<'tcx>> {
-        self.fn_sig(def_id)
-    }
-
-    fn coroutine_movability(self, def_id: DefId) -> rustc_ast::Movability {
-        self.coroutine_movability(def_id)
-    }
-
-    fn coroutine_for_closure(self, def_id: DefId) -> DefId {
-        self.coroutine_for_closure(def_id)
-    }
-
-    fn generics_require_sized_self(self, def_id: DefId) -> bool {
-        self.generics_require_sized_self(def_id)
-    }
-
-    fn item_bounds(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
-        self.item_bounds(def_id).map_bound(IntoIterator::into_iter)
-    }
-
-    fn item_self_bounds(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
-        self.item_self_bounds(def_id).map_bound(IntoIterator::into_iter)
-    }
-
-    fn item_non_self_bounds(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
-        self.item_non_self_bounds(def_id).map_bound(IntoIterator::into_iter)
-    }
-
-    fn predicates_of(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
-        ty::EarlyBinder::bind(
-            self.predicates_of(def_id).instantiate_identity(self).predicates.into_iter(),
-        )
-    }
-
-    fn own_predicates_of(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
-        ty::EarlyBinder::bind(
-            self.predicates_of(def_id).instantiate_own_identity().map(|(clause, _)| clause),
-        )
-    }
-
-    fn explicit_super_predicates_of(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator, Span)>> {
-        self.explicit_super_predicates_of(def_id).map_bound(|preds| preds.into_iter().copied())
-    }
-
-    fn explicit_implied_predicates_of(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator, Span)>> {
-        self.explicit_implied_predicates_of(def_id).map_bound(|preds| preds.into_iter().copied())
-    }
-
-    fn impl_super_outlives(
-        self,
-        impl_def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
-        self.impl_super_outlives(impl_def_id)
-    }
-
-    fn impl_is_const(self, def_id: DefId) -> bool {
-        debug_assert_matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true });
-        self.is_conditionally_const(def_id)
-    }
-
-    fn fn_is_const(self, def_id: DefId) -> bool {
-        debug_assert_matches!(
-            self.def_kind(def_id),
-            DefKind::Fn | DefKind::AssocFn | DefKind::Ctor(CtorOf::Struct, CtorKind::Fn)
-        );
-        self.is_conditionally_const(def_id)
-    }
-
-    fn alias_has_const_conditions(self, def_id: DefId) -> bool {
-        debug_assert_matches!(self.def_kind(def_id), DefKind::AssocTy | DefKind::OpaqueTy);
-        self.is_conditionally_const(def_id)
-    }
-
-    fn const_conditions(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>>> {
-        ty::EarlyBinder::bind(
-            self.const_conditions(def_id).instantiate_identity(self).into_iter().map(|(c, _)| c),
-        )
-    }
-
-    fn explicit_implied_const_bounds(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>>> {
-        ty::EarlyBinder::bind(
-            self.explicit_implied_const_bounds(def_id).iter_identity_copied().map(|(c, _)| c),
-        )
-    }
-
-    fn impl_self_is_guaranteed_unsized(self, impl_def_id: DefId) -> bool {
-        self.impl_self_is_guaranteed_unsized(impl_def_id)
-    }
-
-    fn has_target_features(self, def_id: DefId) -> bool {
-        !self.codegen_fn_attrs(def_id).target_features.is_empty()
-    }
-
-    fn require_lang_item(self, lang_item: SolverLangItem) -> DefId {
-        self.require_lang_item(solver_lang_item_to_lang_item(lang_item), DUMMY_SP)
-    }
-
-    fn require_trait_lang_item(self, lang_item: SolverTraitLangItem) -> DefId {
-        self.require_lang_item(solver_trait_lang_item_to_lang_item(lang_item), DUMMY_SP)
-    }
-
-    fn require_adt_lang_item(self, lang_item: SolverAdtLangItem) -> DefId {
-        self.require_lang_item(solver_adt_lang_item_to_lang_item(lang_item), DUMMY_SP)
-    }
-
-    fn is_lang_item(self, def_id: DefId, lang_item: SolverLangItem) -> bool {
-        self.is_lang_item(def_id, solver_lang_item_to_lang_item(lang_item))
-    }
-
-    fn is_trait_lang_item(self, def_id: DefId, lang_item: SolverTraitLangItem) -> bool {
-        self.is_lang_item(def_id, solver_trait_lang_item_to_lang_item(lang_item))
-    }
-
-    fn is_adt_lang_item(self, def_id: DefId, lang_item: SolverAdtLangItem) -> bool {
-        self.is_lang_item(def_id, solver_adt_lang_item_to_lang_item(lang_item))
-    }
-
-    fn is_default_trait(self, def_id: DefId) -> bool {
-        self.is_default_trait(def_id)
-    }
-
-    fn is_sizedness_trait(self, def_id: DefId) -> bool {
-        self.is_sizedness_trait(def_id)
-    }
-
-    fn as_lang_item(self, def_id: DefId) -> Option {
-        lang_item_to_solver_lang_item(self.lang_items().from_def_id(def_id)?)
-    }
-
-    fn as_trait_lang_item(self, def_id: DefId) -> Option {
-        lang_item_to_solver_trait_lang_item(self.lang_items().from_def_id(def_id)?)
-    }
-
-    fn as_adt_lang_item(self, def_id: DefId) -> Option {
-        lang_item_to_solver_adt_lang_item(self.lang_items().from_def_id(def_id)?)
-    }
-
-    fn associated_type_def_ids(self, def_id: DefId) -> impl IntoIterator {
-        self.associated_items(def_id)
-            .in_definition_order()
-            .filter(|assoc_item| assoc_item.is_type())
-            .map(|assoc_item| assoc_item.def_id)
-    }
-
-    // This implementation is a bit different from `TyCtxt::for_each_relevant_impl`,
-    // since we want to skip over blanket impls for non-rigid aliases, and also we
-    // only want to consider types that *actually* unify with float/int vars.
-    fn for_each_relevant_impl(
-        self,
-        trait_def_id: DefId,
-        self_ty: Ty<'tcx>,
-        mut f: impl FnMut(DefId),
-    ) {
-        let tcx = self;
-        let trait_impls = tcx.trait_impls_of(trait_def_id);
-        let mut consider_impls_for_simplified_type = |simp| {
-            if let Some(impls_for_type) = trait_impls.non_blanket_impls().get(&simp) {
-                for &impl_def_id in impls_for_type {
-                    f(impl_def_id);
-                }
-            }
-        };
-
-        match self_ty.kind() {
-            ty::Bool
-            | ty::Char
-            | ty::Int(_)
-            | ty::Uint(_)
-            | ty::Float(_)
-            | ty::Adt(_, _)
-            | ty::Foreign(_)
-            | ty::Str
-            | ty::Array(_, _)
-            | ty::Pat(_, _)
-            | ty::Slice(_)
-            | ty::RawPtr(_, _)
-            | ty::Ref(_, _, _)
-            | ty::FnDef(_, _)
-            | ty::FnPtr(..)
-            | ty::Dynamic(_, _)
-            | ty::Closure(..)
-            | ty::CoroutineClosure(..)
-            | ty::Coroutine(_, _)
-            | ty::Never
-            | ty::Tuple(_)
-            | ty::UnsafeBinder(_) => {
-                if let Some(simp) = ty::fast_reject::simplify_type(
-                    tcx,
-                    self_ty,
-                    ty::fast_reject::TreatParams::AsRigid,
-                ) {
-                    consider_impls_for_simplified_type(simp);
-                }
-            }
-
-            // HACK: For integer and float variables we have to manually look at all impls
-            // which have some integer or float as a self type.
-            ty::Infer(ty::IntVar(_)) => {
-                use ty::IntTy::*;
-                use ty::UintTy::*;
-                // This causes a compiler error if any new integer kinds are added.
-                let (I8 | I16 | I32 | I64 | I128 | Isize): ty::IntTy;
-                let (U8 | U16 | U32 | U64 | U128 | Usize): ty::UintTy;
-                let possible_integers = [
-                    // signed integers
-                    ty::SimplifiedType::Int(I8),
-                    ty::SimplifiedType::Int(I16),
-                    ty::SimplifiedType::Int(I32),
-                    ty::SimplifiedType::Int(I64),
-                    ty::SimplifiedType::Int(I128),
-                    ty::SimplifiedType::Int(Isize),
-                    // unsigned integers
-                    ty::SimplifiedType::Uint(U8),
-                    ty::SimplifiedType::Uint(U16),
-                    ty::SimplifiedType::Uint(U32),
-                    ty::SimplifiedType::Uint(U64),
-                    ty::SimplifiedType::Uint(U128),
-                    ty::SimplifiedType::Uint(Usize),
-                ];
-                for simp in possible_integers {
-                    consider_impls_for_simplified_type(simp);
-                }
-            }
-
-            ty::Infer(ty::FloatVar(_)) => {
-                // This causes a compiler error if any new float kinds are added.
-                let (ty::FloatTy::F16 | ty::FloatTy::F32 | ty::FloatTy::F64 | ty::FloatTy::F128);
-                let possible_floats = [
-                    ty::SimplifiedType::Float(ty::FloatTy::F16),
-                    ty::SimplifiedType::Float(ty::FloatTy::F32),
-                    ty::SimplifiedType::Float(ty::FloatTy::F64),
-                    ty::SimplifiedType::Float(ty::FloatTy::F128),
-                ];
-
-                for simp in possible_floats {
-                    consider_impls_for_simplified_type(simp);
-                }
-            }
-
-            // The only traits applying to aliases and placeholders are blanket impls.
-            //
-            // Impls which apply to an alias after normalization are handled by
-            // `assemble_candidates_after_normalizing_self_ty`.
-            ty::Alias(_, _) | ty::Placeholder(..) | ty::Error(_) => (),
-
-            // FIXME: These should ideally not exist as a self type. It would be nice for
-            // the builtin auto trait impls of coroutines to instead directly recurse
-            // into the witness.
-            ty::CoroutineWitness(..) => (),
-
-            // These variants should not exist as a self type.
-            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
-            | ty::Param(_)
-            | ty::Bound(_, _) => bug!("unexpected self type: {self_ty}"),
-        }
-
-        #[allow(rustc::usage_of_type_ir_traits)]
-        self.for_each_blanket_impl(trait_def_id, f)
-    }
-    fn for_each_blanket_impl(self, trait_def_id: DefId, mut f: impl FnMut(DefId)) {
-        let trait_impls = self.trait_impls_of(trait_def_id);
-        for &impl_def_id in trait_impls.blanket_impls() {
-            f(impl_def_id);
-        }
-    }
-
-    fn has_item_definition(self, def_id: DefId) -> bool {
-        self.defaultness(def_id).has_value()
-    }
-
-    fn impl_specializes(self, impl_def_id: Self::DefId, victim_def_id: Self::DefId) -> bool {
-        self.specializes((impl_def_id, victim_def_id))
-    }
-
-    fn impl_is_default(self, impl_def_id: DefId) -> bool {
-        self.defaultness(impl_def_id).is_default()
-    }
-
-    fn impl_trait_ref(self, impl_def_id: DefId) -> ty::EarlyBinder<'tcx, ty::TraitRef<'tcx>> {
-        self.impl_trait_ref(impl_def_id)
-    }
-
-    fn impl_polarity(self, impl_def_id: DefId) -> ty::ImplPolarity {
-        self.impl_polarity(impl_def_id)
-    }
-
-    fn trait_is_auto(self, trait_def_id: DefId) -> bool {
-        self.trait_is_auto(trait_def_id)
-    }
-
-    fn trait_is_coinductive(self, trait_def_id: DefId) -> bool {
-        self.trait_is_coinductive(trait_def_id)
-    }
-
-    fn trait_is_alias(self, trait_def_id: DefId) -> bool {
-        self.trait_is_alias(trait_def_id)
-    }
-
-    fn trait_is_dyn_compatible(self, trait_def_id: DefId) -> bool {
-        self.is_dyn_compatible(trait_def_id)
-    }
-
-    fn trait_is_fundamental(self, def_id: DefId) -> bool {
-        self.trait_def(def_id).is_fundamental
-    }
-
-    fn trait_is_unsafe(self, trait_def_id: Self::DefId) -> bool {
-        self.trait_def(trait_def_id).safety.is_unsafe()
-    }
-
-    fn is_impl_trait_in_trait(self, def_id: DefId) -> bool {
-        self.is_impl_trait_in_trait(def_id)
-    }
-
-    fn delay_bug(self, msg: impl ToString) -> ErrorGuaranteed {
-        self.dcx().span_delayed_bug(DUMMY_SP, msg.to_string())
-    }
-
-    fn is_general_coroutine(self, coroutine_def_id: DefId) -> bool {
-        self.is_general_coroutine(coroutine_def_id)
-    }
-
-    fn coroutine_is_async(self, coroutine_def_id: DefId) -> bool {
-        self.coroutine_is_async(coroutine_def_id)
-    }
-
-    fn coroutine_is_gen(self, coroutine_def_id: DefId) -> bool {
-        self.coroutine_is_gen(coroutine_def_id)
-    }
-
-    fn coroutine_is_async_gen(self, coroutine_def_id: DefId) -> bool {
-        self.coroutine_is_async_gen(coroutine_def_id)
-    }
-
-    type UnsizingParams = &'tcx rustc_index::bit_set::DenseBitSet;
-    fn unsizing_params_for_adt(self, adt_def_id: DefId) -> Self::UnsizingParams {
-        self.unsizing_params_for_adt(adt_def_id)
-    }
-
-    fn anonymize_bound_vars>>(
-        self,
-        binder: ty::Binder<'tcx, T>,
-    ) -> ty::Binder<'tcx, T> {
-        self.anonymize_bound_vars(binder)
-    }
-
-    fn opaque_types_defined_by(self, defining_anchor: LocalDefId) -> Self::LocalDefIds {
-        self.opaque_types_defined_by(defining_anchor)
-    }
-
-    fn opaque_types_and_coroutines_defined_by(
-        self,
-        defining_anchor: Self::LocalDefId,
-    ) -> Self::LocalDefIds {
-        let coroutines_defined_by = self
-            .nested_bodies_within(defining_anchor)
-            .iter()
-            .filter(|def_id| self.is_coroutine(def_id.to_def_id()));
-        self.mk_local_def_ids_from_iter(
-            self.opaque_types_defined_by(defining_anchor).iter().chain(coroutines_defined_by),
-        )
-    }
-
-    type Probe = &'tcx inspect::Probe>;
-    fn mk_probe(self, probe: inspect::Probe) -> &'tcx inspect::Probe> {
-        self.arena.alloc(probe)
-    }
-    fn evaluate_root_goal_for_proof_tree_raw(
-        self,
-        canonical_goal: CanonicalInput<'tcx>,
-    ) -> (QueryResult<'tcx>, &'tcx inspect::Probe>) {
-        self.evaluate_root_goal_for_proof_tree_raw(canonical_goal)
-    }
-
-    fn item_name(self, id: DefId) -> Symbol {
-        let id = id.into_query_param();
-        self.opt_item_name(id).unwrap_or_else(|| {
-            bug!("item_name: no name for {:?}", self.def_path(id));
-        })
-    }
-}
-
-macro_rules! bidirectional_lang_item_map {
-    (
-        $solver_ty:ident, $to_solver:ident, $from_solver:ident;
-        $($name:ident),+ $(,)?
-    ) => {
-        fn $from_solver(lang_item: $solver_ty) -> LangItem {
-            match lang_item {
-                $($solver_ty::$name => LangItem::$name,)+
-            }
-        }
-
-        fn $to_solver(lang_item: LangItem) -> Option<$solver_ty> {
-            Some(match lang_item {
-                $(LangItem::$name => $solver_ty::$name,)+
-                _ => return None,
-            })
-        }
-    }
-}
-
-bidirectional_lang_item_map! {
-    SolverLangItem, lang_item_to_solver_lang_item, solver_lang_item_to_lang_item;
-
-// tidy-alphabetical-start
-    AsyncFnKindUpvars,
-    AsyncFnOnceOutput,
-    CallOnceFuture,
-    CallRefFuture,
-    CoroutineReturn,
-    CoroutineYield,
-    DynMetadata,
-    FutureOutput,
-    Metadata,
-// tidy-alphabetical-end
-}
-
-bidirectional_lang_item_map! {
-    SolverAdtLangItem, lang_item_to_solver_adt_lang_item, solver_adt_lang_item_to_lang_item;
-
-// tidy-alphabetical-start
-    Option,
-    Poll,
-// tidy-alphabetical-end
-}
-
-bidirectional_lang_item_map! {
-    SolverTraitLangItem, lang_item_to_solver_trait_lang_item, solver_trait_lang_item_to_lang_item;
-
-// tidy-alphabetical-start
-    AsyncFn,
-    AsyncFnKindHelper,
-    AsyncFnMut,
-    AsyncFnOnce,
-    AsyncFnOnceOutput,
-    AsyncIterator,
-    BikeshedGuaranteedNoDrop,
-    Clone,
-    Copy,
-    Coroutine,
-    Destruct,
-    DiscriminantKind,
-    Drop,
-    Fn,
-    FnMut,
-    FnOnce,
-    FnPtrTrait,
-    FusedIterator,
-    Future,
-    Iterator,
-    MetaSized,
-    PointeeSized,
-    PointeeTrait,
-    Sized,
-    TransmuteTrait,
-    TrivialClone,
-    Tuple,
-    Unpin,
-    Unsize,
-// tidy-alphabetical-end
-}
-
 impl<'tcx> rustc_type_ir::inherent::DefId> for DefId {
     fn is_local(self) -> bool {
         self.is_local()
diff --git a/compiler/rustc_middle/src/ty/context/impl_interner.rs b/compiler/rustc_middle/src/ty/context/impl_interner.rs
new file mode 100644
index 000000000000..c449f29634e2
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/context/impl_interner.rs
@@ -0,0 +1,804 @@
+//! Implementation of [`rustc_type_ir::Interner`] for [`TyCtxt`].
+
+use std::fmt;
+
+use rustc_abi::ExternAbi;
+use rustc_data_structures::debug_assert_matches;
+use rustc_errors::ErrorGuaranteed;
+use rustc_hir::def::{CtorKind, CtorOf, DefKind};
+use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::lang_items::LangItem;
+use rustc_hir::{self as hir};
+use rustc_query_system::dep_graph::DepNodeIndex;
+use rustc_span::{DUMMY_SP, Span, Symbol};
+use rustc_type_ir::lang_items::{SolverAdtLangItem, SolverLangItem, SolverTraitLangItem};
+use rustc_type_ir::{CollectAndApply, Interner, TypeFoldable, search_graph};
+
+use crate::infer::canonical::CanonicalVarKinds;
+use crate::query::IntoQueryParam;
+use crate::traits::cache::WithDepNode;
+use crate::traits::solve::{
+    self, CanonicalInput, ExternalConstraints, ExternalConstraintsData, QueryResult, inspect,
+};
+use crate::ty::{
+    self, Clause, Const, List, ParamTy, Pattern, PolyExistentialPredicate, Predicate, Region, Ty,
+    TyCtxt,
+};
+
+#[allow(rustc::usage_of_ty_tykind)]
+impl<'tcx> Interner for TyCtxt<'tcx> {
+    fn next_trait_solver_globally(self) -> bool {
+        self.next_trait_solver_globally()
+    }
+
+    type DefId = DefId;
+    type LocalDefId = LocalDefId;
+    type TraitId = DefId;
+    type ForeignId = DefId;
+    type FunctionId = DefId;
+    type ClosureId = DefId;
+    type CoroutineClosureId = DefId;
+    type CoroutineId = DefId;
+    type AdtId = DefId;
+    type ImplId = DefId;
+    type UnevaluatedConstId = DefId;
+    type Span = Span;
+
+    type GenericArgs = ty::GenericArgsRef<'tcx>;
+
+    type GenericArgsSlice = &'tcx [ty::GenericArg<'tcx>];
+    type GenericArg = ty::GenericArg<'tcx>;
+    type Term = ty::Term<'tcx>;
+    type BoundVarKinds = &'tcx List>;
+
+    type PredefinedOpaques = solve::PredefinedOpaques<'tcx>;
+
+    fn mk_predefined_opaques_in_body(
+        self,
+        data: &[(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)],
+    ) -> Self::PredefinedOpaques {
+        self.mk_predefined_opaques_in_body(data)
+    }
+    type LocalDefIds = &'tcx ty::List;
+    type CanonicalVarKinds = CanonicalVarKinds<'tcx>;
+    fn mk_canonical_var_kinds(
+        self,
+        kinds: &[ty::CanonicalVarKind],
+    ) -> Self::CanonicalVarKinds {
+        self.mk_canonical_var_kinds(kinds)
+    }
+
+    type ExternalConstraints = ExternalConstraints<'tcx>;
+    fn mk_external_constraints(
+        self,
+        data: ExternalConstraintsData,
+    ) -> ExternalConstraints<'tcx> {
+        self.mk_external_constraints(data)
+    }
+    type DepNodeIndex = DepNodeIndex;
+    fn with_cached_task(self, task: impl FnOnce() -> T) -> (T, DepNodeIndex) {
+        self.dep_graph.with_anon_task(self, crate::dep_graph::dep_kinds::TraitSelect, task)
+    }
+    type Ty = Ty<'tcx>;
+    type Tys = &'tcx List>;
+
+    type FnInputTys = &'tcx [Ty<'tcx>];
+    type ParamTy = ParamTy;
+    type Symbol = Symbol;
+
+    type ErrorGuaranteed = ErrorGuaranteed;
+    type BoundExistentialPredicates = &'tcx List>;
+
+    type AllocId = crate::mir::interpret::AllocId;
+    type Pat = Pattern<'tcx>;
+    type PatList = &'tcx List>;
+    type Safety = hir::Safety;
+    type Abi = ExternAbi;
+    type Const = ty::Const<'tcx>;
+
+    type ParamConst = ty::ParamConst;
+    type ValueConst = ty::Value<'tcx>;
+    type ExprConst = ty::Expr<'tcx>;
+    type ValTree = ty::ValTree<'tcx>;
+    type ScalarInt = ty::ScalarInt;
+
+    type Region = Region<'tcx>;
+    type EarlyParamRegion = ty::EarlyParamRegion;
+    type LateParamRegion = ty::LateParamRegion;
+
+    type RegionAssumptions = &'tcx ty::List>;
+
+    type ParamEnv = ty::ParamEnv<'tcx>;
+    type Predicate = Predicate<'tcx>;
+
+    type Clause = Clause<'tcx>;
+    type Clauses = ty::Clauses<'tcx>;
+
+    type Tracked = WithDepNode;
+    fn mk_tracked(
+        self,
+        data: T,
+        dep_node: DepNodeIndex,
+    ) -> Self::Tracked {
+        WithDepNode::new(dep_node, data)
+    }
+    fn get_tracked(self, tracked: &Self::Tracked) -> T {
+        tracked.get(self)
+    }
+
+    fn with_global_cache(self, f: impl FnOnce(&mut search_graph::GlobalCache) -> R) -> R {
+        f(&mut *self.new_solver_evaluation_cache.lock())
+    }
+
+    fn canonical_param_env_cache_get_or_insert(
+        self,
+        param_env: ty::ParamEnv<'tcx>,
+        f: impl FnOnce() -> ty::CanonicalParamEnvCacheEntry,
+        from_entry: impl FnOnce(&ty::CanonicalParamEnvCacheEntry) -> R,
+    ) -> R {
+        let mut cache = self.new_solver_canonical_param_env_cache.lock();
+        let entry = cache.entry(param_env).or_insert_with(f);
+        from_entry(entry)
+    }
+
+    fn assert_evaluation_is_concurrent(&self) {
+        // Turns out, the assumption for this function isn't perfect.
+        // See trait-system-refactor-initiative#234.
+    }
+
+    fn expand_abstract_consts>>(self, t: T) -> T {
+        self.expand_abstract_consts(t)
+    }
+
+    type GenericsOf = &'tcx ty::Generics;
+
+    fn generics_of(self, def_id: DefId) -> &'tcx ty::Generics {
+        self.generics_of(def_id)
+    }
+
+    type VariancesOf = &'tcx [ty::Variance];
+
+    fn variances_of(self, def_id: DefId) -> Self::VariancesOf {
+        self.variances_of(def_id)
+    }
+
+    fn opt_alias_variances(
+        self,
+        kind: impl Into,
+        def_id: DefId,
+    ) -> Option<&'tcx [ty::Variance]> {
+        self.opt_alias_variances(kind, def_id)
+    }
+
+    fn type_of(self, def_id: DefId) -> ty::EarlyBinder<'tcx, Ty<'tcx>> {
+        self.type_of(def_id)
+    }
+    fn type_of_opaque_hir_typeck(self, def_id: LocalDefId) -> ty::EarlyBinder<'tcx, Ty<'tcx>> {
+        self.type_of_opaque_hir_typeck(def_id)
+    }
+    fn const_of_item(self, def_id: DefId) -> ty::EarlyBinder<'tcx, Const<'tcx>> {
+        self.const_of_item(def_id)
+    }
+    fn anon_const_kind(self, def_id: DefId) -> ty::AnonConstKind {
+        self.anon_const_kind(def_id)
+    }
+
+    type AdtDef = ty::AdtDef<'tcx>;
+    fn adt_def(self, adt_def_id: DefId) -> Self::AdtDef {
+        self.adt_def(adt_def_id)
+    }
+
+    fn alias_ty_kind(self, alias: ty::AliasTy<'tcx>) -> ty::AliasTyKind {
+        match self.def_kind(alias.def_id) {
+            DefKind::AssocTy => {
+                if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(alias.def_id))
+                {
+                    ty::Inherent
+                } else {
+                    ty::Projection
+                }
+            }
+            DefKind::OpaqueTy => ty::Opaque,
+            DefKind::TyAlias => ty::Free,
+            kind => bug!("unexpected DefKind in AliasTy: {kind:?}"),
+        }
+    }
+
+    fn alias_term_kind(self, alias: ty::AliasTerm<'tcx>) -> ty::AliasTermKind {
+        match self.def_kind(alias.def_id) {
+            DefKind::AssocTy => {
+                if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(alias.def_id))
+                {
+                    ty::AliasTermKind::InherentTy
+                } else {
+                    ty::AliasTermKind::ProjectionTy
+                }
+            }
+            DefKind::AssocConst => {
+                if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(alias.def_id))
+                {
+                    ty::AliasTermKind::InherentConst
+                } else {
+                    ty::AliasTermKind::ProjectionConst
+                }
+            }
+            DefKind::OpaqueTy => ty::AliasTermKind::OpaqueTy,
+            DefKind::TyAlias => ty::AliasTermKind::FreeTy,
+            DefKind::Const => ty::AliasTermKind::FreeConst,
+            DefKind::AnonConst | DefKind::Ctor(_, CtorKind::Const) => {
+                ty::AliasTermKind::UnevaluatedConst
+            }
+            kind => bug!("unexpected DefKind in AliasTy: {kind:?}"),
+        }
+    }
+
+    fn trait_ref_and_own_args_for_alias(
+        self,
+        def_id: DefId,
+        args: ty::GenericArgsRef<'tcx>,
+    ) -> (ty::TraitRef<'tcx>, &'tcx [ty::GenericArg<'tcx>]) {
+        debug_assert_matches!(self.def_kind(def_id), DefKind::AssocTy | DefKind::AssocConst);
+        let trait_def_id = self.parent(def_id);
+        debug_assert_matches!(self.def_kind(trait_def_id), DefKind::Trait);
+        let trait_ref = ty::TraitRef::from_assoc(self, trait_def_id, args);
+        (trait_ref, &args[trait_ref.args.len()..])
+    }
+
+    fn mk_args(self, args: &[Self::GenericArg]) -> ty::GenericArgsRef<'tcx> {
+        self.mk_args(args)
+    }
+
+    fn mk_args_from_iter(self, args: I) -> T::Output
+    where
+        I: Iterator,
+        T: CollectAndApply>,
+    {
+        self.mk_args_from_iter(args)
+    }
+
+    fn check_args_compatible(self, def_id: DefId, args: ty::GenericArgsRef<'tcx>) -> bool {
+        self.check_args_compatible(def_id, args)
+    }
+
+    fn debug_assert_args_compatible(self, def_id: DefId, args: ty::GenericArgsRef<'tcx>) {
+        self.debug_assert_args_compatible(def_id, args);
+    }
+
+    /// Assert that the args from an `ExistentialTraitRef` or `ExistentialProjection`
+    /// are compatible with the `DefId`. Since we're missing a `Self` type, stick on
+    /// a dummy self type and forward to `debug_assert_args_compatible`.
+    fn debug_assert_existential_args_compatible(
+        self,
+        def_id: Self::DefId,
+        args: Self::GenericArgs,
+    ) {
+        // FIXME: We could perhaps add a `skip: usize` to `debug_assert_args_compatible`
+        // to avoid needing to reintern the set of args...
+        if cfg!(debug_assertions) {
+            self.debug_assert_args_compatible(
+                def_id,
+                self.mk_args_from_iter(
+                    [self.types.trait_object_dummy_self.into()].into_iter().chain(args.iter()),
+                ),
+            );
+        }
+    }
+
+    fn mk_type_list_from_iter(self, args: I) -> T::Output
+    where
+        I: Iterator,
+        T: CollectAndApply, &'tcx List>>,
+    {
+        self.mk_type_list_from_iter(args)
+    }
+
+    fn parent(self, def_id: DefId) -> DefId {
+        self.parent(def_id)
+    }
+
+    fn recursion_limit(self) -> usize {
+        self.recursion_limit().0
+    }
+
+    type Features = &'tcx rustc_feature::Features;
+
+    fn features(self) -> Self::Features {
+        self.features()
+    }
+
+    fn coroutine_hidden_types(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes>>> {
+        self.coroutine_hidden_types(def_id)
+    }
+
+    fn fn_sig(self, def_id: DefId) -> ty::EarlyBinder<'tcx, ty::PolyFnSig<'tcx>> {
+        self.fn_sig(def_id)
+    }
+
+    fn coroutine_movability(self, def_id: DefId) -> rustc_ast::Movability {
+        self.coroutine_movability(def_id)
+    }
+
+    fn coroutine_for_closure(self, def_id: DefId) -> DefId {
+        self.coroutine_for_closure(def_id)
+    }
+
+    fn generics_require_sized_self(self, def_id: DefId) -> bool {
+        self.generics_require_sized_self(def_id)
+    }
+
+    fn item_bounds(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
+        self.item_bounds(def_id).map_bound(IntoIterator::into_iter)
+    }
+
+    fn item_self_bounds(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
+        self.item_self_bounds(def_id).map_bound(IntoIterator::into_iter)
+    }
+
+    fn item_non_self_bounds(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
+        self.item_non_self_bounds(def_id).map_bound(IntoIterator::into_iter)
+    }
+
+    fn predicates_of(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
+        ty::EarlyBinder::bind(
+            self.predicates_of(def_id).instantiate_identity(self).predicates.into_iter(),
+        )
+    }
+
+    fn own_predicates_of(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
+        ty::EarlyBinder::bind(
+            self.predicates_of(def_id).instantiate_own_identity().map(|(clause, _)| clause),
+        )
+    }
+
+    fn explicit_super_predicates_of(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator, Span)>> {
+        self.explicit_super_predicates_of(def_id).map_bound(|preds| preds.into_iter().copied())
+    }
+
+    fn explicit_implied_predicates_of(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator, Span)>> {
+        self.explicit_implied_predicates_of(def_id).map_bound(|preds| preds.into_iter().copied())
+    }
+
+    fn impl_super_outlives(
+        self,
+        impl_def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>> {
+        self.impl_super_outlives(impl_def_id)
+    }
+
+    fn impl_is_const(self, def_id: DefId) -> bool {
+        debug_assert_matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true });
+        self.is_conditionally_const(def_id)
+    }
+
+    fn fn_is_const(self, def_id: DefId) -> bool {
+        debug_assert_matches!(
+            self.def_kind(def_id),
+            DefKind::Fn | DefKind::AssocFn | DefKind::Ctor(CtorOf::Struct, CtorKind::Fn)
+        );
+        self.is_conditionally_const(def_id)
+    }
+
+    fn alias_has_const_conditions(self, def_id: DefId) -> bool {
+        debug_assert_matches!(self.def_kind(def_id), DefKind::AssocTy | DefKind::OpaqueTy);
+        self.is_conditionally_const(def_id)
+    }
+
+    fn const_conditions(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>>> {
+        ty::EarlyBinder::bind(
+            self.const_conditions(def_id).instantiate_identity(self).into_iter().map(|(c, _)| c),
+        )
+    }
+
+    fn explicit_implied_const_bounds(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator>>> {
+        ty::EarlyBinder::bind(
+            self.explicit_implied_const_bounds(def_id).iter_identity_copied().map(|(c, _)| c),
+        )
+    }
+
+    fn impl_self_is_guaranteed_unsized(self, impl_def_id: DefId) -> bool {
+        self.impl_self_is_guaranteed_unsized(impl_def_id)
+    }
+
+    fn has_target_features(self, def_id: DefId) -> bool {
+        !self.codegen_fn_attrs(def_id).target_features.is_empty()
+    }
+
+    fn require_lang_item(self, lang_item: SolverLangItem) -> DefId {
+        self.require_lang_item(solver_lang_item_to_lang_item(lang_item), DUMMY_SP)
+    }
+
+    fn require_trait_lang_item(self, lang_item: SolverTraitLangItem) -> DefId {
+        self.require_lang_item(solver_trait_lang_item_to_lang_item(lang_item), DUMMY_SP)
+    }
+
+    fn require_adt_lang_item(self, lang_item: SolverAdtLangItem) -> DefId {
+        self.require_lang_item(solver_adt_lang_item_to_lang_item(lang_item), DUMMY_SP)
+    }
+
+    fn is_lang_item(self, def_id: DefId, lang_item: SolverLangItem) -> bool {
+        self.is_lang_item(def_id, solver_lang_item_to_lang_item(lang_item))
+    }
+
+    fn is_trait_lang_item(self, def_id: DefId, lang_item: SolverTraitLangItem) -> bool {
+        self.is_lang_item(def_id, solver_trait_lang_item_to_lang_item(lang_item))
+    }
+
+    fn is_adt_lang_item(self, def_id: DefId, lang_item: SolverAdtLangItem) -> bool {
+        self.is_lang_item(def_id, solver_adt_lang_item_to_lang_item(lang_item))
+    }
+
+    fn is_default_trait(self, def_id: DefId) -> bool {
+        self.is_default_trait(def_id)
+    }
+
+    fn is_sizedness_trait(self, def_id: DefId) -> bool {
+        self.is_sizedness_trait(def_id)
+    }
+
+    fn as_lang_item(self, def_id: DefId) -> Option {
+        lang_item_to_solver_lang_item(self.lang_items().from_def_id(def_id)?)
+    }
+
+    fn as_trait_lang_item(self, def_id: DefId) -> Option {
+        lang_item_to_solver_trait_lang_item(self.lang_items().from_def_id(def_id)?)
+    }
+
+    fn as_adt_lang_item(self, def_id: DefId) -> Option {
+        lang_item_to_solver_adt_lang_item(self.lang_items().from_def_id(def_id)?)
+    }
+
+    fn associated_type_def_ids(self, def_id: DefId) -> impl IntoIterator {
+        self.associated_items(def_id)
+            .in_definition_order()
+            .filter(|assoc_item| assoc_item.is_type())
+            .map(|assoc_item| assoc_item.def_id)
+    }
+
+    // This implementation is a bit different from `TyCtxt::for_each_relevant_impl`,
+    // since we want to skip over blanket impls for non-rigid aliases, and also we
+    // only want to consider types that *actually* unify with float/int vars.
+    fn for_each_relevant_impl(
+        self,
+        trait_def_id: DefId,
+        self_ty: Ty<'tcx>,
+        mut f: impl FnMut(DefId),
+    ) {
+        let tcx = self;
+        let trait_impls = tcx.trait_impls_of(trait_def_id);
+        let mut consider_impls_for_simplified_type = |simp| {
+            if let Some(impls_for_type) = trait_impls.non_blanket_impls().get(&simp) {
+                for &impl_def_id in impls_for_type {
+                    f(impl_def_id);
+                }
+            }
+        };
+
+        match self_ty.kind() {
+            ty::Bool
+            | ty::Char
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Adt(_, _)
+            | ty::Foreign(_)
+            | ty::Str
+            | ty::Array(_, _)
+            | ty::Pat(_, _)
+            | ty::Slice(_)
+            | ty::RawPtr(_, _)
+            | ty::Ref(_, _, _)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(..)
+            | ty::Dynamic(_, _)
+            | ty::Closure(..)
+            | ty::CoroutineClosure(..)
+            | ty::Coroutine(_, _)
+            | ty::Never
+            | ty::Tuple(_)
+            | ty::UnsafeBinder(_) => {
+                if let Some(simp) = ty::fast_reject::simplify_type(
+                    tcx,
+                    self_ty,
+                    ty::fast_reject::TreatParams::AsRigid,
+                ) {
+                    consider_impls_for_simplified_type(simp);
+                }
+            }
+
+            // HACK: For integer and float variables we have to manually look at all impls
+            // which have some integer or float as a self type.
+            ty::Infer(ty::IntVar(_)) => {
+                use ty::IntTy::*;
+                use ty::UintTy::*;
+                // This causes a compiler error if any new integer kinds are added.
+                let (I8 | I16 | I32 | I64 | I128 | Isize): ty::IntTy;
+                let (U8 | U16 | U32 | U64 | U128 | Usize): ty::UintTy;
+                let possible_integers = [
+                    // signed integers
+                    ty::SimplifiedType::Int(I8),
+                    ty::SimplifiedType::Int(I16),
+                    ty::SimplifiedType::Int(I32),
+                    ty::SimplifiedType::Int(I64),
+                    ty::SimplifiedType::Int(I128),
+                    ty::SimplifiedType::Int(Isize),
+                    // unsigned integers
+                    ty::SimplifiedType::Uint(U8),
+                    ty::SimplifiedType::Uint(U16),
+                    ty::SimplifiedType::Uint(U32),
+                    ty::SimplifiedType::Uint(U64),
+                    ty::SimplifiedType::Uint(U128),
+                    ty::SimplifiedType::Uint(Usize),
+                ];
+                for simp in possible_integers {
+                    consider_impls_for_simplified_type(simp);
+                }
+            }
+
+            ty::Infer(ty::FloatVar(_)) => {
+                // This causes a compiler error if any new float kinds are added.
+                let (ty::FloatTy::F16 | ty::FloatTy::F32 | ty::FloatTy::F64 | ty::FloatTy::F128);
+                let possible_floats = [
+                    ty::SimplifiedType::Float(ty::FloatTy::F16),
+                    ty::SimplifiedType::Float(ty::FloatTy::F32),
+                    ty::SimplifiedType::Float(ty::FloatTy::F64),
+                    ty::SimplifiedType::Float(ty::FloatTy::F128),
+                ];
+
+                for simp in possible_floats {
+                    consider_impls_for_simplified_type(simp);
+                }
+            }
+
+            // The only traits applying to aliases and placeholders are blanket impls.
+            //
+            // Impls which apply to an alias after normalization are handled by
+            // `assemble_candidates_after_normalizing_self_ty`.
+            ty::Alias(_, _) | ty::Placeholder(..) | ty::Error(_) => (),
+
+            // FIXME: These should ideally not exist as a self type. It would be nice for
+            // the builtin auto trait impls of coroutines to instead directly recurse
+            // into the witness.
+            ty::CoroutineWitness(..) => (),
+
+            // These variants should not exist as a self type.
+            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
+            | ty::Param(_)
+            | ty::Bound(_, _) => bug!("unexpected self type: {self_ty}"),
+        }
+
+        #[allow(rustc::usage_of_type_ir_traits)]
+        self.for_each_blanket_impl(trait_def_id, f)
+    }
+    fn for_each_blanket_impl(self, trait_def_id: DefId, mut f: impl FnMut(DefId)) {
+        let trait_impls = self.trait_impls_of(trait_def_id);
+        for &impl_def_id in trait_impls.blanket_impls() {
+            f(impl_def_id);
+        }
+    }
+
+    fn has_item_definition(self, def_id: DefId) -> bool {
+        self.defaultness(def_id).has_value()
+    }
+
+    fn impl_specializes(self, impl_def_id: Self::DefId, victim_def_id: Self::DefId) -> bool {
+        self.specializes((impl_def_id, victim_def_id))
+    }
+
+    fn impl_is_default(self, impl_def_id: DefId) -> bool {
+        self.defaultness(impl_def_id).is_default()
+    }
+
+    fn impl_trait_ref(self, impl_def_id: DefId) -> ty::EarlyBinder<'tcx, ty::TraitRef<'tcx>> {
+        self.impl_trait_ref(impl_def_id)
+    }
+
+    fn impl_polarity(self, impl_def_id: DefId) -> ty::ImplPolarity {
+        self.impl_polarity(impl_def_id)
+    }
+
+    fn trait_is_auto(self, trait_def_id: DefId) -> bool {
+        self.trait_is_auto(trait_def_id)
+    }
+
+    fn trait_is_coinductive(self, trait_def_id: DefId) -> bool {
+        self.trait_is_coinductive(trait_def_id)
+    }
+
+    fn trait_is_alias(self, trait_def_id: DefId) -> bool {
+        self.trait_is_alias(trait_def_id)
+    }
+
+    fn trait_is_dyn_compatible(self, trait_def_id: DefId) -> bool {
+        self.is_dyn_compatible(trait_def_id)
+    }
+
+    fn trait_is_fundamental(self, def_id: DefId) -> bool {
+        self.trait_def(def_id).is_fundamental
+    }
+
+    fn trait_is_unsafe(self, trait_def_id: Self::DefId) -> bool {
+        self.trait_def(trait_def_id).safety.is_unsafe()
+    }
+
+    fn is_impl_trait_in_trait(self, def_id: DefId) -> bool {
+        self.is_impl_trait_in_trait(def_id)
+    }
+
+    fn delay_bug(self, msg: impl ToString) -> ErrorGuaranteed {
+        self.dcx().span_delayed_bug(DUMMY_SP, msg.to_string())
+    }
+
+    fn is_general_coroutine(self, coroutine_def_id: DefId) -> bool {
+        self.is_general_coroutine(coroutine_def_id)
+    }
+
+    fn coroutine_is_async(self, coroutine_def_id: DefId) -> bool {
+        self.coroutine_is_async(coroutine_def_id)
+    }
+
+    fn coroutine_is_gen(self, coroutine_def_id: DefId) -> bool {
+        self.coroutine_is_gen(coroutine_def_id)
+    }
+
+    fn coroutine_is_async_gen(self, coroutine_def_id: DefId) -> bool {
+        self.coroutine_is_async_gen(coroutine_def_id)
+    }
+
+    type UnsizingParams = &'tcx rustc_index::bit_set::DenseBitSet;
+    fn unsizing_params_for_adt(self, adt_def_id: DefId) -> Self::UnsizingParams {
+        self.unsizing_params_for_adt(adt_def_id)
+    }
+
+    fn anonymize_bound_vars>>(
+        self,
+        binder: ty::Binder<'tcx, T>,
+    ) -> ty::Binder<'tcx, T> {
+        self.anonymize_bound_vars(binder)
+    }
+
+    fn opaque_types_defined_by(self, defining_anchor: LocalDefId) -> Self::LocalDefIds {
+        self.opaque_types_defined_by(defining_anchor)
+    }
+
+    fn opaque_types_and_coroutines_defined_by(
+        self,
+        defining_anchor: Self::LocalDefId,
+    ) -> Self::LocalDefIds {
+        let coroutines_defined_by = self
+            .nested_bodies_within(defining_anchor)
+            .iter()
+            .filter(|def_id| self.is_coroutine(def_id.to_def_id()));
+        self.mk_local_def_ids_from_iter(
+            self.opaque_types_defined_by(defining_anchor).iter().chain(coroutines_defined_by),
+        )
+    }
+
+    type Probe = &'tcx inspect::Probe>;
+    fn mk_probe(self, probe: inspect::Probe) -> &'tcx inspect::Probe> {
+        self.arena.alloc(probe)
+    }
+    fn evaluate_root_goal_for_proof_tree_raw(
+        self,
+        canonical_goal: CanonicalInput<'tcx>,
+    ) -> (QueryResult<'tcx>, &'tcx inspect::Probe>) {
+        self.evaluate_root_goal_for_proof_tree_raw(canonical_goal)
+    }
+
+    fn item_name(self, id: DefId) -> Symbol {
+        let id = id.into_query_param();
+        self.opt_item_name(id).unwrap_or_else(|| {
+            bug!("item_name: no name for {:?}", self.def_path(id));
+        })
+    }
+}
+
+macro_rules! bidirectional_lang_item_map {
+    (
+        $solver_ty:ident, $to_solver:ident, $from_solver:ident;
+        $($name:ident),+ $(,)?
+    ) => {
+        fn $from_solver(lang_item: $solver_ty) -> LangItem {
+            match lang_item {
+                $($solver_ty::$name => LangItem::$name,)+
+            }
+        }
+
+        fn $to_solver(lang_item: LangItem) -> Option<$solver_ty> {
+            Some(match lang_item {
+                $(LangItem::$name => $solver_ty::$name,)+
+                _ => return None,
+            })
+        }
+    }
+}
+
+bidirectional_lang_item_map! {
+    SolverLangItem, lang_item_to_solver_lang_item, solver_lang_item_to_lang_item;
+
+// tidy-alphabetical-start
+    AsyncFnKindUpvars,
+    AsyncFnOnceOutput,
+    CallOnceFuture,
+    CallRefFuture,
+    CoroutineReturn,
+    CoroutineYield,
+    DynMetadata,
+    FutureOutput,
+    Metadata,
+// tidy-alphabetical-end
+}
+
+bidirectional_lang_item_map! {
+    SolverAdtLangItem, lang_item_to_solver_adt_lang_item, solver_adt_lang_item_to_lang_item;
+
+// tidy-alphabetical-start
+    Option,
+    Poll,
+// tidy-alphabetical-end
+}
+
+bidirectional_lang_item_map! {
+    SolverTraitLangItem, lang_item_to_solver_trait_lang_item, solver_trait_lang_item_to_lang_item;
+
+// tidy-alphabetical-start
+    AsyncFn,
+    AsyncFnKindHelper,
+    AsyncFnMut,
+    AsyncFnOnce,
+    AsyncFnOnceOutput,
+    AsyncIterator,
+    BikeshedGuaranteedNoDrop,
+    Clone,
+    Copy,
+    Coroutine,
+    Destruct,
+    DiscriminantKind,
+    Drop,
+    Fn,
+    FnMut,
+    FnOnce,
+    FnPtrTrait,
+    FusedIterator,
+    Future,
+    Iterator,
+    MetaSized,
+    PointeeSized,
+    PointeeTrait,
+    Sized,
+    TransmuteTrait,
+    TrivialClone,
+    Tuple,
+    Unpin,
+    Unsize,
+// tidy-alphabetical-end
+}

From b7bfb768d0f9da6a90c32ad596fd17f57935ae87 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Sun, 8 Feb 2026 13:40:20 +1100
Subject: [PATCH 1521/1843] Indicate that `bidirectional_lang_item_map!`
 declares functions

Requiring `fn` in the macro syntax makes it a little more obvious that the
macro declares functions with those names, and makes them easier to grep for.
---
 compiler/rustc_middle/src/ty/context/impl_interner.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/context/impl_interner.rs b/compiler/rustc_middle/src/ty/context/impl_interner.rs
index c449f29634e2..5a15d132048d 100644
--- a/compiler/rustc_middle/src/ty/context/impl_interner.rs
+++ b/compiler/rustc_middle/src/ty/context/impl_interner.rs
@@ -722,9 +722,11 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     }
 }
 
+/// Defines trivial conversion functions between the main [`LangItem`] enum,
+/// and some other lang-item enum that is a subset of it.
 macro_rules! bidirectional_lang_item_map {
     (
-        $solver_ty:ident, $to_solver:ident, $from_solver:ident;
+        $solver_ty:ident, fn $to_solver:ident, fn $from_solver:ident;
         $($name:ident),+ $(,)?
     ) => {
         fn $from_solver(lang_item: $solver_ty) -> LangItem {
@@ -743,7 +745,7 @@ macro_rules! bidirectional_lang_item_map {
 }
 
 bidirectional_lang_item_map! {
-    SolverLangItem, lang_item_to_solver_lang_item, solver_lang_item_to_lang_item;
+    SolverLangItem, fn lang_item_to_solver_lang_item, fn solver_lang_item_to_lang_item;
 
 // tidy-alphabetical-start
     AsyncFnKindUpvars,
@@ -759,7 +761,7 @@ bidirectional_lang_item_map! {
 }
 
 bidirectional_lang_item_map! {
-    SolverAdtLangItem, lang_item_to_solver_adt_lang_item, solver_adt_lang_item_to_lang_item;
+    SolverAdtLangItem, fn lang_item_to_solver_adt_lang_item, fn solver_adt_lang_item_to_lang_item;
 
 // tidy-alphabetical-start
     Option,
@@ -768,7 +770,7 @@ bidirectional_lang_item_map! {
 }
 
 bidirectional_lang_item_map! {
-    SolverTraitLangItem, lang_item_to_solver_trait_lang_item, solver_trait_lang_item_to_lang_item;
+    SolverTraitLangItem, fn lang_item_to_solver_trait_lang_item, fn solver_trait_lang_item_to_lang_item;
 
 // tidy-alphabetical-start
     AsyncFn,

From e1ed3bf01d602fefe6797fb56b52de5161fe64c4 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Tue, 10 Feb 2026 12:22:40 +1100
Subject: [PATCH 1522/1843] Remove out-of-date comment.

Keys and values now must be `Copy` due to erasing.
---
 compiler/rustc_query_system/src/query/caches.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index eea1c5872290..c1f5e5b67085 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -21,9 +21,6 @@ pub trait QueryCacheKey = Hash + Eq + Copy + Debug + for<'a> HashStable
Date: Tue, 10 Feb 2026 10:16:56 +1100
Subject: [PATCH 1523/1843] Derive `Clone` for `QueryLatch`.

The derived version is equivalent to the hand-written version.
---
 compiler/rustc_query_system/src/query/job.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index bf7c480b05dd..c77ae38ba96f 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -181,17 +181,11 @@ struct QueryLatchInfo<'tcx> {
     waiters: Vec>>,
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct QueryLatch<'tcx> {
     info: Arc>>,
 }
 
-impl<'tcx> Clone for QueryLatch<'tcx> {
-    fn clone(&self) -> Self {
-        Self { info: Arc::clone(&self.info) }
-    }
-}
-
 impl<'tcx> QueryLatch<'tcx> {
     fn new() -> Self {
         QueryLatch {

From 4dc82e99efad0d0fbb91218b82394a48fcd0ed87 Mon Sep 17 00:00:00 2001
From: Zalathar 
Date: Thu, 5 Feb 2026 18:21:27 +1100
Subject: [PATCH 1524/1843] Move the needs-drop check for `arena_cache` queries
 out of macro code

---
 compiler/rustc_middle/src/queries.rs          |  1 -
 .../rustc_middle/src/query/arena_cached.rs    | 33 +++++++++-----
 compiler/rustc_middle/src/query/plumbing.rs   | 44 ++++++++-----------
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/compiler/rustc_middle/src/queries.rs b/compiler/rustc_middle/src/queries.rs
index de2298914438..2fdc44ecd269 100644
--- a/compiler/rustc_middle/src/queries.rs
+++ b/compiler/rustc_middle/src/queries.rs
@@ -63,7 +63,6 @@
 #![allow(unused_parens)]
 
 use std::ffi::OsStr;
-use std::mem;
 use std::path::PathBuf;
 use std::sync::Arc;
 
diff --git a/compiler/rustc_middle/src/query/arena_cached.rs b/compiler/rustc_middle/src/query/arena_cached.rs
index ec6e466ff688..7c7ad1262260 100644
--- a/compiler/rustc_middle/src/query/arena_cached.rs
+++ b/compiler/rustc_middle/src/query/arena_cached.rs
@@ -1,3 +1,9 @@
+use std::mem;
+
+use rustc_arena::TypedArena;
+
+use crate::ty::TyCtxt;
+
 /// Helper trait that allows `arena_cache` queries to return `Option<&T>`
 /// instead of `&Option`, and avoid allocating `None` in the arena.
 ///
@@ -11,10 +17,11 @@ pub trait ArenaCached<'tcx>: Sized {
     /// Type that is stored in the arena.
     type Allocated: 'tcx;
 
-    /// Takes a provided value, and allocates it in the arena (if appropriate)
-    /// with the help of the given `arena_alloc` closure.
+    /// Takes a provided value, and allocates it in an appropriate arena,
+    /// unless the particular value doesn't need allocation (e.g. `None`).
     fn alloc_in_arena(
-        arena_alloc: impl Fn(Self::Allocated) -> &'tcx Self::Allocated,
+        tcx: TyCtxt<'tcx>,
+        typed_arena: &'tcx TypedArena,
         value: Self::Provided,
     ) -> Self;
 }
@@ -23,12 +30,9 @@ impl<'tcx, T> ArenaCached<'tcx> for &'tcx T {
     type Provided = T;
     type Allocated = T;
 
-    fn alloc_in_arena(
-        arena_alloc: impl Fn(Self::Allocated) -> &'tcx Self::Allocated,
-        value: Self::Provided,
-    ) -> Self {
+    fn alloc_in_arena(tcx: TyCtxt<'tcx>, typed_arena: &'tcx TypedArena, value: T) -> Self {
         // Just allocate in the arena normally.
-        arena_alloc(value)
+        do_alloc(tcx, typed_arena, value)
     }
 }
 
@@ -38,10 +42,17 @@ impl<'tcx, T> ArenaCached<'tcx> for Option<&'tcx T> {
     type Allocated = T;
 
     fn alloc_in_arena(
-        arena_alloc: impl Fn(Self::Allocated) -> &'tcx Self::Allocated,
-        value: Self::Provided,
+        tcx: TyCtxt<'tcx>,
+        typed_arena: &'tcx TypedArena,
+        value: Option,
     ) -> Self {
         // Don't store None in the arena, and wrap the allocated reference in Some.
-        value.map(arena_alloc)
+        try { do_alloc(tcx, typed_arena, value?) }
     }
 }
+
+/// Allocates a value in either its dedicated arena, or in the common dropless
+/// arena, depending on whether it needs to be dropped.
+fn do_alloc<'tcx, T>(tcx: TyCtxt<'tcx>, typed_arena: &'tcx TypedArena, value: T) -> &'tcx T {
+    if mem::needs_drop::() { typed_arena.alloc(value) } else { tcx.arena.dropless.alloc(value) }
+}
diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs
index 04a3ceb0918a..b80f096ec993 100644
--- a/compiler/rustc_middle/src/query/plumbing.rs
+++ b/compiler/rustc_middle/src/query/plumbing.rs
@@ -295,36 +295,28 @@ macro_rules! define_callbacks {
                 ($V)
             );
 
-            /// This function takes `ProvidedValue` and converts it to an erased `Value` by
-            /// allocating it on an arena if the query has the `arena_cache` modifier. The
-            /// value is then erased and returned. This will happen when computing the query
-            /// using a provider or decoding a stored result.
+            /// This helper function takes a value returned by the query provider
+            /// (or loaded from disk, or supplied by query feeding), allocates
+            /// it in an arena if requested by the `arena_cache` modifier, and
+            /// then returns an erased copy of it.
             #[inline(always)]
             pub fn provided_to_erased<'tcx>(
-                _tcx: TyCtxt<'tcx>,
+                tcx: TyCtxt<'tcx>,
                 provided_value: ProvidedValue<'tcx>,
             ) -> Erased> {
-                // Store the provided value in an arena and get a reference
-                // to it, for queries with `arena_cache`.
-                let value: Value<'tcx> = query_if_arena!([$($modifiers)*]
-                    {
-                        use $crate::query::arena_cached::ArenaCached;
-
-                        if mem::needs_drop::<<$V as ArenaCached<'tcx>>::Allocated>() {
-                            <$V as ArenaCached>::alloc_in_arena(
-                                |v| _tcx.query_system.arenas.$name.alloc(v),
-                                provided_value,
-                            )
-                        } else {
-                            <$V as ArenaCached>::alloc_in_arena(
-                                |v| _tcx.arena.dropless.alloc(v),
-                                provided_value,
-                            )
-                        }
-                    }
-                    // Otherwise, the provided value is the value.
-                    (provided_value)
-                );
+                // For queries with the `arena_cache` modifier, store the
+                // provided value in an arena and get a reference to it.
+                let value: Value<'tcx> = query_if_arena!([$($modifiers)*] {
+                    <$V as $crate::query::arena_cached::ArenaCached>::alloc_in_arena(
+                        tcx,
+                        &tcx.query_system.arenas.$name,
+                        provided_value,
+                    )
+                } {
+                    // Otherwise, the provided value is the value (and `tcx` is unused).
+                    let _ = tcx;
+                    provided_value
+                });
                 erase::erase_val(value)
             }
 

From b6b6dc0b8260bacc01304c230e53b4328ae4ce23 Mon Sep 17 00:00:00 2001
From: zedddie 
Date: Thu, 5 Feb 2026 18:54:59 +0100
Subject: [PATCH 1525/1843] mGCA: add associated const type check in dyn arm

---
 .../rustc_trait_selection/src/traits/wf.rs    | 28 +++++++++++++++++++
 .../wf-mismatch-1.rs                          | 12 ++++++++
 .../wf-mismatch-2.rs                          | 12 ++++++++
 .../wf-mismatch-3.rs                          | 15 ++++++++++
 4 files changed, 67 insertions(+)
 create mode 100644 tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs
 create mode 100644 tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs
 create mode 100644 tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs

diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index d383cb9d91dd..358f14221a0d 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -951,6 +951,34 @@ impl<'a, 'tcx> TypeVisitor> for WfPredicates<'a, 'tcx> {
                         ty::Binder::dummy(ty::PredicateKind::DynCompatible(principal)),
                     ));
                 }
+
+                if !t.has_escaping_bound_vars() {
+                    for projection in data.projection_bounds() {
+                        let pred_binder = projection
+                            .with_self_ty(tcx, t)
+                            .map_bound(|p| {
+                                p.term.as_const().map(|ct| {
+                                    let assoc_const_ty = tcx
+                                        .type_of(p.projection_term.def_id)
+                                        .instantiate(tcx, p.projection_term.args);
+                                    ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(
+                                        ct,
+                                        assoc_const_ty,
+                                    ))
+                                })
+                            })
+                            .transpose();
+                        if let Some(pred_binder) = pred_binder {
+                            self.out.push(traits::Obligation::with_depth(
+                                tcx,
+                                self.cause(ObligationCauseCode::WellFormed(None)),
+                                self.recursion_depth,
+                                self.param_env,
+                                pred_binder,
+                            ));
+                        }
+                    }
+                }
             }
 
             // Inference variables are the complicated case, since we don't
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs
new file mode 100644
index 000000000000..319e3421a4cd
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs
@@ -0,0 +1,12 @@
+//@ known-bug: unknown
+//@ check-pass
+
+#![feature(min_generic_const_args)]
+#![expect(incomplete_features)]
+
+trait Trait { #[type_const] const CT: bool; }
+
+// FIXME: this should yield a type mismatch (`bool` v `i32`)
+fn f(_: impl Trait) {}
+
+fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs
new file mode 100644
index 000000000000..2adaaa3dd050
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs
@@ -0,0 +1,12 @@
+//@ known-bug: unknown
+//@ check-pass
+
+#![feature(min_generic_const_args)]
+#![expect(incomplete_features)]
+
+trait Trait { #[type_const] const CT: bool; }
+
+fn f() {
+    let _: dyn Trait; // FIXME: this should yield a type mismatch (`bool` v `i32`)
+}
+fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs
new file mode 100644
index 000000000000..29d089bf9d2b
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs
@@ -0,0 +1,15 @@
+//@ known-bug: unknown
+//@ check-pass
+
+#![feature(min_generic_const_args)]
+#![expect(incomplete_features)]
+
+trait Trait { #[type_const] const CT: bool; }
+
+trait Bound { #[type_const] const N: u32; }
+impl Bound for () { #[type_const] const N: u32 = 0; }
+
+fn f() { let _: dyn Trait::N }>; } // FIXME
+fn g(_: impl Trait::N }>) {} // FIXME
+
+fn main() {}

From 963f82d3224ea0b1f7b1ca182b67af3e9293e89f Mon Sep 17 00:00:00 2001
From: zedddie 
Date: Sat, 7 Feb 2026 07:15:24 +0100
Subject: [PATCH 1526/1843] mGCA: add associated const obligations in wfck

---
 .../rustc_hir_analysis/src/check/wfcheck.rs   | 31 ++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index ceb0e138a0da..32af13b88a1a 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -1569,11 +1569,40 @@ pub(super) fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, def_id:
 
     let predicates = predicates.instantiate_identity(tcx);
 
+    let assoc_const_obligations: Vec<_> = predicates
+        .predicates
+        .iter()
+        .copied()
+        .zip(predicates.spans.iter().copied())
+        .filter_map(|(clause, sp)| {
+            let proj = clause.as_projection_clause()?;
+            let pred_binder = proj
+                .map_bound(|pred| {
+                    pred.term.as_const().map(|ct| {
+                        let assoc_const_ty = tcx
+                            .type_of(pred.projection_term.def_id)
+                            .instantiate(tcx, pred.projection_term.args);
+                        ty::ClauseKind::ConstArgHasType(ct, assoc_const_ty)
+                    })
+                })
+                .transpose();
+            pred_binder.map(|pred_binder| {
+                let cause = traits::ObligationCause::new(
+                    sp,
+                    wfcx.body_def_id,
+                    ObligationCauseCode::WhereClause(def_id.to_def_id(), sp),
+                );
+                Obligation::new(tcx, cause, wfcx.param_env, pred_binder)
+            })
+        })
+        .collect();
+
     assert_eq!(predicates.predicates.len(), predicates.spans.len());
     let wf_obligations = predicates.into_iter().flat_map(|(p, sp)| {
         traits::wf::clause_obligations(infcx, wfcx.param_env, wfcx.body_def_id, p, sp)
     });
-    let obligations: Vec<_> = wf_obligations.chain(default_obligations).collect();
+    let obligations: Vec<_> =
+        wf_obligations.chain(default_obligations).chain(assoc_const_obligations).collect();
     wfcx.register_obligations(obligations);
 }
 

From 20f65fc537e26bd92e0da4088c27e5905c3fc52a Mon Sep 17 00:00:00 2001
From: Shoyu Vanilla 
Date: Tue, 10 Feb 2026 01:31:48 +0900
Subject: [PATCH 1527/1843] Uplift `Predicate::allow_normalization` to
 `rustc_type_ir`

---
 compiler/rustc_middle/src/ty/predicate.rs | 24 +---------------------
 compiler/rustc_type_ir/src/inherent.rs    | 25 ++++++++++++++++++++---
 2 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/predicate.rs b/compiler/rustc_middle/src/ty/predicate.rs
index 69dcca54f469..3baeb7141de5 100644
--- a/compiler/rustc_middle/src/ty/predicate.rs
+++ b/compiler/rustc_middle/src/ty/predicate.rs
@@ -49,10 +49,6 @@ impl<'tcx> rustc_type_ir::inherent::Predicate> for Predicate<'tcx>
     fn as_clause(self) -> Option> {
         self.as_clause()
     }
-
-    fn allow_normalization(self) -> bool {
-        self.allow_normalization()
-    }
 }
 
 impl<'tcx> rustc_type_ir::inherent::IntoKind for Predicate<'tcx> {
@@ -121,25 +117,7 @@ impl<'tcx> Predicate<'tcx> {
     /// unsoundly accept some programs. See #91068.
     #[inline]
     pub fn allow_normalization(self) -> bool {
-        match self.kind().skip_binder() {
-            PredicateKind::Clause(ClauseKind::WellFormed(_)) | PredicateKind::AliasRelate(..) => {
-                false
-            }
-            PredicateKind::Clause(ClauseKind::Trait(_))
-            | PredicateKind::Clause(ClauseKind::HostEffect(..))
-            | PredicateKind::Clause(ClauseKind::RegionOutlives(_))
-            | PredicateKind::Clause(ClauseKind::TypeOutlives(_))
-            | PredicateKind::Clause(ClauseKind::Projection(_))
-            | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
-            | PredicateKind::Clause(ClauseKind::UnstableFeature(_))
-            | PredicateKind::DynCompatible(_)
-            | PredicateKind::Subtype(_)
-            | PredicateKind::Coerce(_)
-            | PredicateKind::Clause(ClauseKind::ConstEvaluatable(_))
-            | PredicateKind::ConstEquate(_, _)
-            | PredicateKind::NormalizesTo(..)
-            | PredicateKind::Ambiguous => true,
-        }
+        rustc_type_ir::inherent::Predicate::allow_normalization(self)
     }
 }
 
diff --git a/compiler/rustc_type_ir/src/inherent.rs b/compiler/rustc_type_ir/src/inherent.rs
index 89cb236d38c6..c9580d589d21 100644
--- a/compiler/rustc_type_ir/src/inherent.rs
+++ b/compiler/rustc_type_ir/src/inherent.rs
@@ -13,7 +13,7 @@ use crate::fold::{TypeFoldable, TypeSuperFoldable};
 use crate::relate::Relate;
 use crate::solve::{AdtDestructorKind, SizedTraitKind};
 use crate::visit::{Flags, TypeSuperVisitable, TypeVisitable};
-use crate::{self as ty, CollectAndApply, Interner, UpcastFrom};
+use crate::{self as ty, ClauseKind, CollectAndApply, Interner, PredicateKind, UpcastFrom};
 
 pub trait Ty>:
     Copy
@@ -478,8 +478,27 @@ pub trait Predicate>:
         }
     }
 
-    // FIXME: Eventually uplift the impl out of rustc and make this defaulted.
-    fn allow_normalization(self) -> bool;
+    fn allow_normalization(self) -> bool {
+        match self.kind().skip_binder() {
+            PredicateKind::Clause(ClauseKind::WellFormed(_)) | PredicateKind::AliasRelate(..) => {
+                false
+            }
+            PredicateKind::Clause(ClauseKind::Trait(_))
+            | PredicateKind::Clause(ClauseKind::HostEffect(..))
+            | PredicateKind::Clause(ClauseKind::RegionOutlives(_))
+            | PredicateKind::Clause(ClauseKind::TypeOutlives(_))
+            | PredicateKind::Clause(ClauseKind::Projection(_))
+            | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
+            | PredicateKind::Clause(ClauseKind::UnstableFeature(_))
+            | PredicateKind::DynCompatible(_)
+            | PredicateKind::Subtype(_)
+            | PredicateKind::Coerce(_)
+            | PredicateKind::Clause(ClauseKind::ConstEvaluatable(_))
+            | PredicateKind::ConstEquate(_, _)
+            | PredicateKind::NormalizesTo(..)
+            | PredicateKind::Ambiguous => true,
+        }
+    }
 }
 
 pub trait Clause>:

From ec03e39afd210cff8516b14b92dc259f61d303c5 Mon Sep 17 00:00:00 2001
From: zedddie 
Date: Mon, 9 Feb 2026 10:40:42 +0100
Subject: [PATCH 1528/1843] bless tests

---
 ...ection-behind-trait-alias-mentions-self.rs |  1 +
 ...on-behind-trait-alias-mentions-self.stderr | 16 +++++++++++++--
 ...yn-const-projection-escaping-bound-vars.rs | 11 ++++++++++
 .../wf-mismatch-1.rs                          |  9 ++++-----
 .../wf-mismatch-1.stderr                      | 14 +++++++++++++
 .../wf-mismatch-2.rs                          |  9 +++++----
 .../wf-mismatch-2.stderr                      |  8 ++++++++
 .../wf-mismatch-3.rs                          | 16 ++++++++-------
 .../wf-mismatch-3.stderr                      | 20 +++++++++++++++++++
 9 files changed, 86 insertions(+), 18 deletions(-)
 create mode 100644 tests/ui/const-generics/associated-const-bindings/dyn-const-projection-escaping-bound-vars.rs
 create mode 100644 tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.stderr
 create mode 100644 tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.stderr
 create mode 100644 tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.stderr

diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs
index 415d7bda347b..9564903dce5b 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs
@@ -15,6 +15,7 @@ trait Trait {
 struct Hold(T);
 
 trait Bound = Trait }>;
+//~^ ERROR the constant `Hold::` is not of type `i32`
 
 fn main() {
     let _: dyn Bound; //~ ERROR associated constant binding in trait object type mentions `Self`
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr
index 576d4c3230ee..699b9192cf93 100644
--- a/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-compat-const-projection-behind-trait-alias-mentions-self.stderr
@@ -1,5 +1,17 @@
+error: the constant `Hold::` is not of type `i32`
+  --> $DIR/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs:17:21
+   |
+LL | trait Bound = Trait }>;
+   |                     ^^^^^^^^^^^^^^^^^^^^ expected `i32`, found struct constructor
+   |
+note: required by a const generic parameter in `Bound`
+  --> $DIR/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs:17:21
+   |
+LL | trait Bound = Trait }>;
+   |                     ^^^^^^^^^^^^^^^^^^^^ required by this const generic parameter in `Bound`
+
 error: associated constant binding in trait object type mentions `Self`
-  --> $DIR/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs:20:12
+  --> $DIR/dyn-compat-const-projection-behind-trait-alias-mentions-self.rs:21:12
    |
 LL | trait Bound = Trait }>;
    |                     -------------------- this binding mentions `Self`
@@ -7,5 +19,5 @@ LL | trait Bound = Trait }>;
 LL |     let _: dyn Bound;
    |            ^^^^^^^^^ contains a mention of `Self`
 
-error: aborting due to 1 previous error
+error: aborting due to 2 previous errors
 
diff --git a/tests/ui/const-generics/associated-const-bindings/dyn-const-projection-escaping-bound-vars.rs b/tests/ui/const-generics/associated-const-bindings/dyn-const-projection-escaping-bound-vars.rs
new file mode 100644
index 000000000000..c2ae86232a02
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/dyn-const-projection-escaping-bound-vars.rs
@@ -0,0 +1,11 @@
+//! Check associated const binding with escaping bound vars doesn't cause ICE
+//! (#151642)
+//@ check-pass
+
+#![feature(min_generic_const_args)]
+#![expect(incomplete_features)]
+
+trait Trait2<'a> { type const ASSOC: i32; }
+fn g(_: for<'a> fn(Box>)) {}
+
+fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs
index 319e3421a4cd..1beeb07e995a 100644
--- a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.rs
@@ -1,12 +1,11 @@
-//@ known-bug: unknown
-//@ check-pass
+//! Check that we correctly handle associated const bindings
+//! in `impl Trait` where the RHS is a const param (#151642).
 
 #![feature(min_generic_const_args)]
 #![expect(incomplete_features)]
 
-trait Trait { #[type_const] const CT: bool; }
+trait Trait { type const CT: bool; }
 
-// FIXME: this should yield a type mismatch (`bool` v `i32`)
 fn f(_: impl Trait) {}
-
+//~^ ERROR the constant `N` is not of type `bool`
 fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.stderr b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.stderr
new file mode 100644
index 000000000000..56e01e640078
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-1.stderr
@@ -0,0 +1,14 @@
+error: the constant `N` is not of type `bool`
+  --> $DIR/wf-mismatch-1.rs:9:34
+   |
+LL | fn f(_: impl Trait) {}
+   |                                  ^^^^^^^^^^ expected `bool`, found `i32`
+   |
+note: required by a const generic parameter in `f`
+  --> $DIR/wf-mismatch-1.rs:9:34
+   |
+LL | fn f(_: impl Trait) {}
+   |                                  ^^^^^^^^^^ required by this const generic parameter in `f`
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs
index 2adaaa3dd050..7a75b6da78c5 100644
--- a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.rs
@@ -1,12 +1,13 @@
-//@ known-bug: unknown
-//@ check-pass
+//! Check that we correctly handle associated const bindings
+//! in `dyn Trait` where the RHS is a const param (#151642).
 
 #![feature(min_generic_const_args)]
 #![expect(incomplete_features)]
 
-trait Trait { #[type_const] const CT: bool; }
+trait Trait { type const CT: bool; }
 
 fn f() {
-    let _: dyn Trait; // FIXME: this should yield a type mismatch (`bool` v `i32`)
+    let _: dyn Trait;
+    //~^ ERROR the constant `N` is not of type `bool`
 }
 fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.stderr b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.stderr
new file mode 100644
index 000000000000..8169cc07fc5f
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-2.stderr
@@ -0,0 +1,8 @@
+error: the constant `N` is not of type `bool`
+  --> $DIR/wf-mismatch-2.rs:10:12
+   |
+LL |     let _: dyn Trait;
+   |            ^^^^^^^^^^^^^^^^^^^^^ expected `bool`, found `i32`
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs
index 29d089bf9d2b..3eeb7703aa2b 100644
--- a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.rs
@@ -1,15 +1,17 @@
-//@ known-bug: unknown
-//@ check-pass
+//! Check that we correctly handle associated const bindings
+//! where the RHS is a normalizable const projection (#151642).
 
 #![feature(min_generic_const_args)]
 #![expect(incomplete_features)]
 
-trait Trait { #[type_const] const CT: bool; }
+trait Trait { type const CT: bool; }
 
-trait Bound { #[type_const] const N: u32; }
-impl Bound for () { #[type_const] const N: u32 = 0; }
+trait Bound { type const N: u32; }
+impl Bound for () { type const N: u32 = 0; }
 
-fn f() { let _: dyn Trait::N }>; } // FIXME
-fn g(_: impl Trait::N }>) {} // FIXME
+fn f() { let _: dyn Trait::N }>; }
+//~^ ERROR the constant `0` is not of type `bool`
+fn g(_: impl Trait::N }>) {}
+//~^ ERROR the constant `0` is not of type `bool`
 
 fn main() {}
diff --git a/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.stderr b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.stderr
new file mode 100644
index 000000000000..ac21527e04ed
--- /dev/null
+++ b/tests/ui/const-generics/associated-const-bindings/wf-mismatch-3.stderr
@@ -0,0 +1,20 @@
+error: the constant `0` is not of type `bool`
+  --> $DIR/wf-mismatch-3.rs:14:20
+   |
+LL | fn g(_: impl Trait::N }>) {}
+   |                    ^^^^^^^^^^^^^^^^^^^^^^^^^ expected `bool`, found `u32`
+   |
+note: required by a const generic parameter in `g`
+  --> $DIR/wf-mismatch-3.rs:14:20
+   |
+LL | fn g(_: impl Trait::N }>) {}
+   |                    ^^^^^^^^^^^^^^^^^^^^^^^^^ required by this const generic parameter in `g`
+
+error: the constant `0` is not of type `bool`
+  --> $DIR/wf-mismatch-3.rs:12:17
+   |
+LL | fn f() { let _: dyn Trait::N }>; }
+   |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `bool`, found `u32`
+
+error: aborting due to 2 previous errors
+

From 066a935b0c25608db20da43951def47f3b1539e6 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Tue, 10 Feb 2026 10:17:13 +1100
Subject: [PATCH 1529/1843] Move parts of `rustc_query_system::query::job` to
 `rustc_middle::job`.

The latter is a new module.

As well as the code motion, some other changes were required.
- `QueryJobId` methods became free functions so they could move while
  `QueryJobId` itself stayed put. This was so `QueryMap` and
  `QueryJobInfo` could be moved.
- Some visibilities in `rustc_query_system` required changing.
- `collect_active_jobs_from_all_queries` is no longer required in `trait
  QueryContext`.
---
 Cargo.lock                                   |   1 +
 compiler/rustc_interface/src/interface.rs    |   3 +-
 compiler/rustc_interface/src/util.rs         |   3 +-
 compiler/rustc_query_impl/Cargo.toml         |   1 +
 compiler/rustc_query_impl/src/execution.rs   |   9 +-
 compiler/rustc_query_impl/src/job.rs         | 446 ++++++++++++++++++
 compiler/rustc_query_impl/src/lib.rs         |   4 +-
 compiler/rustc_query_impl/src/plumbing.rs    |  59 +--
 compiler/rustc_query_system/src/query/job.rs | 460 +------------------
 compiler/rustc_query_system/src/query/mod.rs |  12 +-
 10 files changed, 504 insertions(+), 494 deletions(-)
 create mode 100644 compiler/rustc_query_impl/src/job.rs

diff --git a/Cargo.lock b/Cargo.lock
index 545c776b4805..7230d633602d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4503,6 +4503,7 @@ dependencies = [
  "rustc_query_system",
  "rustc_serialize",
  "rustc_span",
+ "rustc_thread_pool",
  "tracing",
 ]
 
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 72080c5f5369..6cbc184e8f1b 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -16,8 +16,7 @@ use rustc_parse::lexer::StripTokens;
 use rustc_parse::new_parser_from_source_str;
 use rustc_parse::parser::Recovery;
 use rustc_parse::parser::attr::AllowLeadingUnsafe;
-use rustc_query_impl::QueryCtxt;
-use rustc_query_system::query::print_query_stack;
+use rustc_query_impl::{QueryCtxt, print_query_stack};
 use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
 use rustc_session::parse::ParseSess;
 use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint};
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 458e7ad32143..59f0b67d6729 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -184,8 +184,7 @@ pub(crate) fn run_in_thread_pool_with_globals<
     use rustc_data_structures::defer;
     use rustc_data_structures::sync::FromDyn;
     use rustc_middle::ty::tls;
-    use rustc_query_impl::QueryCtxt;
-    use rustc_query_system::query::{QueryContext, break_query_cycles};
+    use rustc_query_impl::{QueryCtxt, break_query_cycles};
 
     let thread_stack_size = init_stack_size(thread_builder_diag);
 
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index d611629671a0..511337090d7f 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -16,5 +16,6 @@ rustc_middle = { path = "../rustc_middle" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
+rustc_thread_pool = { path = "../rustc_thread_pool" }
 tracing = "0.1"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
index 88604c91d025..6f16932cb2eb 100644
--- a/compiler/rustc_query_impl/src/execution.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -9,13 +9,14 @@ use rustc_middle::dep_graph::DepsType;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext};
 use rustc_query_system::query::{
-    ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryContext, QueryJob,
-    QueryJobId, QueryJobInfo, QueryLatch, QueryMap, QueryMode, QueryStackDeferred, QueryStackFrame,
-    QueryState, incremental_verify_ich, report_cycle,
+    ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch,
+    QueryMode, QueryStackDeferred, QueryStackFrame, QueryState, incremental_verify_ich,
+    report_cycle,
 };
 use rustc_span::{DUMMY_SP, Span};
 
 use crate::dep_graph::{DepContext, DepNode, DepNodeIndex};
+use crate::job::{QueryJobInfo, QueryMap, find_cycle_in_stack};
 use crate::{QueryCtxt, QueryFlags, SemiDynamicQueryDispatcher};
 
 #[inline]
@@ -218,7 +219,7 @@ fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
         .ok()
         .expect("failed to collect active queries");
 
-    let error = try_execute.find_cycle_in_stack(query_map, &qcx.current_query_job(), span);
+    let error = find_cycle_in_stack(try_execute, query_map, &qcx.current_query_job(), span);
     (mk_cycle(query, qcx, error.lift()), None)
 }
 
diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_query_impl/src/job.rs
new file mode 100644
index 000000000000..cb0a13d32ad2
--- /dev/null
+++ b/compiler/rustc_query_impl/src/job.rs
@@ -0,0 +1,446 @@
+use std::io::Write;
+use std::iter;
+use std::sync::Arc;
+
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_errors::DiagCtxtHandle;
+use rustc_query_system::query::{
+    CycleError, QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryStackDeferred, QueryStackFrame,
+    QueryWaiter,
+};
+use rustc_span::{DUMMY_SP, Span};
+
+use crate::QueryCtxt;
+use crate::dep_graph::DepContext;
+
+/// Map from query job IDs to job information collected by
+/// `collect_active_jobs_from_all_queries`.
+pub type QueryMap<'tcx> = FxHashMap>;
+
+fn query_job_id_frame<'a, 'tcx>(
+    id: QueryJobId,
+    map: &'a QueryMap<'tcx>,
+) -> QueryStackFrame> {
+    map.get(&id).unwrap().frame.clone()
+}
+
+fn query_job_id_span<'a, 'tcx>(id: QueryJobId, map: &'a QueryMap<'tcx>) -> Span {
+    map.get(&id).unwrap().job.span
+}
+
+fn query_job_id_parent<'a, 'tcx>(id: QueryJobId, map: &'a QueryMap<'tcx>) -> Option {
+    map.get(&id).unwrap().job.parent
+}
+
+fn query_job_id_latch<'a, 'tcx>(
+    id: QueryJobId,
+    map: &'a QueryMap<'tcx>,
+) -> Option<&'a QueryLatch<'tcx>> {
+    map.get(&id).unwrap().job.latch.as_ref()
+}
+
+#[derive(Clone, Debug)]
+pub struct QueryJobInfo<'tcx> {
+    pub frame: QueryStackFrame>,
+    pub job: QueryJob<'tcx>,
+}
+
+pub(crate) fn find_cycle_in_stack<'tcx>(
+    id: QueryJobId,
+    query_map: QueryMap<'tcx>,
+    current_job: &Option,
+    span: Span,
+) -> CycleError> {
+    // Find the waitee amongst `current_job` parents
+    let mut cycle = Vec::new();
+    let mut current_job = Option::clone(current_job);
+
+    while let Some(job) = current_job {
+        let info = query_map.get(&job).unwrap();
+        cycle.push(QueryInfo { span: info.job.span, frame: info.frame.clone() });
+
+        if job == id {
+            cycle.reverse();
+
+            // This is the end of the cycle
+            // The span entry we included was for the usage
+            // of the cycle itself, and not part of the cycle
+            // Replace it with the span which caused the cycle to form
+            cycle[0].span = span;
+            // Find out why the cycle itself was used
+            let usage = info
+                .job
+                .parent
+                .as_ref()
+                .map(|parent| (info.job.span, query_job_id_frame(*parent, &query_map)));
+            return CycleError { usage, cycle };
+        }
+
+        current_job = info.job.parent;
+    }
+
+    panic!("did not find a cycle")
+}
+
+#[cold]
+#[inline(never)]
+pub(crate) fn find_dep_kind_root<'tcx>(
+    id: QueryJobId,
+    query_map: QueryMap<'tcx>,
+) -> (QueryJobInfo<'tcx>, usize) {
+    let mut depth = 1;
+    let info = query_map.get(&id).unwrap();
+    let dep_kind = info.frame.dep_kind;
+    let mut current_id = info.job.parent;
+    let mut last_layout = (info.clone(), depth);
+
+    while let Some(id) = current_id {
+        let info = query_map.get(&id).unwrap();
+        if info.frame.dep_kind == dep_kind {
+            depth += 1;
+            last_layout = (info.clone(), depth);
+        }
+        current_id = info.job.parent;
+    }
+    last_layout
+}
+
+/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
+type Waiter = (QueryJobId, usize);
+
+/// Visits all the non-resumable and resumable waiters of a query.
+/// Only waiters in a query are visited.
+/// `visit` is called for every waiter and is passed a query waiting on `query_ref`
+/// and a span indicating the reason the query waited on `query_ref`.
+/// If `visit` returns Some, this function returns.
+/// For visits of non-resumable waiters it returns the return value of `visit`.
+/// For visits of resumable waiters it returns Some(Some(Waiter)) which has the
+/// required information to resume the waiter.
+/// If all `visit` calls returns None, this function also returns None.
+fn visit_waiters<'tcx, F>(
+    query_map: &QueryMap<'tcx>,
+    query: QueryJobId,
+    mut visit: F,
+) -> Option>
+where
+    F: FnMut(Span, QueryJobId) -> Option>,
+{
+    // Visit the parent query which is a non-resumable waiter since it's on the same stack
+    if let Some(parent) = query_job_id_parent(query, query_map)
+        && let Some(cycle) = visit(query_job_id_span(query, query_map), parent)
+    {
+        return Some(cycle);
+    }
+
+    // Visit the explicit waiters which use condvars and are resumable
+    if let Some(latch) = query_job_id_latch(query, query_map) {
+        for (i, waiter) in latch.info.lock().waiters.iter().enumerate() {
+            if let Some(waiter_query) = waiter.query {
+                if visit(waiter.span, waiter_query).is_some() {
+                    // Return a value which indicates that this waiter can be resumed
+                    return Some(Some((query, i)));
+                }
+            }
+        }
+    }
+
+    None
+}
+
+/// Look for query cycles by doing a depth first search starting at `query`.
+/// `span` is the reason for the `query` to execute. This is initially DUMMY_SP.
+/// If a cycle is detected, this initial value is replaced with the span causing
+/// the cycle.
+fn cycle_check<'tcx>(
+    query_map: &QueryMap<'tcx>,
+    query: QueryJobId,
+    span: Span,
+    stack: &mut Vec<(Span, QueryJobId)>,
+    visited: &mut FxHashSet,
+) -> Option> {
+    if !visited.insert(query) {
+        return if let Some(p) = stack.iter().position(|q| q.1 == query) {
+            // We detected a query cycle, fix up the initial span and return Some
+
+            // Remove previous stack entries
+            stack.drain(0..p);
+            // Replace the span for the first query with the cycle cause
+            stack[0].0 = span;
+            Some(None)
+        } else {
+            None
+        };
+    }
+
+    // Query marked as visited is added it to the stack
+    stack.push((span, query));
+
+    // Visit all the waiters
+    let r = visit_waiters(query_map, query, |span, successor| {
+        cycle_check(query_map, successor, span, stack, visited)
+    });
+
+    // Remove the entry in our stack if we didn't find a cycle
+    if r.is_none() {
+        stack.pop();
+    }
+
+    r
+}
+
+/// Finds out if there's a path to the compiler root (aka. code which isn't in a query)
+/// from `query` without going through any of the queries in `visited`.
+/// This is achieved with a depth first search.
+fn connected_to_root<'tcx>(
+    query_map: &QueryMap<'tcx>,
+    query: QueryJobId,
+    visited: &mut FxHashSet,
+) -> bool {
+    // We already visited this or we're deliberately ignoring it
+    if !visited.insert(query) {
+        return false;
+    }
+
+    // This query is connected to the root (it has no query parent), return true
+    if query_job_id_parent(query, query_map).is_none() {
+        return true;
+    }
+
+    visit_waiters(query_map, query, |_, successor| {
+        connected_to_root(query_map, successor, visited).then_some(None)
+    })
+    .is_some()
+}
+
+// Deterministically pick an query from a list
+fn pick_query<'a, 'tcx, T, F>(query_map: &QueryMap<'tcx>, queries: &'a [T], f: F) -> &'a T
+where
+    F: Fn(&T) -> (Span, QueryJobId),
+{
+    // Deterministically pick an entry point
+    // FIXME: Sort this instead
+    queries
+        .iter()
+        .min_by_key(|v| {
+            let (span, query) = f(v);
+            let hash = query_job_id_frame(query, query_map).hash;
+            // Prefer entry points which have valid spans for nicer error messages
+            // We add an integer to the tuple ensuring that entry points
+            // with valid spans are picked first
+            let span_cmp = if span == DUMMY_SP { 1 } else { 0 };
+            (span_cmp, hash)
+        })
+        .unwrap()
+}
+
+/// Looks for query cycles starting from the last query in `jobs`.
+/// If a cycle is found, all queries in the cycle is removed from `jobs` and
+/// the function return true.
+/// If a cycle was not found, the starting query is removed from `jobs` and
+/// the function returns false.
+fn remove_cycle<'tcx>(
+    query_map: &QueryMap<'tcx>,
+    jobs: &mut Vec,
+    wakelist: &mut Vec>>,
+) -> bool {
+    let mut visited = FxHashSet::default();
+    let mut stack = Vec::new();
+    // Look for a cycle starting with the last query in `jobs`
+    if let Some(waiter) =
+        cycle_check(query_map, jobs.pop().unwrap(), DUMMY_SP, &mut stack, &mut visited)
+    {
+        // The stack is a vector of pairs of spans and queries; reverse it so that
+        // the earlier entries require later entries
+        let (mut spans, queries): (Vec<_>, Vec<_>) = stack.into_iter().rev().unzip();
+
+        // Shift the spans so that queries are matched with the span for their waitee
+        spans.rotate_right(1);
+
+        // Zip them back together
+        let mut stack: Vec<_> = iter::zip(spans, queries).collect();
+
+        // Remove the queries in our cycle from the list of jobs to look at
+        for r in &stack {
+            if let Some(pos) = jobs.iter().position(|j| j == &r.1) {
+                jobs.remove(pos);
+            }
+        }
+
+        // Find the queries in the cycle which are
+        // connected to queries outside the cycle
+        let entry_points = stack
+            .iter()
+            .filter_map(|&(span, query)| {
+                if query_job_id_parent(query, query_map).is_none() {
+                    // This query is connected to the root (it has no query parent)
+                    Some((span, query, None))
+                } else {
+                    let mut waiters = Vec::new();
+                    // Find all the direct waiters who lead to the root
+                    visit_waiters(query_map, query, |span, waiter| {
+                        // Mark all the other queries in the cycle as already visited
+                        let mut visited = FxHashSet::from_iter(stack.iter().map(|q| q.1));
+
+                        if connected_to_root(query_map, waiter, &mut visited) {
+                            waiters.push((span, waiter));
+                        }
+
+                        None
+                    });
+                    if waiters.is_empty() {
+                        None
+                    } else {
+                        // Deterministically pick one of the waiters to show to the user
+                        let waiter = *pick_query(query_map, &waiters, |s| *s);
+                        Some((span, query, Some(waiter)))
+                    }
+                }
+            })
+            .collect::)>>();
+
+        // Deterministically pick an entry point
+        let (_, entry_point, usage) = pick_query(query_map, &entry_points, |e| (e.0, e.1));
+
+        // Shift the stack so that our entry point is first
+        let entry_point_pos = stack.iter().position(|(_, query)| query == entry_point);
+        if let Some(pos) = entry_point_pos {
+            stack.rotate_left(pos);
+        }
+
+        let usage =
+            usage.as_ref().map(|(span, query)| (*span, query_job_id_frame(*query, query_map)));
+
+        // Create the cycle error
+        let error = CycleError {
+            usage,
+            cycle: stack
+                .iter()
+                .map(|&(s, ref q)| QueryInfo { span: s, frame: query_job_id_frame(*q, query_map) })
+                .collect(),
+        };
+
+        // We unwrap `waiter` here since there must always be one
+        // edge which is resumable / waited using a query latch
+        let (waitee_query, waiter_idx) = waiter.unwrap();
+
+        // Extract the waiter we want to resume
+        let waiter =
+            query_job_id_latch(waitee_query, query_map).unwrap().extract_waiter(waiter_idx);
+
+        // Set the cycle error so it will be picked up when resumed
+        *waiter.cycle.lock() = Some(error);
+
+        // Put the waiter on the list of things to resume
+        wakelist.push(waiter);
+
+        true
+    } else {
+        false
+    }
+}
+
+/// Detects query cycles by using depth first search over all active query jobs.
+/// If a query cycle is found it will break the cycle by finding an edge which
+/// uses a query latch and then resuming that waiter.
+/// There may be multiple cycles involved in a deadlock, so this searches
+/// all active queries for cycles before finally resuming all the waiters at once.
+pub fn break_query_cycles<'tcx>(query_map: QueryMap<'tcx>, registry: &rustc_thread_pool::Registry) {
+    let mut wakelist = Vec::new();
+    // It is OK per the comments:
+    // - https://github.com/rust-lang/rust/pull/131200#issuecomment-2798854932
+    // - https://github.com/rust-lang/rust/pull/131200#issuecomment-2798866392
+    #[allow(rustc::potential_query_instability)]
+    let mut jobs: Vec = query_map.keys().cloned().collect();
+
+    let mut found_cycle = false;
+
+    while jobs.len() > 0 {
+        if remove_cycle(&query_map, &mut jobs, &mut wakelist) {
+            found_cycle = true;
+        }
+    }
+
+    // Check that a cycle was found. It is possible for a deadlock to occur without
+    // a query cycle if a query which can be waited on uses Rayon to do multithreading
+    // internally. Such a query (X) may be executing on 2 threads (A and B) and A may
+    // wait using Rayon on B. Rayon may then switch to executing another query (Y)
+    // which in turn will wait on X causing a deadlock. We have a false dependency from
+    // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
+    // only considers the true dependency and won't detect a cycle.
+    if !found_cycle {
+        panic!(
+            "deadlock detected as we're unable to find a query cycle to break\n\
+            current query map:\n{:#?}",
+            query_map
+        );
+    }
+
+    // Mark all the thread we're about to wake up as unblocked. This needs to be done before
+    // we wake the threads up as otherwise Rayon could detect a deadlock if a thread we
+    // resumed fell asleep and this thread had yet to mark the remaining threads as unblocked.
+    for _ in 0..wakelist.len() {
+        rustc_thread_pool::mark_unblocked(registry);
+    }
+
+    for waiter in wakelist.into_iter() {
+        waiter.condvar.notify_one();
+    }
+}
+
+pub fn print_query_stack<'tcx>(
+    qcx: QueryCtxt<'tcx>,
+    mut current_query: Option,
+    dcx: DiagCtxtHandle<'_>,
+    limit_frames: Option,
+    mut file: Option,
+) -> usize {
+    // Be careful relying on global state here: this code is called from
+    // a panic hook, which means that the global `DiagCtxt` may be in a weird
+    // state if it was responsible for triggering the panic.
+    let mut count_printed = 0;
+    let mut count_total = 0;
+
+    // Make use of a partial query map if we fail to take locks collecting active queries.
+    let query_map = match qcx.collect_active_jobs_from_all_queries(false) {
+        Ok(query_map) => query_map,
+        Err(query_map) => query_map,
+    };
+
+    if let Some(ref mut file) = file {
+        let _ = writeln!(file, "\n\nquery stack during panic:");
+    }
+    while let Some(query) = current_query {
+        let Some(query_info) = query_map.get(&query) else {
+            break;
+        };
+        let query_extra = query_info.frame.info.extract();
+        if Some(count_printed) < limit_frames || limit_frames.is_none() {
+            // Only print to stderr as many stack frames as `num_frames` when present.
+            dcx.struct_failure_note(format!(
+                "#{} [{:?}] {}",
+                count_printed, query_info.frame.dep_kind, query_extra.description
+            ))
+            .with_span(query_info.job.span)
+            .emit();
+            count_printed += 1;
+        }
+
+        if let Some(ref mut file) = file {
+            let _ = writeln!(
+                file,
+                "#{} [{}] {}",
+                count_total,
+                qcx.tcx.dep_kind_vtable(query_info.frame.dep_kind).name,
+                query_extra.description
+            );
+        }
+
+        current_query = query_info.job.parent;
+        count_total += 1;
+    }
+
+    if let Some(ref mut file) = file {
+        let _ = writeln!(file, "end of query stack");
+    }
+    count_total
+}
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index a33bdd22a797..c1ead8da344b 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -23,10 +23,11 @@ use rustc_middle::query::values::Value;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::query::{
-    CycleError, CycleErrorHandling, QueryCache, QueryMap, QueryMode, QueryState,
+    CycleError, CycleErrorHandling, QueryCache, QueryMode, QueryState,
 };
 use rustc_span::{ErrorGuaranteed, Span};
 
+pub use crate::job::{QueryMap, break_query_cycles, print_query_stack};
 pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all};
 use crate::plumbing::{encode_all_query_results, try_mark_green};
 use crate::profiling_support::QueryKeyStringCache;
@@ -34,6 +35,7 @@ pub use crate::profiling_support::alloc_self_profile_query_strings;
 
 mod error;
 mod execution;
+mod job;
 #[macro_use]
 mod plumbing;
 mod profiling_support;
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 776d720f500b..5fba9dd4a68a 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -28,14 +28,15 @@ use rustc_middle::ty::tls::{self, ImplicitCtxt};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::{DepNodeKey, FingerprintStyle, HasDepContext};
 use rustc_query_system::query::{
-    QueryCache, QueryContext, QueryJobId, QueryMap, QuerySideEffect, QueryStackDeferred,
-    QueryStackFrame, QueryStackFrameExtra,
+    QueryCache, QueryContext, QueryJobId, QuerySideEffect, QueryStackDeferred, QueryStackFrame,
+    QueryStackFrameExtra,
 };
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::def_id::LOCAL_CRATE;
 
 use crate::error::{QueryOverflow, QueryOverflowNote};
 use crate::execution::{all_inactive, force_query};
+use crate::job::{QueryMap, find_dep_kind_root};
 use crate::{QueryDispatcherUnerased, QueryFlags, SemiDynamicQueryDispatcher};
 
 /// Implements [`QueryContext`] for use by [`rustc_query_system`], since that
@@ -55,7 +56,7 @@ impl<'tcx> QueryCtxt<'tcx> {
         let query_map = self
             .collect_active_jobs_from_all_queries(true)
             .expect("failed to collect active queries");
-        let (info, depth) = job.find_dep_kind_root(query_map);
+        let (info, depth) = find_dep_kind_root(job, query_map);
 
         let suggested_limit = match self.tcx.recursion_limit() {
             Limit(0) => Limit(2),
@@ -116,6 +117,32 @@ impl<'tcx> QueryCtxt<'tcx> {
             tls::enter_context(&new_icx, compute)
         })
     }
+
+    /// Returns a map of currently active query jobs, collected from all queries.
+    ///
+    /// If `require_complete` is `true`, this function locks all shards of the
+    /// query results to produce a complete map, which always returns `Ok`.
+    /// Otherwise, it may return an incomplete map as an error if any shard
+    /// lock cannot be acquired.
+    ///
+    /// Prefer passing `false` to `require_complete` to avoid potential deadlocks,
+    /// especially when called from within a deadlock handler, unless a
+    /// complete map is needed and no deadlock is possible at this call site.
+    pub fn collect_active_jobs_from_all_queries(
+        self,
+        require_complete: bool,
+    ) -> Result, QueryMap<'tcx>> {
+        let mut jobs = QueryMap::default();
+        let mut complete = true;
+
+        for gather_fn in crate::PER_QUERY_GATHER_ACTIVE_JOBS_FNS.iter() {
+            if gather_fn(self.tcx, &mut jobs, require_complete).is_none() {
+                complete = false;
+            }
+        }
+
+        if complete { Ok(jobs) } else { Err(jobs) }
+    }
 }
 
 impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
@@ -134,32 +161,6 @@ impl<'tcx> QueryContext<'tcx> for QueryCtxt<'tcx> {
         &self.tcx.jobserver_proxy
     }
 
-    /// Returns a map of currently active query jobs, collected from all queries.
-    ///
-    /// If `require_complete` is `true`, this function locks all shards of the
-    /// query results to produce a complete map, which always returns `Ok`.
-    /// Otherwise, it may return an incomplete map as an error if any shard
-    /// lock cannot be acquired.
-    ///
-    /// Prefer passing `false` to `require_complete` to avoid potential deadlocks,
-    /// especially when called from within a deadlock handler, unless a
-    /// complete map is needed and no deadlock is possible at this call site.
-    fn collect_active_jobs_from_all_queries(
-        self,
-        require_complete: bool,
-    ) -> Result, QueryMap<'tcx>> {
-        let mut jobs = QueryMap::default();
-        let mut complete = true;
-
-        for gather_fn in crate::PER_QUERY_GATHER_ACTIVE_JOBS_FNS.iter() {
-            if gather_fn(self.tcx, &mut jobs, require_complete).is_none() {
-                complete = false;
-            }
-        }
-
-        if complete { Ok(jobs) } else { Err(jobs) }
-    }
-
     // Interactions with on_disk_cache
     fn load_side_effect(
         self,
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index c77ae38ba96f..e8acb8064f7e 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -1,19 +1,15 @@
 use std::fmt::Debug;
 use std::hash::Hash;
-use std::io::Write;
-use std::iter;
 use std::num::NonZero;
 use std::sync::Arc;
 
 use parking_lot::{Condvar, Mutex};
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_errors::{Diag, DiagCtxtHandle};
+use rustc_errors::Diag;
 use rustc_hir::def::DefKind;
 use rustc_session::Session;
-use rustc_span::{DUMMY_SP, Span};
+use rustc_span::Span;
 
 use super::{QueryStackDeferred, QueryStackFrameExtra};
-use crate::dep_graph::DepContext;
 use crate::error::CycleStack;
 use crate::query::plumbing::CycleError;
 use crate::query::{QueryContext, QueryStackFrame};
@@ -32,38 +28,10 @@ impl<'tcx> QueryInfo> {
     }
 }
 
-/// Map from query job IDs to job information collected by
-/// [`QueryContext::collect_active_jobs_from_all_queries`].
-pub type QueryMap<'tcx> = FxHashMap>;
-
 /// A value uniquely identifying an active query job.
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 pub struct QueryJobId(pub NonZero);
 
-impl QueryJobId {
-    fn frame<'a, 'tcx>(self, map: &'a QueryMap<'tcx>) -> QueryStackFrame> {
-        map.get(&self).unwrap().frame.clone()
-    }
-
-    fn span<'a, 'tcx>(self, map: &'a QueryMap<'tcx>) -> Span {
-        map.get(&self).unwrap().job.span
-    }
-
-    fn parent<'a, 'tcx>(self, map: &'a QueryMap<'tcx>) -> Option {
-        map.get(&self).unwrap().job.parent
-    }
-
-    fn latch<'a, 'tcx>(self, map: &'a QueryMap<'tcx>) -> Option<&'a QueryLatch<'tcx>> {
-        map.get(&self).unwrap().job.latch.as_ref()
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct QueryJobInfo<'tcx> {
-    pub frame: QueryStackFrame>,
-    pub job: QueryJob<'tcx>,
-}
-
 /// Represents an active query job.
 #[derive(Clone, Debug)]
 pub struct QueryJob<'tcx> {
@@ -76,7 +44,7 @@ pub struct QueryJob<'tcx> {
     pub parent: Option,
 
     /// The latch that is used to wait on this job.
-    latch: Option>,
+    pub latch: Option>,
 }
 
 impl<'tcx> QueryJob<'tcx> {
@@ -105,85 +73,23 @@ impl<'tcx> QueryJob<'tcx> {
     }
 }
 
-impl QueryJobId {
-    pub fn find_cycle_in_stack<'tcx>(
-        &self,
-        query_map: QueryMap<'tcx>,
-        current_job: &Option,
-        span: Span,
-    ) -> CycleError> {
-        // Find the waitee amongst `current_job` parents
-        let mut cycle = Vec::new();
-        let mut current_job = Option::clone(current_job);
-
-        while let Some(job) = current_job {
-            let info = query_map.get(&job).unwrap();
-            cycle.push(QueryInfo { span: info.job.span, frame: info.frame.clone() });
-
-            if job == *self {
-                cycle.reverse();
-
-                // This is the end of the cycle
-                // The span entry we included was for the usage
-                // of the cycle itself, and not part of the cycle
-                // Replace it with the span which caused the cycle to form
-                cycle[0].span = span;
-                // Find out why the cycle itself was used
-                let usage = info
-                    .job
-                    .parent
-                    .as_ref()
-                    .map(|parent| (info.job.span, parent.frame(&query_map)));
-                return CycleError { usage, cycle };
-            }
-
-            current_job = info.job.parent;
-        }
-
-        panic!("did not find a cycle")
-    }
-
-    #[cold]
-    #[inline(never)]
-    pub fn find_dep_kind_root<'tcx>(
-        &self,
-        query_map: QueryMap<'tcx>,
-    ) -> (QueryJobInfo<'tcx>, usize) {
-        let mut depth = 1;
-        let info = query_map.get(&self).unwrap();
-        let dep_kind = info.frame.dep_kind;
-        let mut current_id = info.job.parent;
-        let mut last_layout = (info.clone(), depth);
-
-        while let Some(id) = current_id {
-            let info = query_map.get(&id).unwrap();
-            if info.frame.dep_kind == dep_kind {
-                depth += 1;
-                last_layout = (info.clone(), depth);
-            }
-            current_id = info.job.parent;
-        }
-        last_layout
-    }
+#[derive(Debug)]
+pub struct QueryWaiter<'tcx> {
+    pub query: Option,
+    pub condvar: Condvar,
+    pub span: Span,
+    pub cycle: Mutex>>>,
 }
 
 #[derive(Debug)]
-struct QueryWaiter<'tcx> {
-    query: Option,
-    condvar: Condvar,
-    span: Span,
-    cycle: Mutex>>>,
-}
-
-#[derive(Debug)]
-struct QueryLatchInfo<'tcx> {
-    complete: bool,
-    waiters: Vec>>,
+pub struct QueryLatchInfo<'tcx> {
+    pub complete: bool,
+    pub waiters: Vec>>,
 }
 
 #[derive(Clone, Debug)]
 pub struct QueryLatch<'tcx> {
-    info: Arc>>,
+    pub info: Arc>>,
 }
 
 impl<'tcx> QueryLatch<'tcx> {
@@ -250,7 +156,7 @@ impl<'tcx> QueryLatch<'tcx> {
 
     /// Removes a single waiter from the list of waiters.
     /// This is used to break query cycles.
-    fn extract_waiter(&self, waiter: usize) -> Arc> {
+    pub fn extract_waiter(&self, waiter: usize) -> Arc> {
         let mut info = self.info.lock();
         debug_assert!(!info.complete);
         // Remove the waiter from the list of waiters
@@ -258,286 +164,6 @@ impl<'tcx> QueryLatch<'tcx> {
     }
 }
 
-/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
-type Waiter = (QueryJobId, usize);
-
-/// Visits all the non-resumable and resumable waiters of a query.
-/// Only waiters in a query are visited.
-/// `visit` is called for every waiter and is passed a query waiting on `query_ref`
-/// and a span indicating the reason the query waited on `query_ref`.
-/// If `visit` returns Some, this function returns.
-/// For visits of non-resumable waiters it returns the return value of `visit`.
-/// For visits of resumable waiters it returns Some(Some(Waiter)) which has the
-/// required information to resume the waiter.
-/// If all `visit` calls returns None, this function also returns None.
-fn visit_waiters<'tcx, F>(
-    query_map: &QueryMap<'tcx>,
-    query: QueryJobId,
-    mut visit: F,
-) -> Option>
-where
-    F: FnMut(Span, QueryJobId) -> Option>,
-{
-    // Visit the parent query which is a non-resumable waiter since it's on the same stack
-    if let Some(parent) = query.parent(query_map)
-        && let Some(cycle) = visit(query.span(query_map), parent)
-    {
-        return Some(cycle);
-    }
-
-    // Visit the explicit waiters which use condvars and are resumable
-    if let Some(latch) = query.latch(query_map) {
-        for (i, waiter) in latch.info.lock().waiters.iter().enumerate() {
-            if let Some(waiter_query) = waiter.query {
-                if visit(waiter.span, waiter_query).is_some() {
-                    // Return a value which indicates that this waiter can be resumed
-                    return Some(Some((query, i)));
-                }
-            }
-        }
-    }
-
-    None
-}
-
-/// Look for query cycles by doing a depth first search starting at `query`.
-/// `span` is the reason for the `query` to execute. This is initially DUMMY_SP.
-/// If a cycle is detected, this initial value is replaced with the span causing
-/// the cycle.
-fn cycle_check<'tcx>(
-    query_map: &QueryMap<'tcx>,
-    query: QueryJobId,
-    span: Span,
-    stack: &mut Vec<(Span, QueryJobId)>,
-    visited: &mut FxHashSet,
-) -> Option> {
-    if !visited.insert(query) {
-        return if let Some(p) = stack.iter().position(|q| q.1 == query) {
-            // We detected a query cycle, fix up the initial span and return Some
-
-            // Remove previous stack entries
-            stack.drain(0..p);
-            // Replace the span for the first query with the cycle cause
-            stack[0].0 = span;
-            Some(None)
-        } else {
-            None
-        };
-    }
-
-    // Query marked as visited is added it to the stack
-    stack.push((span, query));
-
-    // Visit all the waiters
-    let r = visit_waiters(query_map, query, |span, successor| {
-        cycle_check(query_map, successor, span, stack, visited)
-    });
-
-    // Remove the entry in our stack if we didn't find a cycle
-    if r.is_none() {
-        stack.pop();
-    }
-
-    r
-}
-
-/// Finds out if there's a path to the compiler root (aka. code which isn't in a query)
-/// from `query` without going through any of the queries in `visited`.
-/// This is achieved with a depth first search.
-fn connected_to_root<'tcx>(
-    query_map: &QueryMap<'tcx>,
-    query: QueryJobId,
-    visited: &mut FxHashSet,
-) -> bool {
-    // We already visited this or we're deliberately ignoring it
-    if !visited.insert(query) {
-        return false;
-    }
-
-    // This query is connected to the root (it has no query parent), return true
-    if query.parent(query_map).is_none() {
-        return true;
-    }
-
-    visit_waiters(query_map, query, |_, successor| {
-        connected_to_root(query_map, successor, visited).then_some(None)
-    })
-    .is_some()
-}
-
-// Deterministically pick an query from a list
-fn pick_query<'a, 'tcx, T, F>(query_map: &QueryMap<'tcx>, queries: &'a [T], f: F) -> &'a T
-where
-    F: Fn(&T) -> (Span, QueryJobId),
-{
-    // Deterministically pick an entry point
-    // FIXME: Sort this instead
-    queries
-        .iter()
-        .min_by_key(|v| {
-            let (span, query) = f(v);
-            let hash = query.frame(query_map).hash;
-            // Prefer entry points which have valid spans for nicer error messages
-            // We add an integer to the tuple ensuring that entry points
-            // with valid spans are picked first
-            let span_cmp = if span == DUMMY_SP { 1 } else { 0 };
-            (span_cmp, hash)
-        })
-        .unwrap()
-}
-
-/// Looks for query cycles starting from the last query in `jobs`.
-/// If a cycle is found, all queries in the cycle is removed from `jobs` and
-/// the function return true.
-/// If a cycle was not found, the starting query is removed from `jobs` and
-/// the function returns false.
-fn remove_cycle<'tcx>(
-    query_map: &QueryMap<'tcx>,
-    jobs: &mut Vec,
-    wakelist: &mut Vec>>,
-) -> bool {
-    let mut visited = FxHashSet::default();
-    let mut stack = Vec::new();
-    // Look for a cycle starting with the last query in `jobs`
-    if let Some(waiter) =
-        cycle_check(query_map, jobs.pop().unwrap(), DUMMY_SP, &mut stack, &mut visited)
-    {
-        // The stack is a vector of pairs of spans and queries; reverse it so that
-        // the earlier entries require later entries
-        let (mut spans, queries): (Vec<_>, Vec<_>) = stack.into_iter().rev().unzip();
-
-        // Shift the spans so that queries are matched with the span for their waitee
-        spans.rotate_right(1);
-
-        // Zip them back together
-        let mut stack: Vec<_> = iter::zip(spans, queries).collect();
-
-        // Remove the queries in our cycle from the list of jobs to look at
-        for r in &stack {
-            if let Some(pos) = jobs.iter().position(|j| j == &r.1) {
-                jobs.remove(pos);
-            }
-        }
-
-        // Find the queries in the cycle which are
-        // connected to queries outside the cycle
-        let entry_points = stack
-            .iter()
-            .filter_map(|&(span, query)| {
-                if query.parent(query_map).is_none() {
-                    // This query is connected to the root (it has no query parent)
-                    Some((span, query, None))
-                } else {
-                    let mut waiters = Vec::new();
-                    // Find all the direct waiters who lead to the root
-                    visit_waiters(query_map, query, |span, waiter| {
-                        // Mark all the other queries in the cycle as already visited
-                        let mut visited = FxHashSet::from_iter(stack.iter().map(|q| q.1));
-
-                        if connected_to_root(query_map, waiter, &mut visited) {
-                            waiters.push((span, waiter));
-                        }
-
-                        None
-                    });
-                    if waiters.is_empty() {
-                        None
-                    } else {
-                        // Deterministically pick one of the waiters to show to the user
-                        let waiter = *pick_query(query_map, &waiters, |s| *s);
-                        Some((span, query, Some(waiter)))
-                    }
-                }
-            })
-            .collect::)>>();
-
-        // Deterministically pick an entry point
-        let (_, entry_point, usage) = pick_query(query_map, &entry_points, |e| (e.0, e.1));
-
-        // Shift the stack so that our entry point is first
-        let entry_point_pos = stack.iter().position(|(_, query)| query == entry_point);
-        if let Some(pos) = entry_point_pos {
-            stack.rotate_left(pos);
-        }
-
-        let usage = usage.as_ref().map(|(span, query)| (*span, query.frame(query_map)));
-
-        // Create the cycle error
-        let error = CycleError {
-            usage,
-            cycle: stack
-                .iter()
-                .map(|&(s, ref q)| QueryInfo { span: s, frame: q.frame(query_map) })
-                .collect(),
-        };
-
-        // We unwrap `waiter` here since there must always be one
-        // edge which is resumable / waited using a query latch
-        let (waitee_query, waiter_idx) = waiter.unwrap();
-
-        // Extract the waiter we want to resume
-        let waiter = waitee_query.latch(query_map).unwrap().extract_waiter(waiter_idx);
-
-        // Set the cycle error so it will be picked up when resumed
-        *waiter.cycle.lock() = Some(error);
-
-        // Put the waiter on the list of things to resume
-        wakelist.push(waiter);
-
-        true
-    } else {
-        false
-    }
-}
-
-/// Detects query cycles by using depth first search over all active query jobs.
-/// If a query cycle is found it will break the cycle by finding an edge which
-/// uses a query latch and then resuming that waiter.
-/// There may be multiple cycles involved in a deadlock, so this searches
-/// all active queries for cycles before finally resuming all the waiters at once.
-pub fn break_query_cycles<'tcx>(query_map: QueryMap<'tcx>, registry: &rustc_thread_pool::Registry) {
-    let mut wakelist = Vec::new();
-    // It is OK per the comments:
-    // - https://github.com/rust-lang/rust/pull/131200#issuecomment-2798854932
-    // - https://github.com/rust-lang/rust/pull/131200#issuecomment-2798866392
-    #[allow(rustc::potential_query_instability)]
-    let mut jobs: Vec = query_map.keys().cloned().collect();
-
-    let mut found_cycle = false;
-
-    while jobs.len() > 0 {
-        if remove_cycle(&query_map, &mut jobs, &mut wakelist) {
-            found_cycle = true;
-        }
-    }
-
-    // Check that a cycle was found. It is possible for a deadlock to occur without
-    // a query cycle if a query which can be waited on uses Rayon to do multithreading
-    // internally. Such a query (X) may be executing on 2 threads (A and B) and A may
-    // wait using Rayon on B. Rayon may then switch to executing another query (Y)
-    // which in turn will wait on X causing a deadlock. We have a false dependency from
-    // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
-    // only considers the true dependency and won't detect a cycle.
-    if !found_cycle {
-        panic!(
-            "deadlock detected as we're unable to find a query cycle to break\n\
-            current query map:\n{:#?}",
-            query_map
-        );
-    }
-
-    // Mark all the thread we're about to wake up as unblocked. This needs to be done before
-    // we wake the threads up as otherwise Rayon could detect a deadlock if a thread we
-    // resumed fell asleep and this thread had yet to mark the remaining threads as unblocked.
-    for _ in 0..wakelist.len() {
-        rustc_thread_pool::mark_unblocked(registry);
-    }
-
-    for waiter in wakelist.into_iter() {
-        waiter.condvar.notify_one();
-    }
-}
-
 #[inline(never)]
 #[cold]
 pub fn report_cycle<'a>(
@@ -588,61 +214,3 @@ pub fn report_cycle<'a>(
 
     sess.dcx().create_err(cycle_diag)
 }
-
-pub fn print_query_stack<'tcx, Qcx: QueryContext<'tcx>>(
-    qcx: Qcx,
-    mut current_query: Option,
-    dcx: DiagCtxtHandle<'_>,
-    limit_frames: Option,
-    mut file: Option,
-) -> usize {
-    // Be careful relying on global state here: this code is called from
-    // a panic hook, which means that the global `DiagCtxt` may be in a weird
-    // state if it was responsible for triggering the panic.
-    let mut count_printed = 0;
-    let mut count_total = 0;
-
-    // Make use of a partial query map if we fail to take locks collecting active queries.
-    let query_map = match qcx.collect_active_jobs_from_all_queries(false) {
-        Ok(query_map) => query_map,
-        Err(query_map) => query_map,
-    };
-
-    if let Some(ref mut file) = file {
-        let _ = writeln!(file, "\n\nquery stack during panic:");
-    }
-    while let Some(query) = current_query {
-        let Some(query_info) = query_map.get(&query) else {
-            break;
-        };
-        let query_extra = query_info.frame.info.extract();
-        if Some(count_printed) < limit_frames || limit_frames.is_none() {
-            // Only print to stderr as many stack frames as `num_frames` when present.
-            dcx.struct_failure_note(format!(
-                "#{} [{:?}] {}",
-                count_printed, query_info.frame.dep_kind, query_extra.description
-            ))
-            .with_span(query_info.job.span)
-            .emit();
-            count_printed += 1;
-        }
-
-        if let Some(ref mut file) = file {
-            let _ = writeln!(
-                file,
-                "#{} [{}] {}",
-                count_total,
-                qcx.dep_context().dep_kind_vtable(query_info.frame.dep_kind).name,
-                query_extra.description
-            );
-        }
-
-        current_query = query_info.job.parent;
-        count_total += 1;
-    }
-
-    if let Some(ref mut file) = file {
-        let _ = writeln!(file, "end of query stack");
-    }
-    count_total
-}
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index f4a3fda7e372..6067f7dafd5c 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -15,10 +15,7 @@ use rustc_span::def_id::DefId;
 pub use self::caches::{
     DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache,
 };
-pub use self::job::{
-    QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, break_query_cycles,
-    print_query_stack, report_cycle,
-};
+pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryWaiter, report_cycle};
 pub use self::plumbing::*;
 use crate::dep_graph::{DepKind, DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
 
@@ -52,7 +49,7 @@ pub struct QueryStackFrame {
     pub dep_kind: DepKind,
     /// This hash is used to deterministically pick
     /// a query to remove cycles in the parallel compiler.
-    hash: Hash64,
+    pub hash: Hash64,
     pub def_id: Option,
     /// A def-id that is extracted from a `Ty` in a query key
     pub def_id_for_ty_in_cycle: Option,
@@ -161,11 +158,6 @@ pub trait QueryContext<'tcx>: HasDepContext {
     /// a token while waiting on a query.
     fn jobserver_proxy(&self) -> &Proxy;
 
-    fn collect_active_jobs_from_all_queries(
-        self,
-        require_complete: bool,
-    ) -> Result, QueryMap<'tcx>>;
-
     /// Load a side effect associated to the node in the previous session.
     fn load_side_effect(
         self,

From c750a1f5c49ba99f15de43fb651c7958012a12f2 Mon Sep 17 00:00:00 2001
From: AprilNEA 
Date: Mon, 9 Feb 2026 19:31:25 +0000
Subject: [PATCH 1530/1843] Add regression test for type_const with unit struct
 ctor under mGCA

Unit struct constructors used as the RHS of a `type const` associated
const used to ICE during normalization because they were lowered as
`Const::new_unevaluated` with a Ctor def_id. The compiler now properly
constructs a concrete ValTree value for const constructors.
---
 .../mgca/type-const-ctor-148953.rs            | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 tests/ui/const-generics/mgca/type-const-ctor-148953.rs

diff --git a/tests/ui/const-generics/mgca/type-const-ctor-148953.rs b/tests/ui/const-generics/mgca/type-const-ctor-148953.rs
new file mode 100644
index 000000000000..bdd3dcf8618f
--- /dev/null
+++ b/tests/ui/const-generics/mgca/type-const-ctor-148953.rs
@@ -0,0 +1,31 @@
+//! Regression test for 
+//!
+//! Unit struct constructors used as the RHS of a `type const` associated
+//! const used to ICE during normalization because they were lowered as
+//! `Const::new_unevaluated` with a Ctor def_id. Fixed by adding proper const
+//! constructor support that produces a concrete ValTree value instead.
+
+//@ check-pass
+
+#![feature(min_generic_const_args, adt_const_params)]
+#![expect(incomplete_features)]
+
+use std::marker::ConstParamTy;
+
+#[derive(ConstParamTy, PartialEq, Eq)]
+struct S;
+
+impl S {
+    type const N: S = S;
+}
+
+#[derive(ConstParamTy, PartialEq, Eq)]
+enum E {
+    V,
+}
+
+impl E {
+    type const M: E = { E::V };
+}
+
+fn main() {}

From 923de04f6a032ccd475dd271287db8faebf42c3f Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Tue, 10 Feb 2026 11:53:47 +1100
Subject: [PATCH 1531/1843] Move `rustc_middle::query::values` to
 `rustc_query_impl`.

Because all uses are now in `rustc_query_impl`. This was made possible
by the previous commit. Less code in `rustc_middle`, hooray.
---
 Cargo.lock                                             |  1 +
 compiler/rustc_middle/src/query/mod.rs                 |  1 -
 compiler/rustc_query_impl/Cargo.toml                   |  1 +
 compiler/rustc_query_impl/src/lib.rs                   |  3 ++-
 .../src/query => rustc_query_impl/src}/values.rs       | 10 +++++-----
 5 files changed, 9 insertions(+), 7 deletions(-)
 rename compiler/{rustc_middle/src/query => rustc_query_impl/src}/values.rs (98%)

diff --git a/Cargo.lock b/Cargo.lock
index 7230d633602d..741c51dcb3e7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4493,6 +4493,7 @@ name = "rustc_query_impl"
 version = "0.0.0"
 dependencies = [
  "measureme",
+ "rustc_abi",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_hashes",
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index a7707cf48c49..24a38e70ff6f 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -13,7 +13,6 @@ mod keys;
 pub mod on_disk_cache;
 #[macro_use]
 pub mod plumbing;
-pub mod values;
 
 pub fn describe_as_module(def_id: impl Into, tcx: TyCtxt<'_>) -> String {
     let def_id = def_id.into();
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index 511337090d7f..ee50ec073bc8 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
 measureme = "12.0.1"
+rustc_abi = { path = "../rustc_abi" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_hashes = { path = "../rustc_hashes" }
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index c1ead8da344b..2241c3f44087 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -19,7 +19,6 @@ use rustc_middle::queries::{
 use rustc_middle::query::AsLocalKey;
 use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache};
 use rustc_middle::query::plumbing::{HashResult, QuerySystem, QuerySystemFns, QueryVTable};
-use rustc_middle::query::values::Value;
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::query::{
@@ -32,6 +31,7 @@ pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all};
 use crate::plumbing::{encode_all_query_results, try_mark_green};
 use crate::profiling_support::QueryKeyStringCache;
 pub use crate::profiling_support::alloc_self_profile_query_strings;
+use crate::values::Value;
 
 mod error;
 mod execution;
@@ -39,6 +39,7 @@ mod job;
 #[macro_use]
 mod plumbing;
 mod profiling_support;
+mod values;
 
 #[derive(ConstParamTy)] // Allow this struct to be used for const-generic values.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
diff --git a/compiler/rustc_middle/src/query/values.rs b/compiler/rustc_query_impl/src/values.rs
similarity index 98%
rename from compiler/rustc_middle/src/query/values.rs
rename to compiler/rustc_query_impl/src/values.rs
index 0828758cc94e..2fe3808f3d6e 100644
--- a/compiler/rustc_middle/src/query/values.rs
+++ b/compiler/rustc_query_impl/src/values.rs
@@ -7,15 +7,15 @@ use rustc_errors::codes::*;
 use rustc_errors::{Applicability, MultiSpan, pluralize, struct_span_code_err};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
+use rustc_middle::dep_graph::dep_kinds;
+use rustc_middle::query::plumbing::CyclePlaceholder;
+use rustc_middle::ty::{self, Representability, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug};
 use rustc_query_system::query::{CycleError, report_cycle};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{ErrorGuaranteed, Span};
 
-use crate::dep_graph::dep_kinds;
-use crate::query::plumbing::CyclePlaceholder;
-use crate::ty::{self, Representability, Ty, TyCtxt};
-
-pub trait Value<'tcx>: Sized {
+pub(crate) trait Value<'tcx>: Sized {
     fn from_cycle_error(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed)
     -> Self;
 }

From a34317e5a5641385706926b109bd259cbc8ec9b7 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote 
Date: Tue, 10 Feb 2026 14:05:13 +1100
Subject: [PATCH 1532/1843] Move `report_cycle`.

From `rustc_query_system::query::job` to `rustc_query_impl::job`.
---
 Cargo.lock                                   |  1 +
 compiler/rustc_query_impl/Cargo.toml         |  1 +
 compiler/rustc_query_impl/src/error.rs       | 57 +++++++++++++++++++
 compiler/rustc_query_impl/src/execution.rs   |  3 +-
 compiler/rustc_query_impl/src/job.rs         | 56 +++++++++++++++++-
 compiler/rustc_query_impl/src/values.rs      |  4 +-
 compiler/rustc_query_system/src/error.rs     | 60 +-------------------
 compiler/rustc_query_system/src/query/job.rs | 55 ------------------
 compiler/rustc_query_system/src/query/mod.rs |  2 +-
 9 files changed, 120 insertions(+), 119 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 741c51dcb3e7..3019ff581f37 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4503,6 +4503,7 @@ dependencies = [
  "rustc_middle",
  "rustc_query_system",
  "rustc_serialize",
+ "rustc_session",
  "rustc_span",
  "rustc_thread_pool",
  "tracing",
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index ee50ec073bc8..a8edc1129481 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -16,6 +16,7 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
+rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_thread_pool = { path = "../rustc_thread_pool" }
 tracing = "0.1"
diff --git a/compiler/rustc_query_impl/src/error.rs b/compiler/rustc_query_impl/src/error.rs
index b109172d8e49..6d3eb2950982 100644
--- a/compiler/rustc_query_impl/src/error.rs
+++ b/compiler/rustc_query_impl/src/error.rs
@@ -1,3 +1,4 @@
+use rustc_errors::codes::*;
 use rustc_hir::limit::Limit;
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::{Span, Symbol};
@@ -22,3 +23,59 @@ pub(crate) struct QueryOverflowNote {
     pub desc: String,
     pub depth: usize,
 }
+
+#[derive(Subdiagnostic)]
+#[note("...which requires {$desc}...")]
+pub(crate) struct CycleStack {
+    #[primary_span]
+    pub span: Span,
+    pub desc: String,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum StackCount {
+    #[note("...which immediately requires {$stack_bottom} again")]
+    Single,
+    #[note("...which again requires {$stack_bottom}, completing the cycle")]
+    Multiple,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum Alias {
+    #[note("type aliases cannot be recursive")]
+    #[help("consider using a struct, enum, or union instead to break the cycle")]
+    #[help(
+        "see  for more information"
+    )]
+    Ty,
+    #[note("trait aliases cannot be recursive")]
+    Trait,
+}
+
+#[derive(Subdiagnostic)]
+#[note("cycle used when {$usage}")]
+pub(crate) struct CycleUsage {
+    #[primary_span]
+    pub span: Span,
+    pub usage: String,
+}
+
+#[derive(Diagnostic)]
+#[diag("cycle detected when {$stack_bottom}", code = E0391)]
+pub(crate) struct Cycle {
+    #[primary_span]
+    pub span: Span,
+    pub stack_bottom: String,
+    #[subdiagnostic]
+    pub cycle_stack: Vec,
+    #[subdiagnostic]
+    pub stack_count: StackCount,
+    #[subdiagnostic]
+    pub alias: Option,
+    #[subdiagnostic]
+    pub cycle_usage: Option,
+    #[note(
+        "see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information"
+    )]
+    pub note_span: (),
+}
diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs
index 6f16932cb2eb..d7e95d43ecaa 100644
--- a/compiler/rustc_query_impl/src/execution.rs
+++ b/compiler/rustc_query_impl/src/execution.rs
@@ -11,12 +11,11 @@ use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext};
 use rustc_query_system::query::{
     ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch,
     QueryMode, QueryStackDeferred, QueryStackFrame, QueryState, incremental_verify_ich,
-    report_cycle,
 };
 use rustc_span::{DUMMY_SP, Span};
 
 use crate::dep_graph::{DepContext, DepNode, DepNodeIndex};
-use crate::job::{QueryJobInfo, QueryMap, find_cycle_in_stack};
+use crate::job::{QueryJobInfo, QueryMap, find_cycle_in_stack, report_cycle};
 use crate::{QueryCtxt, QueryFlags, SemiDynamicQueryDispatcher};
 
 #[inline]
diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_query_impl/src/job.rs
index cb0a13d32ad2..f1eba0f76d17 100644
--- a/compiler/rustc_query_impl/src/job.rs
+++ b/compiler/rustc_query_impl/src/job.rs
@@ -3,11 +3,13 @@ use std::iter;
 use std::sync::Arc;
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_errors::DiagCtxtHandle;
+use rustc_errors::{Diag, DiagCtxtHandle};
+use rustc_hir::def::DefKind;
 use rustc_query_system::query::{
     CycleError, QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryStackDeferred, QueryStackFrame,
     QueryWaiter,
 };
+use rustc_session::Session;
 use rustc_span::{DUMMY_SP, Span};
 
 use crate::QueryCtxt;
@@ -444,3 +446,55 @@ pub fn print_query_stack<'tcx>(
     }
     count_total
 }
+
+#[inline(never)]
+#[cold]
+pub(crate) fn report_cycle<'a>(
+    sess: &'a Session,
+    CycleError { usage, cycle: stack }: &CycleError,
+) -> Diag<'a> {
+    assert!(!stack.is_empty());
+
+    let span = stack[0].frame.info.default_span(stack[1 % stack.len()].span);
+
+    let mut cycle_stack = Vec::new();
+
+    use crate::error::StackCount;
+    let stack_count = if stack.len() == 1 { StackCount::Single } else { StackCount::Multiple };
+
+    for i in 1..stack.len() {
+        let frame = &stack[i].frame;
+        let span = frame.info.default_span(stack[(i + 1) % stack.len()].span);
+        cycle_stack
+            .push(crate::error::CycleStack { span, desc: frame.info.description.to_owned() });
+    }
+
+    let mut cycle_usage = None;
+    if let Some((span, ref query)) = *usage {
+        cycle_usage = Some(crate::error::CycleUsage {
+            span: query.info.default_span(span),
+            usage: query.info.description.to_string(),
+        });
+    }
+
+    let alias =
+        if stack.iter().all(|entry| matches!(entry.frame.info.def_kind, Some(DefKind::TyAlias))) {
+            Some(crate::error::Alias::Ty)
+        } else if stack.iter().all(|entry| entry.frame.info.def_kind == Some(DefKind::TraitAlias)) {
+            Some(crate::error::Alias::Trait)
+        } else {
+            None
+        };
+
+    let cycle_diag = crate::error::Cycle {
+        span,
+        cycle_stack,
+        stack_bottom: stack[0].frame.info.description.to_owned(),
+        alias,
+        cycle_usage,
+        stack_count,
+        note_span: (),
+    };
+
+    sess.dcx().create_err(cycle_diag)
+}
diff --git a/compiler/rustc_query_impl/src/values.rs b/compiler/rustc_query_impl/src/values.rs
index 2fe3808f3d6e..783e7a10d12a 100644
--- a/compiler/rustc_query_impl/src/values.rs
+++ b/compiler/rustc_query_impl/src/values.rs
@@ -11,10 +11,12 @@ use rustc_middle::dep_graph::dep_kinds;
 use rustc_middle::query::plumbing::CyclePlaceholder;
 use rustc_middle::ty::{self, Representability, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
-use rustc_query_system::query::{CycleError, report_cycle};
+use rustc_query_system::query::CycleError;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{ErrorGuaranteed, Span};
 
+use crate::job::report_cycle;
+
 pub(crate) trait Value<'tcx>: Sized {
     fn from_cycle_error(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed)
     -> Self;
diff --git a/compiler/rustc_query_system/src/error.rs b/compiler/rustc_query_system/src/error.rs
index adce85bf31be..f48afe6f75fa 100644
--- a/compiler/rustc_query_system/src/error.rs
+++ b/compiler/rustc_query_system/src/error.rs
@@ -1,62 +1,4 @@
-use rustc_errors::codes::*;
-use rustc_macros::{Diagnostic, Subdiagnostic};
-use rustc_span::Span;
-
-#[derive(Subdiagnostic)]
-#[note("...which requires {$desc}...")]
-pub(crate) struct CycleStack {
-    #[primary_span]
-    pub span: Span,
-    pub desc: String,
-}
-
-#[derive(Subdiagnostic)]
-pub(crate) enum StackCount {
-    #[note("...which immediately requires {$stack_bottom} again")]
-    Single,
-    #[note("...which again requires {$stack_bottom}, completing the cycle")]
-    Multiple,
-}
-
-#[derive(Subdiagnostic)]
-pub(crate) enum Alias {
-    #[note("type aliases cannot be recursive")]
-    #[help("consider using a struct, enum, or union instead to break the cycle")]
-    #[help(
-        "see  for more information"
-    )]
-    Ty,
-    #[note("trait aliases cannot be recursive")]
-    Trait,
-}
-
-#[derive(Subdiagnostic)]
-#[note("cycle used when {$usage}")]
-pub(crate) struct CycleUsage {
-    #[primary_span]
-    pub span: Span,
-    pub usage: String,
-}
-
-#[derive(Diagnostic)]
-#[diag("cycle detected when {$stack_bottom}", code = E0391)]
-pub(crate) struct Cycle {
-    #[primary_span]
-    pub span: Span,
-    pub stack_bottom: String,
-    #[subdiagnostic]
-    pub cycle_stack: Vec,
-    #[subdiagnostic]
-    pub stack_count: StackCount,
-    #[subdiagnostic]
-    pub alias: Option,
-    #[subdiagnostic]
-    pub cycle_usage: Option,
-    #[note(
-        "see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information"
-    )]
-    pub note_span: (),
-}
+use rustc_macros::Diagnostic;
 
 #[derive(Diagnostic)]
 #[diag("internal compiler error: reentrant incremental verify failure, suppressing message")]
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index e8acb8064f7e..7349fe223ef9 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -4,13 +4,9 @@ use std::num::NonZero;
 use std::sync::Arc;
 
 use parking_lot::{Condvar, Mutex};
-use rustc_errors::Diag;
-use rustc_hir::def::DefKind;
-use rustc_session::Session;
 use rustc_span::Span;
 
 use super::{QueryStackDeferred, QueryStackFrameExtra};
-use crate::error::CycleStack;
 use crate::query::plumbing::CycleError;
 use crate::query::{QueryContext, QueryStackFrame};
 
@@ -163,54 +159,3 @@ impl<'tcx> QueryLatch<'tcx> {
         info.waiters.remove(waiter)
     }
 }
-
-#[inline(never)]
-#[cold]
-pub fn report_cycle<'a>(
-    sess: &'a Session,
-    CycleError { usage, cycle: stack }: &CycleError,
-) -> Diag<'a> {
-    assert!(!stack.is_empty());
-
-    let span = stack[0].frame.info.default_span(stack[1 % stack.len()].span);
-
-    let mut cycle_stack = Vec::new();
-
-    use crate::error::StackCount;
-    let stack_count = if stack.len() == 1 { StackCount::Single } else { StackCount::Multiple };
-
-    for i in 1..stack.len() {
-        let frame = &stack[i].frame;
-        let span = frame.info.default_span(stack[(i + 1) % stack.len()].span);
-        cycle_stack.push(CycleStack { span, desc: frame.info.description.to_owned() });
-    }
-
-    let mut cycle_usage = None;
-    if let Some((span, ref query)) = *usage {
-        cycle_usage = Some(crate::error::CycleUsage {
-            span: query.info.default_span(span),
-            usage: query.info.description.to_string(),
-        });
-    }
-
-    let alias =
-        if stack.iter().all(|entry| matches!(entry.frame.info.def_kind, Some(DefKind::TyAlias))) {
-            Some(crate::error::Alias::Ty)
-        } else if stack.iter().all(|entry| entry.frame.info.def_kind == Some(DefKind::TraitAlias)) {
-            Some(crate::error::Alias::Trait)
-        } else {
-            None
-        };
-
-    let cycle_diag = crate::error::Cycle {
-        span,
-        cycle_stack,
-        stack_bottom: stack[0].frame.info.description.to_owned(),
-        alias,
-        cycle_usage,
-        stack_count,
-        note_span: (),
-    };
-
-    sess.dcx().create_err(cycle_diag)
-}
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 6067f7dafd5c..c33af941f802 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -15,7 +15,7 @@ use rustc_span::def_id::DefId;
 pub use self::caches::{
     DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache,
 };
-pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryWaiter, report_cycle};
+pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryWaiter};
 pub use self::plumbing::*;
 use crate::dep_graph::{DepKind, DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
 

From 0ebd56ebbbf3e392c4fc81865ce89fc6f765a885 Mon Sep 17 00:00:00 2001
From: Oli Scherer 
Date: Thu, 4 Dec 2025 11:49:56 +0000
Subject: [PATCH 1533/1843] Remove accidental const stability marker on a
 struct

---
 library/core/src/array/drain.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/library/core/src/array/drain.rs b/library/core/src/array/drain.rs
index 1c6137191324..17792dca583d 100644
--- a/library/core/src/array/drain.rs
+++ b/library/core/src/array/drain.rs
@@ -31,7 +31,6 @@ impl<'l, 'f, T, U, const N: usize, F: FnMut(T) -> U> Drain<'l, 'f, T, N, F> {
 }
 
 /// See [`Drain::new`]; this is our fake iterator.
-#[rustc_const_unstable(feature = "array_try_map", issue = "79711")]
 #[unstable(feature = "array_try_map", issue = "79711")]
 pub(super) struct Drain<'l, 'f, T, const N: usize, F> {
     // FIXME(const-hack): This is essentially a slice::IterMut<'static>, replace when possible.

From 00fef81964699283c09a244c6fb53840eada28e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1534/1843] Port `rustc_expected_cgu_reuse` to the new
 attribute parser

---
 Cargo.lock                                    |   1 -
 .../src/attributes/rustc_internal.rs          | 145 +++++++++++++++-
 compiler/rustc_attr_parsing/src/context.rs    |   1 +
 .../src/session_diagnostics.rs                |   9 +
 compiler/rustc_codegen_ssa/Cargo.toml         |   1 -
 .../src/assert_module_sources.rs              | 160 ++++++++----------
 compiler/rustc_codegen_ssa/src/errors.rs      |  30 +---
 .../rustc_hir/src/attrs/data_structures.rs    |  16 ++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |   1 +
 compiler/rustc_passes/src/check_attr.rs       |   4 +-
 10 files changed, 244 insertions(+), 124 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 545c776b4805..1a930b64f458 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3687,7 +3687,6 @@ dependencies = [
  "serde_json",
  "smallvec",
  "tempfile",
- "thin-vec",
  "thorin-dwp",
  "tracing",
  "wasm-encoder 0.219.2",
diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 0fd8c8a04a57..e5ea7fde3954 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -2,7 +2,8 @@ use std::path::PathBuf;
 
 use rustc_ast::{LitIntType, LitKind, MetaItemLit};
 use rustc_hir::attrs::{
-    BorrowckGraphvizFormatKind, RustcCleanAttribute, RustcCleanQueries, RustcLayoutType,
+    BorrowckGraphvizFormatKind, CguFields, CguKind, DivergingBlockBehavior,
+    DivergingFallbackBehavior, RustcCleanAttribute, RustcCleanQueries, RustcLayoutType,
     RustcMirKind,
 };
 use rustc_session::errors;
@@ -10,7 +11,9 @@ use rustc_span::Symbol;
 
 use super::prelude::*;
 use super::util::parse_single_integer;
-use crate::session_diagnostics::{AttributeRequiresOpt, RustcScalableVectorCountOutOfRange};
+use crate::session_diagnostics::{
+    AttributeRequiresOpt, CguFieldsMissing, RustcScalableVectorCountOutOfRange,
+};
 
 pub(crate) struct RustcMainParser;
 
@@ -204,6 +207,144 @@ impl NoArgsAttributeParser for RustcLintOptTyParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintOptTy;
 }
 
+fn parse_cgu_fields(
+    cx: &mut AcceptContext<'_, '_, S>,
+    args: &ArgParser,
+    accepts_kind: bool,
+) -> Option<(Symbol, Symbol, Option)> {
+    let Some(args) = args.list() else {
+        cx.expected_list(cx.attr_span, args);
+        return None;
+    };
+
+    let mut cfg = None::<(Symbol, Span)>;
+    let mut module = None::<(Symbol, Span)>;
+    let mut kind = None::<(Symbol, Span)>;
+
+    for arg in args.mixed() {
+        let Some(arg) = arg.meta_item() else {
+            cx.expected_name_value(args.span, None);
+            continue;
+        };
+
+        let res = match arg.ident().map(|i| i.name) {
+            Some(sym::cfg) => &mut cfg,
+            Some(sym::module) => &mut module,
+            Some(sym::kind) if accepts_kind => &mut kind,
+            _ => {
+                cx.expected_specific_argument(
+                    arg.path().span(),
+                    if accepts_kind {
+                        &[sym::cfg, sym::module, sym::kind]
+                    } else {
+                        &[sym::cfg, sym::module]
+                    },
+                );
+                continue;
+            }
+        };
+
+        let Some(i) = arg.args().name_value() else {
+            cx.expected_name_value(arg.span(), None);
+            continue;
+        };
+
+        let Some(str) = i.value_as_str() else {
+            cx.expected_string_literal(i.value_span, Some(i.value_as_lit()));
+            continue;
+        };
+
+        if res.is_some() {
+            cx.duplicate_key(arg.span(), arg.ident().unwrap().name);
+            continue;
+        }
+
+        *res = Some((str, i.value_span));
+    }
+
+    let Some((cfg, _)) = cfg else {
+        cx.emit_err(CguFieldsMissing { span: args.span, name: &cx.attr_path, field: sym::cfg });
+        return None;
+    };
+    let Some((module, _)) = module else {
+        cx.emit_err(CguFieldsMissing { span: args.span, name: &cx.attr_path, field: sym::module });
+        return None;
+    };
+    let kind = if let Some((kind, span)) = kind {
+        Some(match kind {
+            sym::no => CguKind::No,
+            sym::pre_dash_lto => CguKind::PreDashLto,
+            sym::post_dash_lto => CguKind::PostDashLto,
+            sym::any => CguKind::Any,
+            _ => {
+                cx.expected_specific_argument_strings(
+                    span,
+                    &[sym::no, sym::pre_dash_lto, sym::post_dash_lto, sym::any],
+                );
+                return None;
+            }
+        })
+    } else {
+        // return None so that an unwrap for the attributes that need it is ok.
+        if accepts_kind {
+            cx.emit_err(CguFieldsMissing {
+                span: args.span,
+                name: &cx.attr_path,
+                field: sym::kind,
+            });
+            return None;
+        };
+
+        None
+    };
+
+    Some((cfg, module, kind))
+}
+
+#[derive(Default)]
+pub(crate) struct RustcCguTestAttributeParser {
+    items: ThinVec<(Span, CguFields)>,
+}
+
+impl AttributeParser for RustcCguTestAttributeParser {
+    const ATTRIBUTES: AcceptMapping = &[
+        (
+            &[sym::rustc_partition_reused],
+            template!(List: &[r#"cfg = "...", module = "...""#]),
+            |this, cx, args| {
+                this.items.extend(parse_cgu_fields(cx, args, false).map(|(cfg, module, _)| {
+                    (cx.attr_span, CguFields::PartitionReused { cfg, module })
+                }));
+            },
+        ),
+        (
+            &[sym::rustc_partition_codegened],
+            template!(List: &[r#"cfg = "...", module = "...""#]),
+            |this, cx, args| {
+                this.items.extend(parse_cgu_fields(cx, args, false).map(|(cfg, module, _)| {
+                    (cx.attr_span, CguFields::PartitionCodegened { cfg, module })
+                }));
+            },
+        ),
+        (
+            &[sym::rustc_expected_cgu_reuse],
+            template!(List: &[r#"cfg = "...", module = "...", kind = "...""#]),
+            |this, cx, args| {
+                this.items.extend(parse_cgu_fields(cx, args, true).map(|(cfg, module, kind)| {
+                    // unwrap ok because if not given, we return None in `parse_cgu_fields`.
+                    (cx.attr_span, CguFields::ExpectedCguReuse { cfg, module, kind: kind.unwrap() })
+                }));
+            },
+        ),
+    ];
+
+    const ALLOWED_TARGETS: AllowedTargets =
+        AllowedTargets::AllowList(&[Allow(Target::Mod), Allow(Target::Crate)]);
+
+    fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option {
+        Some(AttributeKind::RustcCguTestAttr(self.items))
+    }
+}
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 412f960fb7cf..f53f48b337b5 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -149,6 +149,7 @@ attribute_parsers!(
         NakedParser,
         StabilityParser,
         UsedParser,
+        RustcCguTestAttributeParser,
         // tidy-alphabetical-end
 
         // tidy-alphabetical-start
diff --git a/compiler/rustc_attr_parsing/src/session_diagnostics.rs b/compiler/rustc_attr_parsing/src/session_diagnostics.rs
index 6ca47e8d698b..2eb585671fff 100644
--- a/compiler/rustc_attr_parsing/src/session_diagnostics.rs
+++ b/compiler/rustc_attr_parsing/src/session_diagnostics.rs
@@ -45,6 +45,15 @@ pub(crate) struct DocAliasStartEnd<'a> {
     pub attr_str: &'a str,
 }
 
+#[derive(Diagnostic)]
+#[diag("`#[{$name})]` is missing a `{$field}` argument")]
+pub(crate) struct CguFieldsMissing<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub name: &'a AttrPath,
+    pub field: Symbol,
+}
+
 #[derive(Diagnostic)]
 #[diag("`#![doc({$attr_name} = \"...\")]` isn't allowed as a crate-level attribute")]
 pub(crate) struct DocAttrNotCrateLevel {
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 864220635120..c7d7b7429f34 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -36,7 +36,6 @@ rustc_trait_selection = { path = "../rustc_trait_selection" }
 serde_json = "1.0.59"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tempfile = "3.2"
-thin-vec = "0.2.12"
 thorin-dwp = "0.9"
 tracing = "0.1"
 wasm-encoder = "0.219"
diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
index 43e1e135a666..081fe0aa91aa 100644
--- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
+++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
@@ -28,13 +28,13 @@ use std::fmt;
 
 use rustc_data_structures::unord::{UnordMap, UnordSet};
 use rustc_errors::{DiagArgValue, IntoDiagArg};
-use rustc_hir as hir;
+use rustc_hir::attrs::{AttributeKind, CguFields, CguKind};
 use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_hir::{self as hir, find_attr};
 use rustc_middle::mir::mono::CodegenUnitNameBuilder;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
-use rustc_span::{Span, Symbol, sym};
-use thin_vec::ThinVec;
+use rustc_span::{Span, Symbol};
 use tracing::debug;
 
 use crate::errors;
@@ -63,9 +63,7 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>, set_reuse: &dyn Fn(&mut CguReuseTr
             },
         };
 
-        for attr in tcx.hir_attrs(rustc_hir::CRATE_HIR_ID) {
-            ams.check_attr(attr);
-        }
+        ams.check_attrs(tcx.hir_attrs(rustc_hir::CRATE_HIR_ID));
 
         set_reuse(&mut ams.cgu_reuse_tracker);
 
@@ -89,109 +87,91 @@ struct AssertModuleSource<'tcx> {
 }
 
 impl<'tcx> AssertModuleSource<'tcx> {
-    fn check_attr(&mut self, attr: &hir::Attribute) {
-        let (expected_reuse, comp_kind) = if attr.has_name(sym::rustc_partition_reused) {
-            (CguReuse::PreLto, ComparisonKind::AtLeast)
-        } else if attr.has_name(sym::rustc_partition_codegened) {
-            (CguReuse::No, ComparisonKind::Exact)
-        } else if attr.has_name(sym::rustc_expected_cgu_reuse) {
-            match self.field(attr, sym::kind) {
-                sym::no => (CguReuse::No, ComparisonKind::Exact),
-                sym::pre_dash_lto => (CguReuse::PreLto, ComparisonKind::Exact),
-                sym::post_dash_lto => (CguReuse::PostLto, ComparisonKind::Exact),
-                sym::any => (CguReuse::PreLto, ComparisonKind::AtLeast),
-                other => {
-                    self.tcx
-                        .dcx()
-                        .emit_fatal(errors::UnknownReuseKind { span: attr.span(), kind: other });
-                }
+    fn check_attrs(&mut self, attrs: &[hir::Attribute]) {
+        for &(span, cgu_fields) in find_attr!(attrs,
+            AttributeKind::RustcCguTestAttr(e) => e)
+        .into_iter()
+        .flatten()
+        {
+            let (expected_reuse, comp_kind) = match cgu_fields {
+                CguFields::PartitionReused { .. } => (CguReuse::PreLto, ComparisonKind::AtLeast),
+                CguFields::PartitionCodegened { .. } => (CguReuse::No, ComparisonKind::Exact),
+                CguFields::ExpectedCguReuse { kind, .. } => match kind {
+                    CguKind::No => (CguReuse::No, ComparisonKind::Exact),
+                    CguKind::PreDashLto => (CguReuse::PreLto, ComparisonKind::Exact),
+                    CguKind::PostDashLto => (CguReuse::PostLto, ComparisonKind::Exact),
+                    CguKind::Any => (CguReuse::PreLto, ComparisonKind::AtLeast),
+                },
+            };
+            let (CguFields::ExpectedCguReuse { cfg, module, .. }
+            | CguFields::PartitionCodegened { cfg, module }
+            | CguFields::PartitionReused { cfg, module }) = cgu_fields;
+
+            if !self.tcx.sess.opts.unstable_opts.query_dep_graph {
+                self.tcx.dcx().emit_fatal(errors::MissingQueryDepGraph { span });
             }
-        } else {
-            return;
-        };
 
-        if !self.tcx.sess.opts.unstable_opts.query_dep_graph {
-            self.tcx.dcx().emit_fatal(errors::MissingQueryDepGraph { span: attr.span() });
-        }
+            if !self.check_config(cfg) {
+                debug!("check_attr: config does not match, ignoring attr");
+                return;
+            }
 
-        if !self.check_config(attr) {
-            debug!("check_attr: config does not match, ignoring attr");
-            return;
-        }
+            let user_path = module.as_str();
+            let crate_name = self.tcx.crate_name(LOCAL_CRATE);
+            let crate_name = crate_name.as_str();
 
-        let user_path = self.field(attr, sym::module).to_string();
-        let crate_name = self.tcx.crate_name(LOCAL_CRATE).to_string();
+            if !user_path.starts_with(&crate_name) {
+                self.tcx.dcx().emit_fatal(errors::MalformedCguName { span, user_path, crate_name });
+            }
 
-        if !user_path.starts_with(&crate_name) {
-            self.tcx.dcx().emit_fatal(errors::MalformedCguName {
-                span: attr.span(),
-                user_path,
-                crate_name,
-            });
-        }
+            // Split of the "special suffix" if there is one.
+            let (user_path, cgu_special_suffix) = if let Some(index) = user_path.rfind('.') {
+                (&user_path[..index], Some(&user_path[index + 1..]))
+            } else {
+                (&user_path[..], None)
+            };
 
-        // Split of the "special suffix" if there is one.
-        let (user_path, cgu_special_suffix) = if let Some(index) = user_path.rfind('.') {
-            (&user_path[..index], Some(&user_path[index + 1..]))
-        } else {
-            (&user_path[..], None)
-        };
+            let mut iter = user_path.split('-');
 
-        let mut iter = user_path.split('-');
+            // Remove the crate name
+            assert_eq!(iter.next().unwrap(), crate_name);
 
-        // Remove the crate name
-        assert_eq!(iter.next().unwrap(), crate_name);
+            let cgu_path_components = iter.collect::>();
 
-        let cgu_path_components = iter.collect::>();
+            let cgu_name_builder = &mut CodegenUnitNameBuilder::new(self.tcx);
+            let cgu_name = cgu_name_builder.build_cgu_name(
+                LOCAL_CRATE,
+                cgu_path_components,
+                cgu_special_suffix,
+            );
 
-        let cgu_name_builder = &mut CodegenUnitNameBuilder::new(self.tcx);
-        let cgu_name =
-            cgu_name_builder.build_cgu_name(LOCAL_CRATE, cgu_path_components, cgu_special_suffix);
+            debug!("mapping '{user_path}' to cgu name '{cgu_name}'");
 
-        debug!("mapping '{}' to cgu name '{}'", self.field(attr, sym::module), cgu_name);
+            if !self.available_cgus.contains(&cgu_name) {
+                let cgu_names: Vec<&str> =
+                    self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord();
+                self.tcx.dcx().emit_err(errors::NoModuleNamed {
+                    span,
+                    user_path,
+                    cgu_name,
+                    cgu_names: cgu_names.join(", "),
+                });
+            }
 
-        if !self.available_cgus.contains(&cgu_name) {
-            let cgu_names: Vec<&str> =
-                self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord();
-            self.tcx.dcx().emit_err(errors::NoModuleNamed {
-                span: attr.span(),
-                user_path,
+            self.cgu_reuse_tracker.set_expectation(
                 cgu_name,
-                cgu_names: cgu_names.join(", "),
-            });
+                user_path,
+                span,
+                expected_reuse,
+                comp_kind,
+            );
         }
-
-        self.cgu_reuse_tracker.set_expectation(
-            cgu_name,
-            user_path,
-            attr.span(),
-            expected_reuse,
-            comp_kind,
-        );
-    }
-
-    fn field(&self, attr: &hir::Attribute, name: Symbol) -> Symbol {
-        for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
-            if item.has_name(name) {
-                if let Some(value) = item.value_str() {
-                    return value;
-                } else {
-                    self.tcx.dcx().emit_fatal(errors::FieldAssociatedValueExpected {
-                        span: item.span(),
-                        name,
-                    });
-                }
-            }
-        }
-
-        self.tcx.dcx().emit_fatal(errors::NoField { span: attr.span(), name });
     }
 
     /// Scan for a `cfg="foo"` attribute and check whether we have a
     /// cfg flag called `foo`.
-    fn check_config(&self, attr: &hir::Attribute) -> bool {
+    fn check_config(&self, value: Symbol) -> bool {
         let config = &self.tcx.sess.psess.config;
-        let value = self.field(attr, sym::cfg);
         debug!("check_config(config={:?}, value={:?})", config, value);
         if config.iter().any(|&(name, _)| name == value) {
             debug!("check_config: matched");
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index 82a6525adcc9..897b5f957e15 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -42,14 +42,6 @@ pub(crate) struct CguNotRecorded<'a> {
     pub cgu_name: &'a str,
 }
 
-#[derive(Diagnostic)]
-#[diag("unknown cgu-reuse-kind `{$kind}` specified")]
-pub(crate) struct UnknownReuseKind {
-    #[primary_span]
-    pub span: Span,
-    pub kind: Symbol,
-}
-
 #[derive(Diagnostic)]
 #[diag("found CGU-reuse attribute but `-Zquery-dep-graph` was not specified")]
 pub(crate) struct MissingQueryDepGraph {
@@ -61,11 +53,11 @@ pub(crate) struct MissingQueryDepGraph {
 #[diag(
     "found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case)"
 )]
-pub(crate) struct MalformedCguName {
+pub(crate) struct MalformedCguName<'a> {
     #[primary_span]
     pub span: Span,
-    pub user_path: String,
-    pub crate_name: String,
+    pub user_path: &'a str,
+    pub crate_name: &'a str,
 }
 
 #[derive(Diagnostic)]
@@ -78,22 +70,6 @@ pub(crate) struct NoModuleNamed<'a> {
     pub cgu_names: String,
 }
 
-#[derive(Diagnostic)]
-#[diag("associated value expected for `{$name}`")]
-pub(crate) struct FieldAssociatedValueExpected {
-    #[primary_span]
-    pub span: Span,
-    pub name: Symbol,
-}
-
-#[derive(Diagnostic)]
-#[diag("no field `{$name}`")]
-pub(crate) struct NoField {
-    #[primary_span]
-    pub span: Span,
-    pub name: Symbol,
-}
-
 #[derive(Diagnostic)]
 #[diag("failed to write lib.def file: {$error}")]
 pub(crate) struct LibDefWriteFailure {
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 2d3825ccd024..982c69c38afb 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -50,6 +50,19 @@ pub struct EiiDecl {
 }
 
 #[derive(Copy, Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, PrintAttribute)]
+pub enum CguKind {
+    No,
+    PreDashLto,
+    PostDashLto,
+    Any,
+}
+
+#[derive(Copy, Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, PrintAttribute)]
+pub enum CguFields {
+    PartitionReused { cfg: Symbol, module: Symbol },
+    PartitionCodegened { cfg: Symbol, module: Symbol },
+    ExpectedCguReuse { cfg: Symbol, module: Symbol, kind: CguKind },
+}
 pub enum InlineAttr {
     None,
     Hint,
@@ -1117,6 +1130,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_evaluate_where_clauses]`
     RustcEvaluateWhereClauses,
 
+    /// Represents `#[rustc_expected_cgu_reuse]`, `#[rustc_partition_codegened]` and `#[rustc_partition_reused]`.
+    RustcCguTestAttr(ThinVec<(Span, CguFields)>),
+
     /// Represents `#[rustc_has_incoherent_inherent_impls]`
     RustcHasIncoherentInherentImpls,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index ac9ef79e9ab9..116cda09657a 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -146,6 +146,7 @@ impl AttributeKind {
             RustcOffloadKernel => Yes,
             RustcOutlives => No,
             RustcParenSugar(..) => No,
+            RustcCguTestAttr { .. } => No,
             RustcPassByValue(..) => Yes,
             RustcPassIndirectlyInNonRusticAbis(..) => No,
             RustcPreserveUbChecks => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 0ba4450b1722..c8047e517616 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -338,6 +338,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcOffloadKernel
                     | AttributeKind::RustcOutlives
                     | AttributeKind::RustcParenSugar(..)
+                    | AttributeKind::RustcCguTestAttr(..)
                     | AttributeKind::RustcPassByValue (..)
                     | AttributeKind::RustcPassIndirectlyInNonRusticAbis(..)
                     | AttributeKind::RustcPreserveUbChecks
@@ -407,9 +408,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_autodiff
                             | sym::rustc_capture_analysis
                             | sym::rustc_mir
-                            | sym::rustc_partition_reused
-                            | sym::rustc_partition_codegened
-                            | sym::rustc_expected_cgu_reuse
                             // crate-level attrs, are checked below
                             | sym::feature
                             | sym::register_tool

From 9cc2924959717d5ef9db5c9cdb1060a6ac4cd9ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1535/1843] Port `rustc_deprecated_safe_2024` to the new
 attribute parser

---
 .../src/attributes/rustc_internal.rs          | 50 +++++++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs    |  1 +
 .../rustc_hir/src/attrs/data_structures.rs    |  3 ++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 .../rustc_mir_build/src/check_unsafety.rs     | 29 +++--------
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 6 files changed, 63 insertions(+), 23 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index e5ea7fde3954..96f386639803 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -345,6 +345,56 @@ impl AttributeParser for RustcCguTestAttributeParser {
         Some(AttributeKind::RustcCguTestAttr(self.items))
     }
 }
+
+pub(crate) struct RustcDeprecatedSafe2024Parser;
+
+impl SingleAttributeParser for RustcDeprecatedSafe2024Parser {
+    const PATH: &[Symbol] = &[sym::rustc_deprecated_safe_2024];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::Trait { body: false })),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+    ]);
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
+    const TEMPLATE: AttributeTemplate = template!(List: &[r#"audit_that = "...""#]);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option {
+        let Some(args) = args.list() else {
+            cx.expected_list(cx.attr_span, args);
+            return None;
+        };
+
+        let Some(single) = args.single() else {
+            cx.expected_single_argument(args.span);
+            return None;
+        };
+
+        let Some(arg) = single.meta_item() else {
+            cx.expected_name_value(args.span, None);
+            return None;
+        };
+
+        let Some(args) = arg.word_is(sym::audit_that) else {
+            cx.expected_specific_argument(arg.span(), &[sym::audit_that]);
+            return None;
+        };
+
+        let Some(nv) = args.name_value() else {
+            cx.expected_name_value(arg.span(), Some(sym::audit_that));
+            return None;
+        };
+
+        let Some(suggestion) = nv.value_as_str() else {
+            cx.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
+            return None;
+        };
+
+        Some(AttributeKind::RustcDeprecatedSafe2024 { suggestion })
+    }
+}
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index f53f48b337b5..75f48bb39528 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -202,6 +202,7 @@ attribute_parsers!(
         Single,
         Single,
         Single,
+        Single,
         Single,
         Single,
         Single,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 982c69c38afb..d9da0633365e 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1103,6 +1103,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_deny_explicit_impl]`.
     RustcDenyExplicitImpl(Span),
 
+    /// Represents `#[rustc_deprecated_safe_2024]`
+    RustcDeprecatedSafe2024 { suggestion: Symbol },
+
     /// Represents `#[rustc_dummy]`.
     RustcDummy,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 116cda09657a..f26c5a07d430 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -108,6 +108,7 @@ impl AttributeKind {
             RustcDefPath(..) => No,
             RustcDelayedBugFromInsideQuery => No,
             RustcDenyExplicitImpl(..) => No,
+            RustcDeprecatedSafe2024 { .. } => Yes,
             RustcDummy => No,
             RustcDumpDefParents => No,
             RustcDumpItemBounds => No,
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index 4a20b05d1fc2..7bced8168bd1 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -18,7 +18,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::lint::Level;
 use rustc_session::lint::builtin::{DEPRECATED_SAFE_2024, UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE};
 use rustc_span::def_id::{DefId, LocalDefId};
-use rustc_span::{Span, Symbol, sym};
+use rustc_span::{Span, Symbol};
 
 use crate::builder::ExprCategory;
 use crate::errors::*;
@@ -98,29 +98,14 @@ impl<'tcx> UnsafetyVisitor<'_, 'tcx> {
             // from an edition before 2024.
             &UnsafeOpKind::CallToUnsafeFunction(Some(id))
                 if !span.at_least_rust_2024()
-                    && let Some(attr) = self.tcx.get_attr(id, sym::rustc_deprecated_safe_2024) =>
+                    && let Some(suggestion) = find_attr!(self.tcx.get_all_attrs(id), AttributeKind::RustcDeprecatedSafe2024{suggestion} => suggestion) =>
             {
-                let suggestion = attr
-                    .meta_item_list()
-                    .unwrap_or_default()
-                    .into_iter()
-                    .find(|item| item.has_name(sym::audit_that))
-                    .map(|item| {
-                        item.value_str().expect(
-                            "`#[rustc_deprecated_safe_2024(audit_that)]` must have a string value",
-                        )
-                    });
-
                 let sm = self.tcx.sess.source_map();
-                let guarantee = suggestion
-                    .as_ref()
-                    .map(|suggestion| format!("that {}", suggestion))
-                    .unwrap_or_else(|| String::from("its unsafe preconditions"));
-                let suggestion = suggestion
-                    .and_then(|suggestion| {
-                        sm.indentation_before(span).map(|indent| {
-                            format!("{}// TODO: Audit that {}.\n", indent, suggestion) // ignore-tidy-todo
-                        })
+                let guarantee = format!("that {}", suggestion);
+                let suggestion = sm
+                    .indentation_before(span)
+                    .map(|indent| {
+                        format!("{}// TODO: Audit that {}.\n", indent, suggestion) // ignore-tidy-todo
                     })
                     .unwrap_or_default();
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index c8047e517616..738a8aaeaf7a 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -303,6 +303,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcDefPath(..)
                     | AttributeKind::RustcDelayedBugFromInsideQuery
                     | AttributeKind::RustcDenyExplicitImpl(..)
+                    | AttributeKind::RustcDeprecatedSafe2024 {..}
                     | AttributeKind::RustcDummy
                     | AttributeKind::RustcDumpDefParents
                     | AttributeKind::RustcDumpItemBounds
@@ -400,7 +401,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_do_not_const_check
                             | sym::rustc_doc_primitive
                             | sym::rustc_conversion_suggestion
-                            | sym::rustc_deprecated_safe_2024
                             | sym::rustc_test_marker
                             | sym::rustc_layout
                             | sym::rustc_proc_macro_decls

From 9d10b2ff370b0711bec5b542fa837a4f9f15400d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1536/1843] Port `rustc_conversion_suggestion` to the new
 attribute parser

---
 .../src/attributes/rustc_internal.rs          | 16 +++++++++++
 compiler/rustc_attr_parsing/src/context.rs    |  1 +
 .../rustc_hir/src/attrs/data_structures.rs    |  3 +++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 compiler/rustc_hir_typeck/src/demand.rs       | 27 +++++++++----------
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 6 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 96f386639803..73b8b2824b1a 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -395,6 +395,22 @@ impl SingleAttributeParser for RustcDeprecatedSafe2024Parser {
         Some(AttributeKind::RustcDeprecatedSafe2024 { suggestion })
     }
 }
+
+pub(crate) struct RustcConversionSuggestionParser;
+
+impl NoArgsAttributeParser for RustcConversionSuggestionParser {
+    const PATH: &[Symbol] = &[sym::rustc_conversion_suggestion];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::Trait { body: false })),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcConversionSuggestion;
+}
+
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 75f48bb39528..c1889e9ec0e7 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -264,6 +264,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index d9da0633365e..0c5cb8e4ebff 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1091,6 +1091,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_const_stable_indirect]`.
     RustcConstStabilityIndirect,
 
+    /// Represents `#[rustc_conversion_suggestion]`
+    RustcConversionSuggestion,
+
     /// Represents `#[rustc_deallocator]`
     RustcDeallocator,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index f26c5a07d430..3c81abdf3891 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -104,6 +104,7 @@ impl AttributeKind {
             RustcConfusables { .. } => Yes,
             RustcConstStability { .. } => Yes,
             RustcConstStabilityIndirect => No,
+            RustcConversionSuggestion => Yes,
             RustcDeallocator => No,
             RustcDefPath(..) => No,
             RustcDelayedBugFromInsideQuery => No,
diff --git a/compiler/rustc_hir_typeck/src/demand.rs b/compiler/rustc_hir_typeck/src/demand.rs
index 84663ff884b4..68cbfa728058 100644
--- a/compiler/rustc_hir_typeck/src/demand.rs
+++ b/compiler/rustc_hir_typeck/src/demand.rs
@@ -1,7 +1,8 @@
 use rustc_errors::{Applicability, Diag, MultiSpan, listify};
-use rustc_hir as hir;
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::Res;
 use rustc_hir::intravisit::Visitor;
+use rustc_hir::{self as hir, find_attr};
 use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_middle::bug;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
@@ -1081,19 +1082,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             hir_id,
             |m| {
                 self.has_only_self_parameter(m)
-                    && self
-                        .tcx
-                        // This special internal attribute is used to permit
-                        // "identity-like" conversion methods to be suggested here.
-                        //
-                        // FIXME (#46459 and #46460): ideally
-                        // `std::convert::Into::into` and `std::borrow:ToOwned` would
-                        // also be `#[rustc_conversion_suggestion]`, if not for
-                        // method-probing false-positives and -negatives (respectively).
-                        //
-                        // FIXME? Other potential candidate methods: `as_ref` and
-                        // `as_mut`?
-                        .has_attr(m.def_id, sym::rustc_conversion_suggestion)
+                // This special internal attribute is used to permit
+                // "identity-like" conversion methods to be suggested here.
+                //
+                // FIXME (#46459 and #46460): ideally
+                // `std::convert::Into::into` and `std::borrow:ToOwned` would
+                // also be `#[rustc_conversion_suggestion]`, if not for
+                // method-probing false-positives and -negatives (respectively).
+                //
+                // FIXME? Other potential candidate methods: `as_ref` and
+                // `as_mut`?
+                && find_attr!(self.tcx.get_all_attrs(m.def_id), AttributeKind::RustcConversionSuggestion)
             },
         );
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 738a8aaeaf7a..8ca32179a445 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -299,6 +299,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcCoinductive(..)
                     | AttributeKind::RustcConfusables { .. }
                     | AttributeKind::RustcConstStabilityIndirect
+                    | AttributeKind::RustcConversionSuggestion
                     | AttributeKind::RustcDeallocator
                     | AttributeKind::RustcDefPath(..)
                     | AttributeKind::RustcDelayedBugFromInsideQuery
@@ -400,7 +401,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_on_unimplemented
                             | sym::rustc_do_not_const_check
                             | sym::rustc_doc_primitive
-                            | sym::rustc_conversion_suggestion
                             | sym::rustc_test_marker
                             | sym::rustc_layout
                             | sym::rustc_proc_macro_decls

From a17eb934db482303db43a78023e2e8e56e8e600b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1537/1843] Port `rustc_capture_analysis` to the new attribute
 parser

---
 .../src/attributes/rustc_internal.rs                   |  9 +++++++++
 compiler/rustc_attr_parsing/src/context.rs             |  1 +
 compiler/rustc_hir/src/attrs/data_structures.rs        |  3 +++
 compiler/rustc_hir/src/attrs/encode_cross_crate.rs     |  1 +
 compiler/rustc_hir_typeck/src/upvar.rs                 | 10 +++++++---
 compiler/rustc_passes/src/check_attr.rs                |  1 +
 6 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 73b8b2824b1a..52268def8718 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -411,6 +411,15 @@ impl NoArgsAttributeParser for RustcConversionSuggestionParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcConversionSuggestion;
 }
 
+pub(crate) struct RustcCaptureAnalysisParser;
+
+impl NoArgsAttributeParser for RustcCaptureAnalysisParser {
+    const PATH: &[Symbol] = &[sym::rustc_capture_analysis];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Closure)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcCaptureAnalysis;
+}
+
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index c1889e9ec0e7..16e4a6452210 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -263,6 +263,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 0c5cb8e4ebff..09b51cef3c1a 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1066,6 +1066,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_builtin_macro]`.
     RustcBuiltinMacro { builtin_name: Option, helper_attrs: ThinVec, span: Span },
 
+    /// Represents `#[rustc_capture_analysis]`
+    RustcCaptureAnalysis,
+
     /// Represents `#[rustc_clean]`
     RustcClean(ThinVec),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 3c81abdf3891..4c68d51e6334 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -98,6 +98,7 @@ impl AttributeKind {
             RustcAsPtr(..) => Yes,
             RustcBodyStability { .. } => No,
             RustcBuiltinMacro { .. } => Yes,
+            RustcCaptureAnalysis => No,
             RustcClean { .. } => No,
             RustcCoherenceIsCore(..) => No,
             RustcCoinductive(..) => No,
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index 1a2b76485f35..767913ba5261 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -36,10 +36,10 @@ use rustc_abi::FIRST_VARIANT;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::unord::{ExtendUnord, UnordSet};
 use rustc_errors::{Applicability, MultiSpan};
-use rustc_hir as hir;
-use rustc_hir::HirId;
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::{self as hir, HirId, find_attr};
 use rustc_middle::hir::place::{Place, PlaceBase, PlaceWithHirId, Projection, ProjectionKind};
 use rustc_middle::mir::FakeReadCause;
 use rustc_middle::traits::ObligationCauseCode;
@@ -1743,7 +1743,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     fn should_log_capture_analysis(&self, closure_def_id: LocalDefId) -> bool {
-        self.has_rustc_attrs && self.tcx.has_attr(closure_def_id, sym::rustc_capture_analysis)
+        self.has_rustc_attrs
+            && find_attr!(
+                self.tcx.get_all_attrs(closure_def_id),
+                AttributeKind::RustcCaptureAnalysis
+            )
     }
 
     fn log_capture_analysis_first_pass(
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 8ca32179a445..db5c33fcb17c 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -294,6 +294,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcAsPtr(..)
                     | AttributeKind::RustcBodyStability { .. }
                     | AttributeKind::RustcBuiltinMacro { .. }
+                    | AttributeKind::RustcCaptureAnalysis
                     | AttributeKind::RustcClean(..)
                     | AttributeKind::RustcCoherenceIsCore(..)
                     | AttributeKind::RustcCoinductive(..)

From 1c45618335efce02aaf1bcc7235aedff7eae3629 Mon Sep 17 00:00:00 2001
From: Nikita Popov 
Date: Tue, 10 Feb 2026 10:11:25 +0100
Subject: [PATCH 1538/1843] Update to LLVM 22 rc 3

---
 src/llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/llvm-project b/src/llvm-project
index 41256ab128f2..a306f6a8c534 160000
--- a/src/llvm-project
+++ b/src/llvm-project
@@ -1 +1 @@
-Subproject commit 41256ab128f2af613b4a16771cda6baa36b55f45
+Subproject commit a306f6a8c53494d32c171c346c6809c97124c697

From ea361287be9aa101d7ad50b81d216edc5222ce6f Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Tue, 10 Feb 2026 09:13:45 +0000
Subject: [PATCH 1539/1843] Remove SubdiagMessage in favour of the identical
 DiagMessage

---
 compiler/rustc_error_messages/src/lib.rs      |  58 ----------
 compiler/rustc_errors/src/diagnostic.rs       | 108 ++++++------------
 compiler/rustc_errors/src/lib.rs              |  21 ++--
 compiler/rustc_hir_typeck/src/method/mod.rs   |   4 +-
 compiler/rustc_span/src/symbol.rs             |   1 -
 .../src/diagnostics/translation.md            |  12 +-
 .../clippy/clippy_utils/src/diagnostics.rs    |   8 +-
 .../diagnostic-derive-doc-comment-field.rs    |   2 +-
 .../diagnostic-derive-inline.rs               |   2 +-
 .../subdiagnostic-derive-inline.rs            |   2 +-
 10 files changed, 56 insertions(+), 162 deletions(-)

diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 916a370659aa..0b30102eb992 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -230,38 +230,6 @@ pub fn fallback_fluent_bundle(
     })))
 }
 
-/// Abstraction over a message in a subdiagnostic (i.e. label, note, help, etc) to support both
-/// translatable and non-translatable diagnostic messages.
-///
-/// Translatable messages for subdiagnostics are typically attributes attached to a larger Fluent
-/// message so messages of this type must be combined with a `DiagMessage` (using
-/// `DiagMessage::with_subdiagnostic_message`) before rendering. However, subdiagnostics from
-/// the `Subdiagnostic` derive refer to Fluent identifiers directly.
-#[rustc_diagnostic_item = "SubdiagMessage"]
-pub enum SubdiagMessage {
-    /// Non-translatable diagnostic message.
-    Str(Cow<'static, str>),
-    /// An inline Fluent message. Instances of this variant are generated by the
-    /// `Subdiagnostic` derive.
-    Inline(Cow<'static, str>),
-}
-
-impl From for SubdiagMessage {
-    fn from(s: String) -> Self {
-        SubdiagMessage::Str(Cow::Owned(s))
-    }
-}
-impl From<&'static str> for SubdiagMessage {
-    fn from(s: &'static str) -> Self {
-        SubdiagMessage::Str(Cow::Borrowed(s))
-    }
-}
-impl From> for SubdiagMessage {
-    fn from(s: Cow<'static, str>) -> Self {
-        SubdiagMessage::Str(s)
-    }
-}
-
 /// Abstraction over a message in a diagnostic to support both translatable and non-translatable
 /// diagnostic messages.
 ///
@@ -281,18 +249,6 @@ pub enum DiagMessage {
 }
 
 impl DiagMessage {
-    /// Given a `SubdiagMessage` which may contain a Fluent attribute, create a new
-    /// `DiagMessage` that combines that attribute with the Fluent identifier of `self`.
-    ///
-    /// - If the `SubdiagMessage` is non-translatable then return the message as a `DiagMessage`.
-    /// - If `self` is non-translatable then return `self`'s message.
-    pub fn with_subdiagnostic_message(&self, sub: SubdiagMessage) -> Self {
-        match sub {
-            SubdiagMessage::Str(s) => DiagMessage::Str(s),
-            SubdiagMessage::Inline(s) => DiagMessage::Inline(s),
-        }
-    }
-
     pub fn as_str(&self) -> Option<&str> {
         match self {
             DiagMessage::Str(s) => Some(s),
@@ -317,20 +273,6 @@ impl From> for DiagMessage {
     }
 }
 
-/// Translating *into* a subdiagnostic message from a diagnostic message is a little strange - but
-/// the subdiagnostic functions (e.g. `span_label`) take a `SubdiagMessage` and the
-/// subdiagnostic derive refers to typed identifiers that are `DiagMessage`s, so need to be
-/// able to convert between these, as much as they'll be converted back into `DiagMessage`
-/// using `with_subdiagnostic_message` eventually. Don't use this other than for the derive.
-impl From for SubdiagMessage {
-    fn from(val: DiagMessage) -> Self {
-        match val {
-            DiagMessage::Str(s) => SubdiagMessage::Str(s),
-            DiagMessage::Inline(s) => SubdiagMessage::Inline(s),
-        }
-    }
-}
-
 /// A span together with some additional data.
 #[derive(Clone, Debug)]
 pub struct SpanLabel {
diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index a9e81354fc6a..087c5e700df0 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -17,8 +17,7 @@ use tracing::debug;
 
 use crate::{
     CodeSuggestion, DiagCtxtHandle, DiagMessage, ErrCode, ErrorGuaranteed, ExplicitBug, Level,
-    MultiSpan, StashKey, Style, SubdiagMessage, Substitution, SubstitutionPart, SuggestionStyle,
-    Suggestions,
+    MultiSpan, StashKey, Style, Substitution, SubstitutionPart, SuggestionStyle, Suggestions,
 };
 
 pub type DiagArgMap = FxIndexMap;
@@ -325,30 +324,8 @@ impl DiagInner {
         }
     }
 
-    // See comment on `Diag::subdiagnostic_message_to_diagnostic_message`.
-    pub(crate) fn subdiagnostic_message_to_diagnostic_message(
-        &self,
-        attr: impl Into,
-    ) -> DiagMessage {
-        let msg =
-            self.messages.iter().map(|(msg, _)| msg).next().expect("diagnostic with no messages");
-        msg.with_subdiagnostic_message(attr.into())
-    }
-
-    pub(crate) fn sub(
-        &mut self,
-        level: Level,
-        message: impl Into,
-        span: MultiSpan,
-    ) {
-        let sub = Subdiag {
-            level,
-            messages: vec![(
-                self.subdiagnostic_message_to_diagnostic_message(message),
-                Style::NoStyle,
-            )],
-            span,
-        };
+    pub(crate) fn sub(&mut self, level: Level, message: impl Into, span: MultiSpan) {
+        let sub = Subdiag { level, messages: vec![(message.into(), Style::NoStyle)], span };
         self.children.push(sub);
     }
 
@@ -609,9 +586,8 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// the diagnostic was constructed. However, the label span is *not* considered a
     /// ["primary span"][`MultiSpan`]; only the `Span` supplied when creating the diagnostic is
     /// primary.
-    pub fn span_label(&mut self, span: Span, label: impl Into) -> &mut Self {
-        let msg = self.subdiagnostic_message_to_diagnostic_message(label);
-        self.span.push_span_label(span, msg);
+    pub fn span_label(&mut self, span: Span, label: impl Into) -> &mut Self {
+        self.span.push_span_label(span, label.into());
         self
     } }
 
@@ -713,7 +689,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
 
     with_fn! { with_note,
     /// Add a note attached to this diagnostic.
-    pub fn note(&mut self, msg: impl Into) -> &mut Self {
+    pub fn note(&mut self, msg: impl Into) -> &mut Self {
         self.sub(Level::Note, msg, MultiSpan::new());
         self
     } }
@@ -733,7 +709,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     }
 
     /// This is like [`Diag::note()`], but it's only printed once.
-    pub fn note_once(&mut self, msg: impl Into) -> &mut Self {
+    pub fn note_once(&mut self, msg: impl Into) -> &mut Self {
         self.sub(Level::OnceNote, msg, MultiSpan::new());
         self
     }
@@ -744,7 +720,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_note(
         &mut self,
         sp: impl Into,
-        msg: impl Into,
+        msg: impl Into,
     ) -> &mut Self {
         self.sub(Level::Note, msg, sp.into());
         self
@@ -755,7 +731,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_note_once>(
         &mut self,
         sp: S,
-        msg: impl Into,
+        msg: impl Into,
     ) -> &mut Self {
         self.sub(Level::OnceNote, msg, sp.into());
         self
@@ -763,7 +739,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
 
     with_fn! { with_warn,
     /// Add a warning attached to this diagnostic.
-    pub fn warn(&mut self, msg: impl Into) -> &mut Self {
+    pub fn warn(&mut self, msg: impl Into) -> &mut Self {
         self.sub(Level::Warning, msg, MultiSpan::new());
         self
     } }
@@ -773,7 +749,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_warn>(
         &mut self,
         sp: S,
-        msg: impl Into,
+        msg: impl Into,
     ) -> &mut Self {
         self.sub(Level::Warning, msg, sp.into());
         self
@@ -781,13 +757,13 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
 
     with_fn! { with_help,
     /// Add a help message attached to this diagnostic.
-    pub fn help(&mut self, msg: impl Into) -> &mut Self {
+    pub fn help(&mut self, msg: impl Into) -> &mut Self {
         self.sub(Level::Help, msg, MultiSpan::new());
         self
     } }
 
     /// This is like [`Diag::help()`], but it's only printed once.
-    pub fn help_once(&mut self, msg: impl Into) -> &mut Self {
+    pub fn help_once(&mut self, msg: impl Into) -> &mut Self {
         self.sub(Level::OnceHelp, msg, MultiSpan::new());
         self
     }
@@ -814,7 +790,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_help(
         &mut self,
         sp: impl Into,
-        msg: impl Into,
+        msg: impl Into,
     ) -> &mut Self {
         self.sub(Level::Help, msg, sp.into());
         self
@@ -866,7 +842,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// In other words, multiple changes need to be applied as part of this suggestion.
     pub fn multipart_suggestion(
         &mut self,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: Vec<(Span, String)>,
         applicability: Applicability,
     ) -> &mut Self {
@@ -882,7 +858,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// In other words, multiple changes need to be applied as part of this suggestion.
     pub fn multipart_suggestion_verbose(
         &mut self,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: Vec<(Span, String)>,
         applicability: Applicability,
     ) -> &mut Self {
@@ -897,7 +873,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// [`Diag::multipart_suggestion()`] but you can set the [`SuggestionStyle`].
     pub fn multipart_suggestion_with_style(
         &mut self,
-        msg: impl Into,
+        msg: impl Into,
         mut suggestion: Vec<(Span, String)>,
         applicability: Applicability,
         style: SuggestionStyle,
@@ -924,7 +900,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
 
         self.push_suggestion(CodeSuggestion {
             substitutions: vec![Substitution { parts }],
-            msg: self.subdiagnostic_message_to_diagnostic_message(msg),
+            msg: msg.into(),
             style,
             applicability,
         });
@@ -939,7 +915,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// improve understandability.
     pub fn tool_only_multipart_suggestion(
         &mut self,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: Vec<(Span, String)>,
         applicability: Applicability,
     ) -> &mut Self {
@@ -972,7 +948,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestion(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: impl ToString,
         applicability: Applicability,
     ) -> &mut Self {
@@ -990,7 +966,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestion_with_style(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: impl ToString,
         applicability: Applicability,
         style: SuggestionStyle,
@@ -1003,7 +979,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
             substitutions: vec![Substitution {
                 parts: vec![SubstitutionPart { snippet: suggestion.to_string(), span: sp }],
             }],
-            msg: self.subdiagnostic_message_to_diagnostic_message(msg),
+            msg: msg.into(),
             style,
             applicability,
         });
@@ -1015,7 +991,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestion_verbose(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: impl ToString,
         applicability: Applicability,
     ) -> &mut Self {
@@ -1035,7 +1011,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestions(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestions: impl IntoIterator,
         applicability: Applicability,
     ) -> &mut Self {
@@ -1051,7 +1027,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestions_with_style(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestions: impl IntoIterator,
         applicability: Applicability,
         style: SuggestionStyle,
@@ -1068,7 +1044,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
             .collect();
         self.push_suggestion(CodeSuggestion {
             substitutions,
-            msg: self.subdiagnostic_message_to_diagnostic_message(msg),
+            msg: msg.into(),
             style,
             applicability,
         });
@@ -1080,7 +1056,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// See also [`Diag::multipart_suggestion()`].
     pub fn multipart_suggestions(
         &mut self,
-        msg: impl Into,
+        msg: impl Into,
         suggestions: impl IntoIterator>,
         applicability: Applicability,
     ) -> &mut Self {
@@ -1112,7 +1088,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
 
         self.push_suggestion(CodeSuggestion {
             substitutions,
-            msg: self.subdiagnostic_message_to_diagnostic_message(msg),
+            msg: msg.into(),
             style: SuggestionStyle::ShowAlways,
             applicability,
         });
@@ -1127,7 +1103,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestion_short(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: impl ToString,
         applicability: Applicability,
     ) -> &mut Self {
@@ -1150,7 +1126,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn span_suggestion_hidden(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: impl ToString,
         applicability: Applicability,
     ) -> &mut Self {
@@ -1172,7 +1148,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     pub fn tool_only_span_suggestion(
         &mut self,
         sp: Span,
-        msg: impl Into,
+        msg: impl Into,
         suggestion: impl ToString,
         applicability: Applicability,
     ) -> &mut Self {
@@ -1200,10 +1176,9 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
     /// one value will clobber the other. Eagerly translating the
     /// diagnostic uses the variables defined right then, before the
     /// clobbering occurs.
-    pub fn eagerly_translate(&self, msg: impl Into) -> SubdiagMessage {
+    pub fn eagerly_translate(&self, msg: impl Into) -> DiagMessage {
         let args = self.args.iter();
-        let msg = self.subdiagnostic_message_to_diagnostic_message(msg.into());
-        self.dcx.eagerly_translate(msg, args)
+        self.dcx.eagerly_translate(msg.into(), args)
     }
 
     with_fn! { with_span,
@@ -1256,31 +1231,18 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
         self
     } }
 
-    /// Helper function that takes a `SubdiagMessage` and returns a `DiagMessage` by
-    /// combining it with the primary message of the diagnostic (if translatable, otherwise it just
-    /// passes the user's string along).
-    pub(crate) fn subdiagnostic_message_to_diagnostic_message(
-        &self,
-        attr: impl Into,
-    ) -> DiagMessage {
-        self.deref().subdiagnostic_message_to_diagnostic_message(attr)
-    }
-
     /// Convenience function for internal use, clients should use one of the
     /// public methods above.
     ///
     /// Used by `proc_macro_server` for implementing `server::Diagnostic`.
-    pub fn sub(&mut self, level: Level, message: impl Into, span: MultiSpan) {
+    pub fn sub(&mut self, level: Level, message: impl Into, span: MultiSpan) {
         self.deref_mut().sub(level, message, span);
     }
 
     /// Convenience function for internal use, clients should use one of the
     /// public methods above.
     fn sub_with_highlights(&mut self, level: Level, messages: Vec, span: MultiSpan) {
-        let messages = messages
-            .into_iter()
-            .map(|m| (self.subdiagnostic_message_to_diagnostic_message(m.content), m.style))
-            .collect();
+        let messages = messages.into_iter().map(|m| (m.content.into(), m.style)).collect();
         let sub = Subdiag { level, messages, span };
         self.children.push(sub);
     }
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 2e0afb8fc4c2..c0398b3e8da0 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -57,8 +57,8 @@ use rustc_data_structures::sync::{DynSend, Lock};
 use rustc_data_structures::{AtomicRef, assert_matches};
 pub use rustc_error_messages::{
     DiagArg, DiagArgFromDisplay, DiagArgName, DiagArgValue, DiagMessage, FluentBundle, IntoDiagArg,
-    LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, SubdiagMessage,
-    fallback_fluent_bundle, fluent_bundle, into_diag_arg_using_display,
+    LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, fallback_fluent_bundle,
+    fluent_bundle, into_diag_arg_using_display,
 };
 use rustc_hashes::Hash128;
 use rustc_lint_defs::LintExpectationId;
@@ -488,12 +488,12 @@ impl DiagCtxt {
         self.inner.borrow_mut().emitter = emitter;
     }
 
-    /// Translate `message` eagerly with `args` to `SubdiagMessage::Eager`.
+    /// Translate `message` eagerly with `args` to `DiagMessage::Eager`.
     pub fn eagerly_translate<'a>(
         &self,
         message: DiagMessage,
         args: impl Iterator>,
-    ) -> SubdiagMessage {
+    ) -> DiagMessage {
         let inner = self.inner.borrow();
         inner.eagerly_translate(message, args)
     }
@@ -1423,13 +1423,13 @@ impl DiagCtxtInner {
         self.has_errors().or_else(|| self.delayed_bugs.get(0).map(|(_, guar)| guar).copied())
     }
 
-    /// Translate `message` eagerly with `args` to `SubdiagMessage::Eager`.
+    /// Translate `message` eagerly with `args` to `DiagMessage::Eager`.
     fn eagerly_translate<'a>(
         &self,
         message: DiagMessage,
         args: impl Iterator>,
-    ) -> SubdiagMessage {
-        SubdiagMessage::Str(Cow::from(self.eagerly_translate_to_string(message, args)))
+    ) -> DiagMessage {
+        DiagMessage::Str(Cow::from(self.eagerly_translate_to_string(message, args)))
     }
 
     /// Translate `message` eagerly with `args` to `String`.
@@ -1450,10 +1450,9 @@ impl DiagCtxtInner {
     fn eagerly_translate_for_subdiag(
         &self,
         diag: &DiagInner,
-        msg: impl Into,
-    ) -> SubdiagMessage {
-        let msg = diag.subdiagnostic_message_to_diagnostic_message(msg);
-        self.eagerly_translate(msg, diag.args.iter())
+        msg: impl Into,
+    ) -> DiagMessage {
+        self.eagerly_translate(msg.into(), diag.args.iter())
     }
 
     fn flush_delayed(&mut self) {
diff --git a/compiler/rustc_hir_typeck/src/method/mod.rs b/compiler/rustc_hir_typeck/src/method/mod.rs
index bf829c2d645d..746678e2865f 100644
--- a/compiler/rustc_hir_typeck/src/method/mod.rs
+++ b/compiler/rustc_hir_typeck/src/method/mod.rs
@@ -7,7 +7,7 @@ mod prelude_edition_lints;
 pub(crate) mod probe;
 mod suggest;
 
-use rustc_errors::{Applicability, Diag, SubdiagMessage};
+use rustc_errors::{Applicability, Diag, DiagMessage};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Namespace};
 use rustc_hir::def_id::DefId;
@@ -127,7 +127,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub(crate) fn suggest_method_call(
         &self,
         err: &mut Diag<'_>,
-        msg: impl Into + std::fmt::Debug,
+        msg: impl Into + std::fmt::Debug,
         method_name: Ident,
         self_ty: Ty<'tcx>,
         call_expr: &hir::Expr<'tcx>,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 6aa2eae556e2..4b1faf718cd5 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -373,7 +373,6 @@ symbols! {
         Str,
         String,
         StructuralPartialEq,
-        SubdiagMessage,
         Subdiagnostic,
         SymbolIntern,
         Sync,
diff --git a/src/doc/rustc-dev-guide/src/diagnostics/translation.md b/src/doc/rustc-dev-guide/src/diagnostics/translation.md
index 58d75f54a005..e66c9430914a 100644
--- a/src/doc/rustc-dev-guide/src/diagnostics/translation.md
+++ b/src/doc/rustc-dev-guide/src/diagnostics/translation.md
@@ -135,8 +135,7 @@ translation.
 ### Messages
 
 All of rustc's traditional diagnostic APIs (e.g. `struct_span_err` or `note`)
-take any message that can be converted into a `DiagMessage` (or
-`SubdiagMessage`).
+take any message that can be converted into a `DiagMessage`.
 
 [`rustc_error_messages::DiagMessage`] can represent legacy non-translatable
 diagnostic messages and translatable messages. Non-translatable messages are
@@ -149,14 +148,7 @@ with an attribute).
 Fluent resource (described in more detail below), or `DiagMessage`s will
 either be created in the macro-generated code of a diagnostic derive.
 
-`rustc_error_messages::SubdiagMessage` is similar, it can correspond to a
-legacy non-translatable diagnostic message or the name of an attribute to a
-Fluent message. Translatable `SubdiagMessage`s must be combined with a
-`DiagMessage` (using `DiagMessage::with_subdiagnostic_message`) to
-be emitted (an attribute name on its own is meaningless without a corresponding
-message identifier, which is what `DiagMessage` provides).
-
-Both `DiagMessage` and `SubdiagMessage` implement `Into` for any
+`DiagMessage`  implements `Into` for any
 type that can be converted into a string, and converts these into
 non-translatable diagnostics - this keeps all existing diagnostic calls
 working.
diff --git a/src/tools/clippy/clippy_utils/src/diagnostics.rs b/src/tools/clippy/clippy_utils/src/diagnostics.rs
index 8a19039a7fe7..4ba9af58f90a 100644
--- a/src/tools/clippy/clippy_utils/src/diagnostics.rs
+++ b/src/tools/clippy/clippy_utils/src/diagnostics.rs
@@ -8,7 +8,7 @@
 //! Thank you!
 //! ~The `INTERNAL_METADATA_COLLECTOR` lint
 
-use rustc_errors::{Applicability, Diag, DiagMessage, MultiSpan, SubdiagMessage};
+use rustc_errors::{Applicability, Diag, DiagMessage, MultiSpan};
 #[cfg(debug_assertions)]
 use rustc_errors::{EmissionGuarantee, SubstitutionPart, Suggestions};
 use rustc_hir::HirId;
@@ -155,7 +155,7 @@ pub fn span_lint_and_help(
     span: impl Into,
     msg: impl Into,
     help_span: Option,
-    help: impl Into,
+    help: impl Into,
 ) {
     #[expect(clippy::disallowed_methods)]
     cx.span_lint(lint, span, |diag| {
@@ -216,7 +216,7 @@ pub fn span_lint_and_note(
     span: impl Into,
     msg: impl Into,
     note_span: Option,
-    note: impl Into,
+    note: impl Into,
 ) {
     #[expect(clippy::disallowed_methods)]
     cx.span_lint(lint, span, |diag| {
@@ -390,7 +390,7 @@ pub fn span_lint_and_sugg(
     lint: &'static Lint,
     sp: Span,
     msg: impl Into,
-    help: impl Into,
+    help: impl Into,
     sugg: String,
     applicability: Applicability,
 ) {
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs
index e63ab03cda47..c1146de0fef1 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-doc-comment-field.rs
@@ -19,7 +19,7 @@ extern crate rustc_session;
 extern crate rustc_span;
 extern crate core;
 
-use rustc_errors::{Applicability, DiagMessage, SubdiagMessage};
+use rustc_errors::{Applicability, DiagMessage};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::Span;
 
diff --git a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
index ca803d450b92..fa2b6e907a26 100644
--- a/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/diagnostic-derive-inline.rs
@@ -26,7 +26,7 @@ extern crate rustc_middle;
 use rustc_middle::ty::Ty;
 
 extern crate rustc_errors;
-use rustc_errors::{Applicability, DiagMessage, ErrCode, MultiSpan, SubdiagMessage};
+use rustc_errors::{Applicability, DiagMessage, ErrCode, MultiSpan};
 
 extern crate rustc_session;
 
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
index 99b2cba89da4..c7636bca80b5 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
@@ -17,7 +17,7 @@ extern crate rustc_session;
 extern crate rustc_span;
 extern crate core;
 
-use rustc_errors::{Applicability, DiagMessage, SubdiagMessage};
+use rustc_errors::{Applicability, DiagMessage};
 use rustc_macros::Subdiagnostic;
 use rustc_span::Span;
 

From b1847b59cc0fecb8973d82366b72f994e8841331 Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Tue, 10 Feb 2026 09:13:45 +0000
Subject: [PATCH 1540/1843] Remove SubdiagMessage in favour of the identical
 DiagMessage

---
 clippy_utils/src/diagnostics.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clippy_utils/src/diagnostics.rs b/clippy_utils/src/diagnostics.rs
index 8a19039a7fe7..4ba9af58f90a 100644
--- a/clippy_utils/src/diagnostics.rs
+++ b/clippy_utils/src/diagnostics.rs
@@ -8,7 +8,7 @@
 //! Thank you!
 //! ~The `INTERNAL_METADATA_COLLECTOR` lint
 
-use rustc_errors::{Applicability, Diag, DiagMessage, MultiSpan, SubdiagMessage};
+use rustc_errors::{Applicability, Diag, DiagMessage, MultiSpan};
 #[cfg(debug_assertions)]
 use rustc_errors::{EmissionGuarantee, SubstitutionPart, Suggestions};
 use rustc_hir::HirId;
@@ -155,7 +155,7 @@ pub fn span_lint_and_help(
     span: impl Into,
     msg: impl Into,
     help_span: Option,
-    help: impl Into,
+    help: impl Into,
 ) {
     #[expect(clippy::disallowed_methods)]
     cx.span_lint(lint, span, |diag| {
@@ -216,7 +216,7 @@ pub fn span_lint_and_note(
     span: impl Into,
     msg: impl Into,
     note_span: Option,
-    note: impl Into,
+    note: impl Into,
 ) {
     #[expect(clippy::disallowed_methods)]
     cx.span_lint(lint, span, |diag| {
@@ -390,7 +390,7 @@ pub fn span_lint_and_sugg(
     lint: &'static Lint,
     sp: Span,
     msg: impl Into,
-    help: impl Into,
+    help: impl Into,
     sugg: String,
     applicability: Applicability,
 ) {

From 954f483557e24c8559bde3d4b2ef4c48f2b34651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1541/1843] Port `rustc_never_type_options` to the new
 attribute parser

---
 .../src/attributes/rustc_internal.rs          | 82 +++++++++++++++++++
 compiler/rustc_attr_parsing/src/context.rs    |  3 +-
 .../rustc_hir/src/attrs/data_structures.rs    | 50 ++++++++++-
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  3 +-
 compiler/rustc_hir_typeck/src/fallback.rs     | 12 +--
 .../rustc_hir_typeck/src/fn_ctxt/checks.rs    | 24 +-----
 compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs  | 55 +------------
 compiler/rustc_passes/src/check_attr.rs       |  4 +-
 .../malformed-never-type-options.rs           |  2 +-
 .../malformed-never-type-options.stderr       | 12 +--
 10 files changed, 148 insertions(+), 99 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 52268def8718..218488898e4b 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -420,6 +420,88 @@ impl NoArgsAttributeParser for RustcCaptureAnalysisParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcCaptureAnalysis;
 }
 
+pub(crate) struct RustcNeverTypeOptionsParser;
+
+impl SingleAttributeParser for RustcNeverTypeOptionsParser {
+    const PATH: &[Symbol] = &[sym::rustc_never_type_options];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Crate)]);
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
+    const TEMPLATE: AttributeTemplate = template!(List: &[
+        r#"fallback = "unit", "never", "no""#,
+        r#"diverging_block_default = "unit", "never""#,
+    ]);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option {
+        let Some(list) = args.list() else {
+            cx.expected_list(cx.attr_span, args);
+            return None;
+        };
+
+        let mut fallback = None::;
+        let mut diverging_block_default = None::;
+
+        for arg in list.mixed() {
+            let Some(meta) = arg.meta_item() else {
+                cx.expected_name_value(arg.span(), None);
+                continue;
+            };
+
+            let res = match meta.ident().map(|i| i.name) {
+                Some(sym::fallback) => &mut fallback,
+                Some(sym::diverging_block_default) => &mut diverging_block_default,
+                _ => {
+                    cx.expected_specific_argument(
+                        meta.path().span(),
+                        &[sym::fallback, sym::diverging_block_default],
+                    );
+                    continue;
+                }
+            };
+
+            let Some(nv) = meta.args().name_value() else {
+                cx.expected_name_value(meta.span(), None);
+                continue;
+            };
+
+            let Some(field) = nv.value_as_str() else {
+                cx.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
+                continue;
+            };
+
+            if res.is_some() {
+                cx.duplicate_key(meta.span(), meta.ident().unwrap().name);
+                continue;
+            }
+
+            *res = Some(Ident { name: field, span: nv.value_span });
+        }
+
+        let fallback = match fallback {
+            None => None,
+            Some(Ident { name: sym::unit, .. }) => Some(DivergingFallbackBehavior::ToUnit),
+            Some(Ident { name: sym::never, .. }) => Some(DivergingFallbackBehavior::ToNever),
+            Some(Ident { name: sym::no, .. }) => Some(DivergingFallbackBehavior::NoFallback),
+            Some(Ident { span, .. }) => {
+                cx.expected_specific_argument_strings(span, &[sym::unit, sym::never, sym::no]);
+                return None;
+            }
+        };
+
+        let diverging_block_default = match diverging_block_default {
+            None => None,
+            Some(Ident { name: sym::unit, .. }) => Some(DivergingBlockBehavior::Unit),
+            Some(Ident { name: sym::never, .. }) => Some(DivergingBlockBehavior::Never),
+            Some(Ident { span, .. }) => {
+                cx.expected_specific_argument_strings(span, &[sym::unit, sym::no]);
+                return None;
+            }
+        };
+
+        Some(AttributeKind::RustcNeverTypeOptions { fallback, diverging_block_default })
+    }
+}
+
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 16e4a6452210..4d7dc41f1546 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -147,9 +147,9 @@ attribute_parsers!(
         DocParser,
         MacroUseParser,
         NakedParser,
+        RustcCguTestAttributeParser,
         StabilityParser,
         UsedParser,
-        RustcCguTestAttributeParser,
         // tidy-alphabetical-end
 
         // tidy-alphabetical-start
@@ -210,6 +210,7 @@ attribute_parsers!(
         Single,
         Single,
         Single,
+        Single,
         Single,
         Single,
         Single,
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 09b51cef3c1a..e5f45928fe7b 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -63,6 +63,44 @@ pub enum CguFields {
     PartitionCodegened { cfg: Symbol, module: Symbol },
     ExpectedCguReuse { cfg: Symbol, module: Symbol, kind: CguKind },
 }
+
+#[derive(Copy, Clone, PartialEq, Debug, PrintAttribute)]
+#[derive(HashStable_Generic, Encodable, Decodable)]
+pub enum DivergingFallbackBehavior {
+    /// Always fallback to `()` (aka "always spontaneous decay")
+    ToUnit,
+    /// Always fallback to `!` (which should be equivalent to never falling back + not making
+    /// never-to-any coercions unless necessary)
+    ToNever,
+    /// Don't fallback at all
+    NoFallback,
+}
+
+#[derive(Copy, Clone, PartialEq, Debug, PrintAttribute, Default)]
+#[derive(HashStable_Generic, Encodable, Decodable)]
+pub enum DivergingBlockBehavior {
+    /// This is the current stable behavior:
+    ///
+    /// ```rust
+    /// {
+    ///     return;
+    /// } // block has type = !, even though we are supposedly dropping it with `;`
+    /// ```
+    #[default]
+    Never,
+
+    /// Alternative behavior:
+    ///
+    /// ```ignore (very-unstable-new-attribute)
+    /// #![rustc_never_type_options(diverging_block_default = "unit")]
+    /// {
+    ///     return;
+    /// } // block has type = (), since we are dropping `!` from `return` with `;`
+    /// ```
+    Unit,
+}
+
+#[derive(Copy, Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, PrintAttribute)]
 pub enum InlineAttr {
     None,
     Hint,
@@ -1069,6 +1107,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_capture_analysis]`
     RustcCaptureAnalysis,
 
+    /// Represents `#[rustc_expected_cgu_reuse]`, `#[rustc_partition_codegened]` and `#[rustc_partition_reused]`.
+    RustcCguTestAttr(ThinVec<(Span, CguFields)>),
+
     /// Represents `#[rustc_clean]`
     RustcClean(ThinVec),
 
@@ -1139,9 +1180,6 @@ pub enum AttributeKind {
     /// Represents `#[rustc_evaluate_where_clauses]`
     RustcEvaluateWhereClauses,
 
-    /// Represents `#[rustc_expected_cgu_reuse]`, `#[rustc_partition_codegened]` and `#[rustc_partition_reused]`.
-    RustcCguTestAttr(ThinVec<(Span, CguFields)>),
-
     /// Represents `#[rustc_has_incoherent_inherent_impls]`
     RustcHasIncoherentInherentImpls,
 
@@ -1199,6 +1237,12 @@ pub enum AttributeKind {
     /// Represents `#[rustc_never_returns_null_ptr]`
     RustcNeverReturnsNullPointer,
 
+    /// Represents `#[rustc_never_type_options]`.
+    RustcNeverTypeOptions {
+        fallback: Option,
+        diverging_block_default: Option,
+    },
+
     /// Represents `#[rustc_no_implicit_autorefs]`
     RustcNoImplicitAutorefs,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 4c68d51e6334..5384a12a0ffb 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -99,6 +99,7 @@ impl AttributeKind {
             RustcBodyStability { .. } => No,
             RustcBuiltinMacro { .. } => Yes,
             RustcCaptureAnalysis => No,
+            RustcCguTestAttr { .. } => No,
             RustcClean { .. } => No,
             RustcCoherenceIsCore(..) => No,
             RustcCoinductive(..) => No,
@@ -139,6 +140,7 @@ impl AttributeKind {
             RustcMir(..) => Yes,
             RustcMustImplementOneOf { .. } => No,
             RustcNeverReturnsNullPointer => Yes,
+            RustcNeverTypeOptions { .. } => No,
             RustcNoImplicitAutorefs => Yes,
             RustcNoImplicitBounds => No,
             RustcNonConstTraitMethod => No, // should be reported via other queries like `constness`
@@ -149,7 +151,6 @@ impl AttributeKind {
             RustcOffloadKernel => Yes,
             RustcOutlives => No,
             RustcParenSugar(..) => No,
-            RustcCguTestAttr { .. } => No,
             RustcPassByValue(..) => Yes,
             RustcPassIndirectlyInNonRusticAbis(..) => No,
             RustcPreserveUbChecks => No,
diff --git a/compiler/rustc_hir_typeck/src/fallback.rs b/compiler/rustc_hir_typeck/src/fallback.rs
index 727666de3c47..2e421c610e7a 100644
--- a/compiler/rustc_hir_typeck/src/fallback.rs
+++ b/compiler/rustc_hir_typeck/src/fallback.rs
@@ -7,6 +7,7 @@ use rustc_data_structures::graph::{self};
 use rustc_data_structures::unord::{UnordMap, UnordSet};
 use rustc_hir as hir;
 use rustc_hir::HirId;
+use rustc_hir::attrs::DivergingFallbackBehavior;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::{InferKind, Visitor};
@@ -19,17 +20,6 @@ use tracing::debug;
 
 use crate::{FnCtxt, errors};
 
-#[derive(Copy, Clone)]
-pub(crate) enum DivergingFallbackBehavior {
-    /// Always fallback to `()` (aka "always spontaneous decay")
-    ToUnit,
-    /// Always fallback to `!` (which should be equivalent to never falling back + not making
-    /// never-to-any coercions unless necessary)
-    ToNever,
-    /// Don't fallback at all
-    NoFallback,
-}
-
 impl<'tcx> FnCtxt<'_, 'tcx> {
     /// Performs type inference fallback, setting [`FnCtxt::diverging_fallback_has_occurred`]
     /// if the never type fallback has occurred.
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
index 2e85410c8960..54d8306936dd 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
@@ -5,6 +5,7 @@ use itertools::Itertools;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, MultiSpan, a_or_an, listify, pluralize};
+use rustc_hir::attrs::DivergingBlockBehavior;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::Visitor;
@@ -47,29 +48,6 @@ rustc_index::newtype_index! {
     pub(crate) struct GenericIdx {}
 }
 
-#[derive(Clone, Copy, Default)]
-pub(crate) enum DivergingBlockBehavior {
-    /// This is the current stable behavior:
-    ///
-    /// ```rust
-    /// {
-    ///     return;
-    /// } // block has type = !, even though we are supposedly dropping it with `;`
-    /// ```
-    #[default]
-    Never,
-
-    /// Alternative behavior:
-    ///
-    /// ```ignore (very-unstable-new-attribute)
-    /// #![rustc_never_type_options(diverging_block_default = "unit")]
-    /// {
-    ///     return;
-    /// } // block has type = (), since we are dropping `!` from `return` with `;`
-    /// ```
-    Unit,
-}
-
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub(in super::super) fn check_casts(&mut self) {
         let mut deferred_cast_checks = self.root_ctxt.deferred_cast_checks.borrow_mut();
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
index c875e2e50d70..412df9162e9f 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
@@ -10,8 +10,9 @@ use std::ops::Deref;
 
 use hir::def_id::CRATE_DEF_ID;
 use rustc_errors::DiagCtxtHandle;
+use rustc_hir::attrs::{AttributeKind, DivergingBlockBehavior, DivergingFallbackBehavior};
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::{self as hir, HirId, ItemLocalMap};
+use rustc_hir::{self as hir, HirId, ItemLocalMap, find_attr};
 use rustc_hir_analysis::hir_ty_lowering::{
     HirTyLowerer, InherentAssocCandidate, RegionInferReason,
 };
@@ -19,15 +20,13 @@ use rustc_infer::infer::{self, RegionVariableOrigin};
 use rustc_infer::traits::{DynCompatibilityViolation, Obligation};
 use rustc_middle::ty::{self, Const, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::Session;
-use rustc_span::{self, DUMMY_SP, ErrorGuaranteed, Ident, Span, sym};
+use rustc_span::{self, DUMMY_SP, ErrorGuaranteed, Ident, Span};
 use rustc_trait_selection::error_reporting::TypeErrCtxt;
 use rustc_trait_selection::traits::{
     self, FulfillmentError, ObligationCause, ObligationCauseCode, ObligationCtxt,
 };
 
 use crate::coercion::CoerceMany;
-use crate::fallback::DivergingFallbackBehavior;
-use crate::fn_ctxt::checks::DivergingBlockBehavior;
 use crate::{CoroutineTypes, Diverges, EnclosingBreakables, TypeckRootCtxt};
 
 /// The `FnCtxt` stores type-checking context needed to type-check bodies of
@@ -517,51 +516,5 @@ fn parse_never_type_options_attr(
     // Error handling is dubious here (unwraps), but that's probably fine for an internal attribute.
     // Just don't write incorrect attributes <3
 
-    let mut fallback = None;
-    let mut block = None;
-
-    let items = if tcx.features().rustc_attrs() {
-        tcx.get_attr(CRATE_DEF_ID, sym::rustc_never_type_options)
-            .map(|attr| attr.meta_item_list().unwrap())
-    } else {
-        None
-    };
-    let items = items.unwrap_or_default();
-
-    for item in items {
-        if item.has_name(sym::fallback) && fallback.is_none() {
-            let mode = item.value_str().unwrap();
-            match mode {
-                sym::unit => fallback = Some(DivergingFallbackBehavior::ToUnit),
-                sym::never => fallback = Some(DivergingFallbackBehavior::ToNever),
-                sym::no => fallback = Some(DivergingFallbackBehavior::NoFallback),
-                _ => {
-                    tcx.dcx().span_err(item.span(), format!("unknown never type fallback mode: `{mode}` (supported: `unit`, `niko`, `never` and `no`)"));
-                }
-            };
-            continue;
-        }
-
-        if item.has_name(sym::diverging_block_default) && block.is_none() {
-            let default = item.value_str().unwrap();
-            match default {
-                sym::unit => block = Some(DivergingBlockBehavior::Unit),
-                sym::never => block = Some(DivergingBlockBehavior::Never),
-                _ => {
-                    tcx.dcx().span_err(item.span(), format!("unknown diverging block default: `{default}` (supported: `unit` and `never`)"));
-                }
-            };
-            continue;
-        }
-
-        tcx.dcx().span_err(
-            item.span(),
-            format!(
-                "unknown or duplicate never type option: `{}` (supported: `fallback`, `diverging_block_default`)",
-                item.name().unwrap()
-            ),
-        );
-    }
-
-    (fallback, block)
+    find_attr!(tcx.get_all_attrs(CRATE_DEF_ID), AttributeKind::RustcNeverTypeOptions {fallback, diverging_block_default} => (*fallback, *diverging_block_default)).unwrap_or_default()
 }
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index db5c33fcb17c..4f29b5e715d6 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -295,6 +295,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcBodyStability { .. }
                     | AttributeKind::RustcBuiltinMacro { .. }
                     | AttributeKind::RustcCaptureAnalysis
+                    | AttributeKind::RustcCguTestAttr(..)
                     | AttributeKind::RustcClean(..)
                     | AttributeKind::RustcCoherenceIsCore(..)
                     | AttributeKind::RustcCoinductive(..)
@@ -332,6 +333,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcMain
                     | AttributeKind::RustcMir(_)
                     | AttributeKind::RustcNeverReturnsNullPointer
+                    | AttributeKind::RustcNeverTypeOptions {..}
                     | AttributeKind::RustcNoImplicitAutorefs
                     | AttributeKind::RustcNoImplicitBounds
                     | AttributeKind::RustcNonConstTraitMethod
@@ -341,7 +343,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcOffloadKernel
                     | AttributeKind::RustcOutlives
                     | AttributeKind::RustcParenSugar(..)
-                    | AttributeKind::RustcCguTestAttr(..)
                     | AttributeKind::RustcPassByValue (..)
                     | AttributeKind::RustcPassIndirectlyInNonRusticAbis(..)
                     | AttributeKind::RustcPreserveUbChecks
@@ -405,7 +406,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_test_marker
                             | sym::rustc_layout
                             | sym::rustc_proc_macro_decls
-                            | sym::rustc_never_type_options
                             | sym::rustc_autodiff
                             | sym::rustc_capture_analysis
                             | sym::rustc_mir
diff --git a/tests/ui/attributes/malformed-never-type-options.rs b/tests/ui/attributes/malformed-never-type-options.rs
index 0c384be0e226..7dd7a854ed2e 100644
--- a/tests/ui/attributes/malformed-never-type-options.rs
+++ b/tests/ui/attributes/malformed-never-type-options.rs
@@ -2,7 +2,7 @@
 //! The `rustc_*` attribute is malformed, but ICEing without a `feature(rustc_attrs)` is still bad.
 
 #![rustc_never_type_options(: Unsize = "hi")]
-//~^ ERROR expected unsuffixed literal, found `:`
+//~^ ERROR expected a literal
 //~| ERROR use of an internal attribute
 
 fn main() {}
diff --git a/tests/ui/attributes/malformed-never-type-options.stderr b/tests/ui/attributes/malformed-never-type-options.stderr
index 0d2ff4881f2a..98870c8bdded 100644
--- a/tests/ui/attributes/malformed-never-type-options.stderr
+++ b/tests/ui/attributes/malformed-never-type-options.stderr
@@ -1,9 +1,3 @@
-error: expected unsuffixed literal, found `:`
-  --> $DIR/malformed-never-type-options.rs:4:29
-   |
-LL | #![rustc_never_type_options(: Unsize = "hi")]
-   |                             ^
-
 error[E0658]: use of an internal attribute
   --> $DIR/malformed-never-type-options.rs:4:1
    |
@@ -14,6 +8,12 @@ LL | #![rustc_never_type_options(: Unsize = "hi")]
    = note: the `#[rustc_never_type_options]` attribute is an internal implementation detail that will never be stable
    = note: `rustc_never_type_options` is used to experiment with never type fallback and work on never type stabilization
 
+error: expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found `:`
+  --> $DIR/malformed-never-type-options.rs:4:29
+   |
+LL | #![rustc_never_type_options(: Unsize = "hi")]
+   |                             ^
+
 error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0658`.

From 2377d355ad61ce1bf5cb33632e101d3016cf152e Mon Sep 17 00:00:00 2001
From: Jonathan Brouwer 
Date: Tue, 10 Feb 2026 10:17:59 +0100
Subject: [PATCH 1542/1843] Update documentation of rustc_macros

---
 compiler/rustc_macros/src/diagnostics/mod.rs | 115 ++++++-------------
 1 file changed, 35 insertions(+), 80 deletions(-)

diff --git a/compiler/rustc_macros/src/diagnostics/mod.rs b/compiler/rustc_macros/src/diagnostics/mod.rs
index 09f05ce972f1..a036e61c3789 100644
--- a/compiler/rustc_macros/src/diagnostics/mod.rs
+++ b/compiler/rustc_macros/src/diagnostics/mod.rs
@@ -21,36 +21,25 @@ use synstructure::Structure;
 /// # extern crate rust_middle;
 /// # use rustc_middle::ty::Ty;
 /// #[derive(Diagnostic)]
-/// #[diag(borrowck_move_out_of_borrow, code = E0505)]
-/// pub struct MoveOutOfBorrowError<'tcx> {
+/// #[diag("this is an example message", code = E0123)]
+/// pub(crate) struct ExampleError<'tcx> {
 ///     pub name: Ident,
 ///     pub ty: Ty<'tcx>,
 ///     #[primary_span]
-///     #[label]
+///     #[label("with a label")]
 ///     pub span: Span,
-///     #[label(first_borrow_label)]
-///     pub first_borrow_span: Span,
-///     #[suggestion(code = "{name}.clone()")]
-///     pub clone_sugg: Option<(Span, Applicability)>
+///     #[label("with a label")]
+///     pub other_span: Span,
+///     #[suggestion("with a suggestion", code = "{name}.clone()")]
+///     pub opt_sugg: Option<(Span, Applicability)>,
 /// }
 /// ```
 ///
-/// ```fluent
-/// move_out_of_borrow = cannot move out of {$name} because it is borrowed
-///     .label = cannot move out of borrow
-///     .first_borrow_label = `{$ty}` first borrowed here
-///     .suggestion = consider cloning here
-/// ```
-///
 /// Then, later, to emit the error:
 ///
 /// ```ignore (rust)
-/// sess.emit_err(MoveOutOfBorrowError {
-///     expected,
-///     actual,
-///     span,
-///     first_borrow_span,
-///     clone_sugg: Some(suggestion, Applicability::MachineApplicable),
+/// sess.emit_err(ExampleError {
+///     name, ty, span, other_span, opt_sugg
 /// });
 /// ```
 ///
@@ -65,38 +54,24 @@ pub(super) fn diagnostic_derive(s: Structure<'_>) -> TokenStream {
 ///
 /// ```ignore (rust)
 /// #[derive(LintDiagnostic)]
-/// #[diag(lint_atomic_ordering_invalid_fail_success)]
-/// pub struct AtomicOrderingInvalidLint {
-///     method: Symbol,
-///     success_ordering: Symbol,
-///     fail_ordering: Symbol,
-///     #[label(fail_label)]
-///     fail_order_arg_span: Span,
-///     #[label(success_label)]
-///     #[suggestion(
-///         code = "std::sync::atomic::Ordering::{success_suggestion}",
-///         applicability = "maybe-incorrect"
+/// #[diag("unused attribute")]
+/// pub(crate) struct UnusedAttribute {
+///     #[suggestion("remove this attribute", code = "", applicability = "machine-applicable")]
+///     pub this: Span,
+///     #[note("attribute also specified here")]
+///     pub other: Span,
+///     #[warning(
+///         "this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!"
 ///     )]
-///     success_order_arg_span: Span,
+///     pub warning: bool,
 /// }
 /// ```
 ///
-/// ```fluent
-/// lint_atomic_ordering_invalid_fail_success = `{$method}`'s success ordering must be at least as strong as its failure ordering
-///     .fail_label = `{$fail_ordering}` failure ordering
-///     .success_label = `{$success_ordering}` success ordering
-///     .suggestion = consider using `{$success_suggestion}` success ordering instead
-/// ```
-///
 /// Then, later, to emit the error:
 ///
 /// ```ignore (rust)
-/// cx.emit_span_lint(INVALID_ATOMIC_ORDERING, fail_order_arg_span, AtomicOrderingInvalidLint {
-///     method,
-///     success_ordering,
-///     fail_ordering,
-///     fail_order_arg_span,
-///     success_order_arg_span,
+/// cx.emit_span_lint(UNUSED_ATTRIBUTES, span, UnusedAttribute {
+///     ...
 /// });
 /// ```
 ///
@@ -112,45 +87,25 @@ pub(super) fn lint_diagnostic_derive(s: Structure<'_>) -> TokenStream {
 ///
 /// ```ignore (rust)
 /// #[derive(Subdiagnostic)]
-/// pub enum ExpectedIdentifierLabel<'tcx> {
-///     #[label(expected_identifier)]
-///     WithoutFound {
-///         #[primary_span]
-///         span: Span,
-///     }
-///     #[label(expected_identifier_found)]
-///     WithFound {
-///         #[primary_span]
-///         span: Span,
-///         found: String,
-///     }
-/// }
-///
-/// #[derive(Subdiagnostic)]
-/// #[suggestion(style = "verbose",parser::raw_identifier)]
-/// pub struct RawIdentifierSuggestion<'tcx> {
-///     #[primary_span]
-///     span: Span,
-///     #[applicability]
-///     applicability: Applicability,
-///     ident: Ident,
+/// pub(crate) enum BuiltinUnusedDocCommentSub {
+///     #[help("use `//` for a plain comment")]
+///     PlainHelp,
+///     #[help("use `/* */` for a plain comment")]
+///     BlockHelp,
 /// }
 /// ```
-///
-/// ```fluent
-/// parser_expected_identifier = expected identifier
-///
-/// parser_expected_identifier_found = expected identifier, found {$found}
-///
-/// parser_raw_identifier = escape `{$ident}` to use it as an identifier
-/// ```
-///
-/// Then, later, to add the subdiagnostic:
+/// Then, later, use the subdiagnostic in a diagnostic:
 ///
 /// ```ignore (rust)
-/// diag.subdiagnostic(ExpectedIdentifierLabel::WithoutFound { span });
-///
-/// diag.subdiagnostic(RawIdentifierSuggestion { span, applicability, ident });
+/// #[derive(LintDiagnostic)]
+/// #[diag("unused doc comment")]
+/// pub(crate) struct BuiltinUnusedDocComment<'a> {
+///     pub kind: &'a str,
+///     #[label("rustdoc does not generate documentation for {$kind}")]
+///     pub label: Span,
+///     #[subdiagnostic]
+///     pub sub: BuiltinUnusedDocCommentSub,
+/// }
 /// ```
 pub(super) fn subdiagnostic_derive(s: Structure<'_>) -> TokenStream {
     SubdiagnosticDerive::new().into_tokens(s)

From f4c135f91bfd13920faf671f044ea037ab7f7ea0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1543/1843] Port `rustc_trivial_field_reads` to the new
 attribute parser

---
 .../src/attributes/rustc_internal.rs          |  9 +++++
 compiler/rustc_attr_parsing/src/context.rs    | 36 ++++++++++++++-----
 .../rustc_hir/src/attrs/data_structures.rs    |  3 ++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 compiler/rustc_passes/src/dead.rs             |  8 +++--
 6 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 218488898e4b..09d0cffe3966 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -502,6 +502,15 @@ impl SingleAttributeParser for RustcNeverTypeOptionsParser {
     }
 }
 
+pub(crate) struct RustcTrivialFieldReadsParser;
+
+impl NoArgsAttributeParser for RustcTrivialFieldReadsParser {
+    const PATH: &[Symbol] = &[sym::rustc_trivial_field_reads];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcTrivialFieldReads;
+}
+
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 4d7dc41f1546..c42a9c6031b4 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -169,6 +169,32 @@ attribute_parsers!(
         // tidy-alphabetical-end
 
         // tidy-alphabetical-start
+%%%%%%% Changes from base to side #1
+%%%%%%%%%%% Changes from base to side #1
+         Single>,
+         Single>,
+         Single>,
+         Single>,
+         Single>,
+         Single>,
+         Single>,
++        Single>,
++++++++ Contents of side #2
+        Single>,
++++++++++++ Contents of side #2
+-        Single>,
+         Single>,
+<<<<<<< Conflict 1 of 1
+<<<<<<<<<<< Conflict 2 of 2
+>>>>>>> Conflict 1 of 1 ends
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
         Single,
         Single,
         Single,
@@ -213,6 +239,7 @@ attribute_parsers!(
         Single,
         Single,
         Single,
+        Single,
         Single,
         Single,
         Single,
@@ -297,14 +324,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
+>>>>>>>>>>> Conflict 2 of 2 ends
         // tidy-alphabetical-end
     ];
 );
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index e5f45928fe7b..123a38131ecd 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1326,6 +1326,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_then_this_would_need]`
     RustcThenThisWouldNeed(Span, ThinVec),
 
+    /// Represents `#[rustc_trivial_field_reads]`
+    RustcTrivialFieldReads,
+
     /// Represents `#[rustc_unsafe_specialization_marker]`.
     RustcUnsafeSpecializationMarker(Span),
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 5384a12a0ffb..9d7483dab125 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -167,6 +167,7 @@ impl AttributeKind {
             RustcStrictCoherence(..) => Yes,
             RustcSymbolName(..) => Yes,
             RustcThenThisWouldNeed(..) => No,
+            RustcTrivialFieldReads => Yes,
             RustcUnsafeSpecializationMarker(..) => No,
             RustcVariance => No,
             RustcVarianceOfOpaques => No,
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 4f29b5e715d6..a9d044ec9ecc 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -358,6 +358,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcStrictCoherence(..)
                     | AttributeKind::RustcSymbolName(..)
                     | AttributeKind::RustcThenThisWouldNeed(..)
+                    | AttributeKind::RustcTrivialFieldReads
                     | AttributeKind::RustcUnsafeSpecializationMarker(..)
                     | AttributeKind::RustcVariance
                     | AttributeKind::RustcVarianceOfOpaques
@@ -399,7 +400,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::rustc_no_mir_inline
                             | sym::rustc_nonnull_optimization_guaranteed
                             | sym::rustc_inherit_overflow_checks
-                            | sym::rustc_trivial_field_reads
                             | sym::rustc_on_unimplemented
                             | sym::rustc_do_not_const_check
                             | sym::rustc_doc_primitive
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 3294b6802a71..75bac9eff2c7 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -10,10 +10,11 @@ use hir::def_id::{LocalDefIdMap, LocalDefIdSet};
 use rustc_abi::FieldIdx;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::{ErrorGuaranteed, MultiSpan};
+use rustc_hir::attrs::AttributeKind;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
 use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::{self as hir, Node, PatKind, QPath};
+use rustc_hir::{self as hir, Node, PatKind, QPath, find_attr};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::privacy::Level;
 use rustc_middle::query::Providers;
@@ -380,7 +381,10 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
             && let impl_of = self.tcx.parent(impl_item.owner_id.to_def_id())
             && self.tcx.is_automatically_derived(impl_of)
             && let trait_ref = self.tcx.impl_trait_ref(impl_of).instantiate_identity()
-            && self.tcx.has_attr(trait_ref.def_id, sym::rustc_trivial_field_reads)
+            && find_attr!(
+                self.tcx.get_all_attrs(trait_ref.def_id),
+                AttributeKind::RustcTrivialFieldReads
+            )
         {
             if let ty::Adt(adt_def, _) = trait_ref.self_ty().kind()
                 && let Some(adt_def_id) = adt_def.did().as_local()

From eab26e329b246f98f86f64f935c9ef49d85ade04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1544/1843] Port `rustc_no_mir_inline` to the new attribute
 parser

---
 .../src/attributes/rustc_internal.rs          | 15 ++++++
 compiler/rustc_attr_parsing/src/context.rs    | 47 ++++++++-----------
 .../rustc_hir/src/attrs/data_structures.rs    |  3 ++
 .../rustc_hir/src/attrs/encode_cross_crate.rs |  1 +
 .../rustc_mir_transform/src/check_inline.rs   |  4 +-
 compiler/rustc_passes/src/check_attr.rs       |  2 +-
 6 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index 09d0cffe3966..d71fa791193f 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -511,6 +511,21 @@ impl NoArgsAttributeParser for RustcTrivialFieldReadsParser {
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcTrivialFieldReads;
 }
 
+pub(crate) struct RustcNoMirInlineParser;
+
+impl NoArgsAttributeParser for RustcNoMirInlineParser {
+    const PATH: &[Symbol] = &[sym::rustc_no_mir_inline];
+    const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::Trait { body: false })),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+    ]);
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNoMirInline;
+}
+
 pub(crate) struct RustcLintQueryInstabilityParser;
 
 impl NoArgsAttributeParser for RustcLintQueryInstabilityParser {
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index c42a9c6031b4..d6e9caeafc8a 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -169,32 +169,6 @@ attribute_parsers!(
         // tidy-alphabetical-end
 
         // tidy-alphabetical-start
-%%%%%%% Changes from base to side #1
-%%%%%%%%%%% Changes from base to side #1
-         Single>,
-         Single>,
-         Single>,
-         Single>,
-         Single>,
-         Single>,
-         Single>,
-+        Single>,
-+++++++ Contents of side #2
-        Single>,
-+++++++++++ Contents of side #2
--        Single>,
-         Single>,
-<<<<<<< Conflict 1 of 1
-<<<<<<<<<<< Conflict 2 of 2
->>>>>>> Conflict 1 of 1 ends
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
         Single,
         Single,
         Single,
@@ -237,9 +211,9 @@ attribute_parsers!(
         Single,
         Single,
         Single,
+        Single,
         Single,
         Single,
-        Single,
         Single,
         Single,
         Single,
@@ -315,6 +289,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
@@ -324,7 +299,23 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
->>>>>>>>>>> Conflict 2 of 2 ends
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
+        Single>,
         // tidy-alphabetical-end
     ];
 );
diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs
index 123a38131ecd..7050bb6ea70d 100644
--- a/compiler/rustc_hir/src/attrs/data_structures.rs
+++ b/compiler/rustc_hir/src/attrs/data_structures.rs
@@ -1249,6 +1249,9 @@ pub enum AttributeKind {
     /// Represents `#[rustc_no_implicit_bounds]`
     RustcNoImplicitBounds,
 
+    /// Represents `#[rustc_no_mir_inline]`
+    RustcNoMirInline,
+
     /// Represents `#[rustc_non_const_trait_method]`.
     RustcNonConstTraitMethod,
 
diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
index 9d7483dab125..73ac1533cef5 100644
--- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
+++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs
@@ -143,6 +143,7 @@ impl AttributeKind {
             RustcNeverTypeOptions { .. } => No,
             RustcNoImplicitAutorefs => Yes,
             RustcNoImplicitBounds => No,
+            RustcNoMirInline => Yes,
             RustcNonConstTraitMethod => No, // should be reported via other queries like `constness`
             RustcNounwind => No,
             RustcObjcClass { .. } => No,
diff --git a/compiler/rustc_mir_transform/src/check_inline.rs b/compiler/rustc_mir_transform/src/check_inline.rs
index 2d1876426727..1f65bd1ba69b 100644
--- a/compiler/rustc_mir_transform/src/check_inline.rs
+++ b/compiler/rustc_mir_transform/src/check_inline.rs
@@ -8,7 +8,6 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::{Body, TerminatorKind};
 use rustc_middle::ty;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::sym;
 
 use crate::pass_manager::MirLint;
 
@@ -42,7 +41,8 @@ pub(super) fn is_inline_valid_on_fn<'tcx>(
     def_id: DefId,
 ) -> Result<(), &'static str> {
     let codegen_attrs = tcx.codegen_fn_attrs(def_id);
-    if tcx.has_attr(def_id, sym::rustc_no_mir_inline) {
+
+    if find_attr!(tcx.get_all_attrs(def_id), AttributeKind::RustcNoMirInline) {
         return Err("#[rustc_no_mir_inline]");
     }
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index a9d044ec9ecc..2287b8bf483f 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -336,6 +336,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     | AttributeKind::RustcNeverTypeOptions {..}
                     | AttributeKind::RustcNoImplicitAutorefs
                     | AttributeKind::RustcNoImplicitBounds
+                    | AttributeKind::RustcNoMirInline
                     | AttributeKind::RustcNonConstTraitMethod
                     | AttributeKind::RustcNounwind
                     | AttributeKind::RustcObjcClass { .. }
@@ -397,7 +398,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             | sym::lang
                             | sym::default_lib_allocator
                             | sym::rustc_diagnostic_item
-                            | sym::rustc_no_mir_inline
                             | sym::rustc_nonnull_optimization_guaranteed
                             | sym::rustc_inherit_overflow_checks
                             | sym::rustc_on_unimplemented

From fe7530cad2736e1cb841793c20a23a2d3d3ab2f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sat, 7 Feb 2026 17:57:43 +0100
Subject: [PATCH 1545/1843] Make `rustc_object_lifetime_default` use
 NoArgsAttributeParser

---
 .../src/attributes/rustc_internal.rs               | 14 ++------------
 compiler/rustc_attr_parsing/src/context.rs         | 11 ++++++++++-
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index d71fa791193f..fa1a43058561 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -575,21 +575,11 @@ impl NoArgsAttributeParser for RustcLintUntrackedQueryInformationPa
 
 pub(crate) struct RustcObjectLifetimeDefaultParser;
 
-impl SingleAttributeParser for RustcObjectLifetimeDefaultParser {
+impl NoArgsAttributeParser for RustcObjectLifetimeDefaultParser {
     const PATH: &[Symbol] = &[sym::rustc_object_lifetime_default];
-    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]);
-    const TEMPLATE: AttributeTemplate = template!(Word);
-
-    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option {
-        if let Err(span) = args.no_args() {
-            cx.expected_no_args(span);
-            return None;
-        }
-
-        Some(AttributeKind::RustcObjectLifetimeDefault)
-    }
+    const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcObjectLifetimeDefault;
 }
 
 pub(crate) struct RustcSimdMonomorphizeLaneLimitParser;
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index d6e9caeafc8a..5906d7a4d513 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -212,7 +212,6 @@ attribute_parsers!(
         Single,
         Single,
         Single,
-        Single,
         Single,
         Single,
         Single,
@@ -292,6 +291,7 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
@@ -301,19 +301,28 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
+        Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
+        Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
         Single>,
+        Single>,
+        Single>,
         Single>,
         Single>,
         Single>,
         Single>,
+        Single>,
         Single>,
         Single>,
+        Single>,
+        Single>,
         Single>,
         Single>,
         // tidy-alphabetical-end

From 628c37afadaea6702f95fef3b086e99980afb7ce Mon Sep 17 00:00:00 2001
From: Oli Scherer 
Date: Thu, 4 Dec 2025 08:55:33 +0000
Subject: [PATCH 1546/1843] Restrict the set of things that const stability can
 be applied

---
 .../src/attributes/stability.rs                  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/stability.rs b/compiler/rustc_attr_parsing/src/attributes/stability.rs
index 1f01cadcb43e..fb204e77abc8 100644
--- a/compiler/rustc_attr_parsing/src/attributes/stability.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/stability.rs
@@ -244,7 +244,21 @@ impl AttributeParser for ConstStabilityParser {
             this.promotable = true;
         }),
     ];
-    const ALLOWED_TARGETS: AllowedTargets = ALLOWED_TARGETS;
+    const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
+        Allow(Target::Fn),
+        Allow(Target::Method(MethodKind::Inherent)),
+        Allow(Target::Method(MethodKind::TraitImpl)),
+        Allow(Target::Method(MethodKind::Trait { body: true })),
+        Allow(Target::Impl { of_trait: false }),
+        Allow(Target::Impl { of_trait: true }),
+        Allow(Target::Use), // FIXME I don't think this does anything?
+        Allow(Target::Const),
+        Allow(Target::AssocConst),
+        Allow(Target::Trait),
+        Allow(Target::Static),
+        Allow(Target::Crate),
+        Allow(Target::MacroDef), // FIXME(oli-obk): remove this and eliminate the manual check for it
+    ]);
 
     fn finalize(mut self, cx: &FinalizeContext<'_, '_, S>) -> Option {
         if self.promotable {

From 475b9d9819a092e1820a07c688fdc7cd7b26c6aa Mon Sep 17 00:00:00 2001
From: Oli Scherer 
Date: Tue, 10 Feb 2026 09:37:30 +0000
Subject: [PATCH 1547/1843] Prevent const stability attrs from being applied to
 macros via the normal attribute logic instead of special cased checks

---
 .../src/attributes/stability.rs               |  1 -
 compiler/rustc_expand/src/base.rs             |  9 ---------
 compiler/rustc_expand/src/errors.rs           | 10 ----------
 .../ui/attributes/const-stability-on-macro.rs |  6 +++---
 .../const-stability-on-macro.stderr           | 20 +++++++++----------
 5 files changed, 12 insertions(+), 34 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/stability.rs b/compiler/rustc_attr_parsing/src/attributes/stability.rs
index fb204e77abc8..a2be2d42b3e1 100644
--- a/compiler/rustc_attr_parsing/src/attributes/stability.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/stability.rs
@@ -257,7 +257,6 @@ impl AttributeParser for ConstStabilityParser {
         Allow(Target::Trait),
         Allow(Target::Static),
         Allow(Target::Crate),
-        Allow(Target::MacroDef), // FIXME(oli-obk): remove this and eliminate the manual check for it
     ]);
 
     fn finalize(mut self, cx: &FinalizeContext<'_, '_, S>) -> Option {
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 6b3c7fe97932..91f98d475f00 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -962,15 +962,6 @@ impl SyntaxExtension {
 
         let stability = find_attr!(attrs, AttributeKind::Stability { stability, .. } => *stability);
 
-        // FIXME(jdonszelmann): make it impossible to miss the or_else in the typesystem
-        if let Some(sp) =
-            find_attr!(attrs, AttributeKind::RustcConstStability { span, .. } => *span)
-        {
-            sess.dcx().emit_err(errors::MacroConstStability {
-                span: sp,
-                head_span: sess.source_map().guess_head_span(span),
-            });
-        }
         if let Some(sp) = find_attr!(attrs, AttributeKind::RustcBodyStability{ span, .. } => *span)
         {
             sess.dcx().emit_err(errors::MacroBodyStability {
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index e64bfc136d16..b6fcc13321ee 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -80,16 +80,6 @@ pub(crate) struct ResolveRelativePath {
     pub path: String,
 }
 
-#[derive(Diagnostic)]
-#[diag("macros cannot have const stability attributes")]
-pub(crate) struct MacroConstStability {
-    #[primary_span]
-    #[label("invalid const stability attribute")]
-    pub span: Span,
-    #[label("const stability attribute affects this macro")]
-    pub head_span: Span,
-}
-
 #[derive(Diagnostic)]
 #[diag("macros cannot have body stability attributes")]
 pub(crate) struct MacroBodyStability {
diff --git a/tests/ui/attributes/const-stability-on-macro.rs b/tests/ui/attributes/const-stability-on-macro.rs
index af268ccd5366..4336370430d4 100644
--- a/tests/ui/attributes/const-stability-on-macro.rs
+++ b/tests/ui/attributes/const-stability-on-macro.rs
@@ -2,13 +2,13 @@
 #![stable(feature = "rust1", since = "1.0.0")]
 
 #[rustc_const_stable(feature = "foo", since = "3.3.3")]
-//~^ ERROR macros cannot have const stability attributes
+//~^ ERROR attribute cannot be used on macro defs
 macro_rules! foo {
     () => {};
 }
 
-#[rustc_const_unstable(feature = "bar", issue="none")]
-//~^ ERROR macros cannot have const stability attributes
+#[rustc_const_unstable(feature = "bar", issue = "none")]
+//~^ ERROR attribute cannot be used on macro defs
 macro_rules! bar {
     () => {};
 }
diff --git a/tests/ui/attributes/const-stability-on-macro.stderr b/tests/ui/attributes/const-stability-on-macro.stderr
index 28f31e3d4f61..a8b56edfa59d 100644
--- a/tests/ui/attributes/const-stability-on-macro.stderr
+++ b/tests/ui/attributes/const-stability-on-macro.stderr
@@ -1,20 +1,18 @@
-error: macros cannot have const stability attributes
+error: `#[rustc_const_stable]` attribute cannot be used on macro defs
   --> $DIR/const-stability-on-macro.rs:4:1
    |
 LL | #[rustc_const_stable(feature = "foo", since = "3.3.3")]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid const stability attribute
-LL |
-LL | macro_rules! foo {
-   | ---------------- const stability attribute affects this macro
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: `#[rustc_const_stable]` can be applied to associated consts, constants, crates, functions, impl blocks, statics, traits, and use statements
 
-error: macros cannot have const stability attributes
+error: `#[rustc_const_unstable]` attribute cannot be used on macro defs
   --> $DIR/const-stability-on-macro.rs:10:1
    |
-LL | #[rustc_const_unstable(feature = "bar", issue="none")]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid const stability attribute
-LL |
-LL | macro_rules! bar {
-   | ---------------- const stability attribute affects this macro
+LL | #[rustc_const_unstable(feature = "bar", issue = "none")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: `#[rustc_const_unstable]` can be applied to associated consts, constants, crates, functions, impl blocks, statics, traits, and use statements
 
 error: aborting due to 2 previous errors
 

From 12dbc1bc635f6b4c86e1316a2bf4ca59749f9f09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= 
Date: Sun, 8 Feb 2026 22:14:00 +0100
Subject: [PATCH 1548/1843] remove 'static in many places

---
 .../src/attributes/allow_unstable.rs           |  2 +-
 .../src/attributes/inline.rs                   |  4 ++--
 .../rustc_attr_parsing/src/attributes/repr.rs  |  4 ++--
 .../src/attributes/rustc_internal.rs           | 10 +++++-----
 compiler/rustc_attr_parsing/src/context.rs     | 18 ------------------
 5 files changed, 10 insertions(+), 28 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
index d825d770fa35..4f1a8cd8b403 100644
--- a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
@@ -29,7 +29,7 @@ impl CombineAttributeParser for AllowInternalUnstableParser {
 
 pub(crate) struct UnstableFeatureBoundParser;
 impl CombineAttributeParser for UnstableFeatureBoundParser {
-    const PATH: &'static [rustc_span::Symbol] = &[sym::unstable_feature_bound];
+    const PATH: &[rustc_span::Symbol] = &[sym::unstable_feature_bound];
     type Item = (Symbol, Span);
     const CONVERT: ConvertFn = |items, _| AttributeKind::UnstableFeatureBound(items);
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
diff --git a/compiler/rustc_attr_parsing/src/attributes/inline.rs b/compiler/rustc_attr_parsing/src/attributes/inline.rs
index f6aab9ea0ee2..767200bfa9bf 100644
--- a/compiler/rustc_attr_parsing/src/attributes/inline.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/inline.rs
@@ -10,7 +10,7 @@ use super::prelude::*;
 pub(crate) struct InlineParser;
 
 impl SingleAttributeParser for InlineParser {
-    const PATH: &'static [Symbol] = &[sym::inline];
+    const PATH: &[Symbol] = &[sym::inline];
     const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
@@ -67,7 +67,7 @@ impl SingleAttributeParser for InlineParser {
 pub(crate) struct RustcForceInlineParser;
 
 impl SingleAttributeParser for RustcForceInlineParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_force_inline];
+    const PATH: &[Symbol] = &[sym::rustc_force_inline];
     const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
diff --git a/compiler/rustc_attr_parsing/src/attributes/repr.rs b/compiler/rustc_attr_parsing/src/attributes/repr.rs
index 34d4957edc70..fb0b8df65284 100644
--- a/compiler/rustc_attr_parsing/src/attributes/repr.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/repr.rs
@@ -272,7 +272,7 @@ fn parse_alignment(node: &LitKind) -> Result {
 pub(crate) struct AlignParser(Option<(Align, Span)>);
 
 impl AlignParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_align];
+    const PATH: &[Symbol] = &[sym::rustc_align];
     const TEMPLATE: AttributeTemplate = template!(List: &[""]);
 
     fn parse(&mut self, cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) {
@@ -329,7 +329,7 @@ impl AttributeParser for AlignParser {
 pub(crate) struct AlignStaticParser(AlignParser);
 
 impl AlignStaticParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_align_static];
+    const PATH: &[Symbol] = &[sym::rustc_align_static];
     const TEMPLATE: AttributeTemplate = AlignParser::TEMPLATE;
 
     fn parse(&mut self, cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) {
diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
index fa1a43058561..0d19dc25d402 100644
--- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
@@ -18,7 +18,7 @@ use crate::session_diagnostics::{
 pub(crate) struct RustcMainParser;
 
 impl NoArgsAttributeParser for RustcMainParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_main];
+    const PATH: &[Symbol] = &[sym::rustc_main];
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
     const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcMain;
@@ -103,7 +103,7 @@ impl NoArgsAttributeParser for RustcNoImplicitAutorefsParser {
 pub(crate) struct RustcLayoutScalarValidRangeStartParser;
 
 impl SingleAttributeParser for RustcLayoutScalarValidRangeStartParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_layout_scalar_valid_range_start];
+    const PATH: &[Symbol] = &[sym::rustc_layout_scalar_valid_range_start];
     const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]);
@@ -118,7 +118,7 @@ impl SingleAttributeParser for RustcLayoutScalarValidRangeStartPars
 pub(crate) struct RustcLayoutScalarValidRangeEndParser;
 
 impl SingleAttributeParser for RustcLayoutScalarValidRangeEndParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_layout_scalar_valid_range_end];
+    const PATH: &[Symbol] = &[sym::rustc_layout_scalar_valid_range_end];
     const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]);
@@ -822,7 +822,7 @@ impl CombineAttributeParser for RustcMirParser {
 pub(crate) struct RustcNonConstTraitMethodParser;
 
 impl NoArgsAttributeParser for RustcNonConstTraitMethodParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_non_const_trait_method];
+    const PATH: &[Symbol] = &[sym::rustc_non_const_trait_method];
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
         Allow(Target::Method(MethodKind::Trait { body: true })),
@@ -1067,7 +1067,7 @@ impl NoArgsAttributeParser for RustcInsignificantDtorParser {
 pub(crate) struct RustcEffectiveVisibilityParser;
 
 impl NoArgsAttributeParser for RustcEffectiveVisibilityParser {
-    const PATH: &'static [Symbol] = &[sym::rustc_effective_visibility];
+    const PATH: &[Symbol] = &[sym::rustc_effective_visibility];
     const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error;
     const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
         Allow(Target::Use),
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 5906d7a4d513..47b988bcf338 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -211,7 +211,6 @@ attribute_parsers!(
         Single,
         Single,
         Single,
-        Single,
         Single,
         Single,
         Single,
@@ -300,30 +299,13 @@ attribute_parsers!(
         Single>,
         Single>,
         Single>,
-        Single>,
-        Single>,
-        Single>,
         Single>,
         Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
         Single>,
         Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
         Single>,
         Single>,
-        Single>,
-        Single>,
         Single>,
-        Single>,
-        Single>,
-        Single>,
-        Single>,
         Single>,
         // tidy-alphabetical-end
     ];

From 32e49a8b4016265459a39ca0c7cf193a76dfe814 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= 
Date: Tue, 10 Feb 2026 12:04:09 +0100
Subject: [PATCH 1549/1843] Reenable a GCI+mGCA+GCPT test case

---
 .../assoc-const-bindings.rs                    | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/ui/generic-const-items/assoc-const-bindings.rs b/tests/ui/generic-const-items/assoc-const-bindings.rs
index 2bf4bea16deb..c187c87c7763 100644
--- a/tests/ui/generic-const-items/assoc-const-bindings.rs
+++ b/tests/ui/generic-const-items/assoc-const-bindings.rs
@@ -1,35 +1,33 @@
 //@ check-pass
 
 #![feature(generic_const_items, min_generic_const_args)]
-#![feature(adt_const_params)]
-#![allow(incomplete_features)]
+#![feature(adt_const_params, unsized_const_params, generic_const_parameter_types)]
+#![expect(incomplete_features)]
+
+use std::marker::{ConstParamTy, ConstParamTy_};
 
 trait Owner {
     type const C: u32;
     type const K: u32;
-    // #[type_const]
-    // const Q: Maybe;
+    type const Q: Maybe;
 }
 
 impl Owner for () {
     type const C: u32 = N;
     type const K: u32 = const { 99 + 1 };
-    // FIXME(mgca): re-enable once we properly support ctors and generics on paths
-    // #[type_const]
-    // const Q: Maybe = Maybe::Nothing;
+    type const Q: Maybe = Maybe::Nothing::;
 }
 
 fn take0(_: impl Owner = { N }>) {}
 fn take1(_: impl Owner = 100>) {}
-// FIXME(mgca): re-enable once we properly support ctors and generics on paths
-// fn take2(_: impl Owner = { Maybe::Just(()) }>) {}
+fn take2(_: impl Owner = { Maybe::Just::<()>(()) }>) {}
 
 fn main() {
     take0::<128>(());
     take1(());
 }
 
-#[derive(PartialEq, Eq, std::marker::ConstParamTy)]
+#[derive(PartialEq, Eq, ConstParamTy)]
 enum Maybe {
     Nothing,
     Just(T),

From 85e8282fabe0beb071aca75a2c3a51068eaa1c12 Mon Sep 17 00:00:00 2001
From: Oli Scherer 
Date: Fri, 24 Jan 2025 15:57:13 +0000
Subject: [PATCH 1550/1843] Start using pattern types in libcore

---
 .../src/debuginfo/metadata.rs                 |  13 ++-
 .../rustc_const_eval/src/interpret/call.rs    |   3 +
 library/core/src/num/niche_types.rs           | 103 +++++++-----------
 src/bootstrap/src/core/build_steps/test.rs    |   3 +
 .../cast_fn_ptr_invalid_callee_ret.stderr     |   2 +-
 .../cast_fn_ptr_invalid_caller_arg.stderr     |   2 +-
 .../consts/const-eval/raw-bytes.32bit.stderr  |   4 +-
 .../consts/const-eval/raw-bytes.64bit.stderr  |   4 +-
 tests/ui/consts/const-eval/ub-nonnull.stderr  |   4 +-
 tests/ui/lint/invalid_value.stderr            |   2 -
 10 files changed, 65 insertions(+), 75 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index 1c674f0e8a85..c759d46b7d2f 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -479,7 +479,18 @@ pub(crate) fn spanned_type_di_node<'ll, 'tcx>(
             AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span),
         },
         ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id),
-        _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t),
+        ty::Pat(base, _) => return type_di_node(cx, base),
+        // FIXME(unsafe_binders): impl debug info
+        ty::UnsafeBinder(_) => unimplemented!(),
+        ty::Alias(..)
+        | ty::Param(_)
+        | ty::Bound(..)
+        | ty::Infer(_)
+        | ty::Placeholder(_)
+        | ty::CoroutineWitness(..)
+        | ty::Error(_) => {
+            bug!("debuginfo: unexpected type in type_di_node(): {:?}", t)
+        }
     };
 
     {
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs
index c7ddcaa73157..ffa7b09205d5 100644
--- a/compiler/rustc_const_eval/src/interpret/call.rs
+++ b/compiler/rustc_const_eval/src/interpret/call.rs
@@ -91,6 +91,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let (_, field) = layout.non_1zst_field(self).unwrap();
                 self.unfold_transparent(field, may_unfold)
             }
+            ty::Pat(base, _) => self.layout_of(*base).expect(
+                "if the layout of a pattern type could be computed, so can the layout of its base",
+            ),
             // Not a transparent type, no further unfolding.
             _ => layout,
         }
diff --git a/library/core/src/num/niche_types.rs b/library/core/src/num/niche_types.rs
index 9ac0eb72bdcb..33b2a6741abd 100644
--- a/library/core/src/num/niche_types.rs
+++ b/library/core/src/num/niche_types.rs
@@ -5,45 +5,33 @@
 )]
 
 use crate::cmp::Ordering;
-use crate::fmt;
 use crate::hash::{Hash, Hasher};
 use crate::marker::StructuralPartialEq;
+use crate::{fmt, pattern_type};
 
 macro_rules! define_valid_range_type {
     ($(
         $(#[$m:meta])*
-        $vis:vis struct $name:ident($int:ident as $uint:ident in $low:literal..=$high:literal);
+        $vis:vis struct $name:ident($int:ident is $pat:pat);
     )+) => {$(
-        #[derive(Clone, Copy, Eq)]
+        #[derive(Clone, Copy)]
         #[repr(transparent)]
-        #[rustc_layout_scalar_valid_range_start($low)]
-        #[rustc_layout_scalar_valid_range_end($high)]
         $(#[$m])*
-        $vis struct $name($int);
-
-        const _: () = {
-            // With the `valid_range` attributes, it's always specified as unsigned
-            assert!(<$uint>::MIN == 0);
-            let ulow: $uint = $low;
-            let uhigh: $uint = $high;
-            assert!(ulow <= uhigh);
-
-            assert!(size_of::<$int>() == size_of::<$uint>());
-        };
-
+        $vis struct $name(pattern_type!($int is $pat));
         impl $name {
             #[inline]
             pub const fn new(val: $int) -> Option {
-                if (val as $uint) >= ($low as $uint) && (val as $uint) <= ($high as $uint) {
-                    // SAFETY: just checked the inclusive range
-                    Some(unsafe { $name(val) })
+                #[allow(non_contiguous_range_endpoints)]
+                if let $pat = val {
+                    // SAFETY: just checked that the value matches the pattern
+                    Some(unsafe { $name(crate::mem::transmute(val)) })
                 } else {
                     None
                 }
             }
 
             /// Constructs an instance of this type from the underlying integer
-            /// primitive without checking whether its zero.
+            /// primitive without checking whether its valid.
             ///
             /// # Safety
             /// Immediate language UB if `val` is not within the valid range for this
@@ -51,13 +39,13 @@ macro_rules! define_valid_range_type {
             #[inline]
             pub const unsafe fn new_unchecked(val: $int) -> Self {
                 // SAFETY: Caller promised that `val` is within the valid range.
-                unsafe { $name(val) }
+                unsafe { crate::mem::transmute(val) }
             }
 
             #[inline]
             pub const fn as_inner(self) -> $int {
-                // SAFETY: This is a transparent wrapper, so unwrapping it is sound
-                // (Not using `.0` due to MCP#807.)
+                // SAFETY: pattern types are always legal values of their base type
+                // (Not using `.0` because that has perf regressions.)
                 unsafe { crate::mem::transmute(self) }
             }
         }
@@ -67,6 +55,8 @@ macro_rules! define_valid_range_type {
         // by .
         impl StructuralPartialEq for $name {}
 
+        impl Eq for $name {}
+
         impl PartialEq for $name {
             #[inline]
             fn eq(&self, other: &Self) -> bool {
@@ -104,7 +94,7 @@ macro_rules! define_valid_range_type {
 }
 
 define_valid_range_type! {
-    pub struct Nanoseconds(u32 as u32 in 0..=999_999_999);
+    pub struct Nanoseconds(u32 is 0..=999_999_999);
 }
 
 impl Nanoseconds {
@@ -120,47 +110,32 @@ impl const Default for Nanoseconds {
     }
 }
 
-define_valid_range_type! {
-    pub struct NonZeroU8Inner(u8 as u8 in 1..=0xff);
-    pub struct NonZeroU16Inner(u16 as u16 in 1..=0xff_ff);
-    pub struct NonZeroU32Inner(u32 as u32 in 1..=0xffff_ffff);
-    pub struct NonZeroU64Inner(u64 as u64 in 1..=0xffffffff_ffffffff);
-    pub struct NonZeroU128Inner(u128 as u128 in 1..=0xffffffffffffffff_ffffffffffffffff);
-
-    pub struct NonZeroI8Inner(i8 as u8 in 1..=0xff);
-    pub struct NonZeroI16Inner(i16 as u16 in 1..=0xff_ff);
-    pub struct NonZeroI32Inner(i32 as u32 in 1..=0xffff_ffff);
-    pub struct NonZeroI64Inner(i64 as u64 in 1..=0xffffffff_ffffffff);
-    pub struct NonZeroI128Inner(i128 as u128 in 1..=0xffffffffffffffff_ffffffffffffffff);
-
-    pub struct NonZeroCharInner(char as u32 in 1..=0x10ffff);
-}
-
-#[cfg(target_pointer_width = "16")]
-define_valid_range_type! {
-    pub struct UsizeNoHighBit(usize as usize in 0..=0x7fff);
-    pub struct NonZeroUsizeInner(usize as usize in 1..=0xffff);
-    pub struct NonZeroIsizeInner(isize as usize in 1..=0xffff);
-}
-#[cfg(target_pointer_width = "32")]
-define_valid_range_type! {
-    pub struct UsizeNoHighBit(usize as usize in 0..=0x7fff_ffff);
-    pub struct NonZeroUsizeInner(usize as usize in 1..=0xffff_ffff);
-    pub struct NonZeroIsizeInner(isize as usize in 1..=0xffff_ffff);
-}
-#[cfg(target_pointer_width = "64")]
-define_valid_range_type! {
-    pub struct UsizeNoHighBit(usize as usize in 0..=0x7fff_ffff_ffff_ffff);
-    pub struct NonZeroUsizeInner(usize as usize in 1..=0xffff_ffff_ffff_ffff);
-    pub struct NonZeroIsizeInner(isize as usize in 1..=0xffff_ffff_ffff_ffff);
-}
+const HALF_USIZE: usize = usize::MAX >> 1;
 
 define_valid_range_type! {
-    pub struct U32NotAllOnes(u32 as u32 in 0..=0xffff_fffe);
-    pub struct I32NotAllOnes(i32 as u32 in 0..=0xffff_fffe);
+    pub struct NonZeroU8Inner(u8 is 1..);
+    pub struct NonZeroU16Inner(u16 is 1..);
+    pub struct NonZeroU32Inner(u32 is 1..);
+    pub struct NonZeroU64Inner(u64 is 1..);
+    pub struct NonZeroU128Inner(u128 is 1..);
 
-    pub struct U64NotAllOnes(u64 as u64 in 0..=0xffff_ffff_ffff_fffe);
-    pub struct I64NotAllOnes(i64 as u64 in 0..=0xffff_ffff_ffff_fffe);
+    pub struct NonZeroI8Inner(i8 is ..0 | 1..);
+    pub struct NonZeroI16Inner(i16 is ..0 | 1..);
+    pub struct NonZeroI32Inner(i32 is ..0 | 1..);
+    pub struct NonZeroI64Inner(i64 is ..0 | 1..);
+    pub struct NonZeroI128Inner(i128 is ..0 | 1..);
+
+    pub struct UsizeNoHighBit(usize is 0..=HALF_USIZE);
+    pub struct NonZeroUsizeInner(usize is 1..);
+    pub struct NonZeroIsizeInner(isize is ..0 | 1..);
+
+    pub struct U32NotAllOnes(u32 is 0..u32::MAX);
+    pub struct I32NotAllOnes(i32 is ..-1 | 0..);
+
+    pub struct U64NotAllOnes(u64 is 0..u64::MAX);
+    pub struct I64NotAllOnes(i64 is ..-1 | 0..);
+
+    pub struct NonZeroCharInner(char is '\u{1}' ..= '\u{10ffff}');
 }
 
 pub trait NotAllOnesHelper {
@@ -181,7 +156,7 @@ impl NotAllOnesHelper for i64 {
 }
 
 define_valid_range_type! {
-    pub struct CodePointInner(u32 as u32 in 0..=0x10ffff);
+    pub struct CodePointInner(u32 is 0..=0x10ffff);
 }
 
 impl CodePointInner {
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index 1ab76ef8f91d..fda9f3bbba3a 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -2184,6 +2184,9 @@ Please disable assertions with `rust.debug-assertions = false`.
         for flag in targetflags {
             cmd.arg("--target-rustcflags").arg(flag);
         }
+        if target.is_synthetic() {
+            cmd.arg("--target-rustcflags").arg("-Zunstable-options");
+        }
 
         cmd.arg("--python").arg(
             builder.config.python.as_ref().expect("python is required for running rustdoc tests"),
diff --git a/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_callee_ret.stderr b/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_callee_ret.stderr
index a00d167df2f7..6fc1bc01655c 100644
--- a/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_callee_ret.stderr
+++ b/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_callee_ret.stderr
@@ -1,4 +1,4 @@
-error: Undefined Behavior: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error: Undefined Behavior: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> tests/fail/validity/cast_fn_ptr_invalid_callee_ret.rs:LL:CC
    |
 LL |     f();
diff --git a/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_caller_arg.stderr b/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_caller_arg.stderr
index c2e647aa18fc..3b23a6cc4e85 100644
--- a/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_caller_arg.stderr
+++ b/src/tools/miri/tests/fail/validity/cast_fn_ptr_invalid_caller_arg.stderr
@@ -1,4 +1,4 @@
-error: Undefined Behavior: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error: Undefined Behavior: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> tests/fail/validity/cast_fn_ptr_invalid_caller_arg.rs:LL:CC
    |
 LL |             Call(_res = f(*ptr), ReturnTo(retblock), UnwindContinue())
diff --git a/tests/ui/consts/const-eval/raw-bytes.32bit.stderr b/tests/ui/consts/const-eval/raw-bytes.32bit.stderr
index 2861f82ec53b..be3b539b269a 100644
--- a/tests/ui/consts/const-eval/raw-bytes.32bit.stderr
+++ b/tests/ui/consts/const-eval/raw-bytes.32bit.stderr
@@ -64,7 +64,7 @@ LL | const NULL_PTR: NonNull = unsafe { mem::transmute(0usize) };
                00 00 00 00                                     │ ....
            }
 
-error[E0080]: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error[E0080]: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> $DIR/raw-bytes.rs:61:1
    |
 LL | const NULL_U8: NonZero = unsafe { mem::transmute(0u8) };
@@ -75,7 +75,7 @@ LL | const NULL_U8: NonZero = unsafe { mem::transmute(0u8) };
                00                                              │ .
            }
 
-error[E0080]: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error[E0080]: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> $DIR/raw-bytes.rs:63:1
    |
 LL | const NULL_USIZE: NonZero = unsafe { mem::transmute(0usize) };
diff --git a/tests/ui/consts/const-eval/raw-bytes.64bit.stderr b/tests/ui/consts/const-eval/raw-bytes.64bit.stderr
index 8e6dc66a40ee..9950ac726ca7 100644
--- a/tests/ui/consts/const-eval/raw-bytes.64bit.stderr
+++ b/tests/ui/consts/const-eval/raw-bytes.64bit.stderr
@@ -64,7 +64,7 @@ LL | const NULL_PTR: NonNull = unsafe { mem::transmute(0usize) };
                00 00 00 00 00 00 00 00                         │ ........
            }
 
-error[E0080]: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error[E0080]: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> $DIR/raw-bytes.rs:61:1
    |
 LL | const NULL_U8: NonZero = unsafe { mem::transmute(0u8) };
@@ -75,7 +75,7 @@ LL | const NULL_U8: NonZero = unsafe { mem::transmute(0u8) };
                00                                              │ .
            }
 
-error[E0080]: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error[E0080]: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> $DIR/raw-bytes.rs:63:1
    |
 LL | const NULL_USIZE: NonZero = unsafe { mem::transmute(0usize) };
diff --git a/tests/ui/consts/const-eval/ub-nonnull.stderr b/tests/ui/consts/const-eval/ub-nonnull.stderr
index e4486e3c500b..be5c6f77a087 100644
--- a/tests/ui/consts/const-eval/ub-nonnull.stderr
+++ b/tests/ui/consts/const-eval/ub-nonnull.stderr
@@ -15,7 +15,7 @@ error[E0080]: in-bounds pointer arithmetic failed: attempting to offset pointer
 LL |     let out_of_bounds_ptr = &ptr[255];
    |                             ^^^^^^^^^ evaluation of `OUT_OF_BOUNDS_PTR` failed here
 
-error[E0080]: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error[E0080]: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> $DIR/ub-nonnull.rs:26:1
    |
 LL | const NULL_U8: NonZero = unsafe { mem::transmute(0u8) };
@@ -26,7 +26,7 @@ LL | const NULL_U8: NonZero = unsafe { mem::transmute(0u8) };
                HEX_DUMP
            }
 
-error[E0080]: constructing invalid value at .0: encountered 0, but expected something greater or equal to 1
+error[E0080]: constructing invalid value at .0.0: encountered 0, but expected something greater or equal to 1
   --> $DIR/ub-nonnull.rs:28:1
    |
 LL | const NULL_USIZE: NonZero = unsafe { mem::transmute(0usize) };
diff --git a/tests/ui/lint/invalid_value.stderr b/tests/ui/lint/invalid_value.stderr
index cc6a2a1c8e53..3dd2a521ff2e 100644
--- a/tests/ui/lint/invalid_value.stderr
+++ b/tests/ui/lint/invalid_value.stderr
@@ -333,7 +333,6 @@ LL |         let _val: (NonZero, i32) = mem::uninitialized();
    |
    = note: `std::num::NonZero` must be non-null
    = note: because `core::num::niche_types::NonZeroU32Inner` must be non-null
-   = note: integers must be initialized
 
 error: the type `*const dyn Send` does not permit zero-initialization
   --> $DIR/invalid_value.rs:97:37
@@ -430,7 +429,6 @@ note: because `std::num::NonZero` must be non-null (in this field of the on
 LL |     Banana(NonZero),
    |            ^^^^^^^^^^^^
    = note: because `core::num::niche_types::NonZeroU32Inner` must be non-null
-   = note: integers must be initialized
 
 error: the type `bool` does not permit being left uninitialized
   --> $DIR/invalid_value.rs:111:26

From c9b5c934ca7acc33ff4aebbe1630cb02bb4c0f1f Mon Sep 17 00:00:00 2001
From: Folkert de Vries 
Date: Tue, 10 Feb 2026 11:39:06 +0100
Subject: [PATCH 1551/1843] Fix passing/returning structs with the 64-bit SPARC
 ABI

Co-authored-by: beetrees 
---
 compiler/rustc_abi/src/callconv/reg.rs        |   1 +
 compiler/rustc_target/src/callconv/sparc64.rs | 342 +++++++++---------
 tests/assembly-llvm/sparc-struct-abi.rs       | 193 +++++++++-
 tests/codegen-llvm/cast-target-abi.rs         |  16 +-
 4 files changed, 366 insertions(+), 186 deletions(-)

diff --git a/compiler/rustc_abi/src/callconv/reg.rs b/compiler/rustc_abi/src/callconv/reg.rs
index 66c8056d0c2a..66d4dca00726 100644
--- a/compiler/rustc_abi/src/callconv/reg.rs
+++ b/compiler/rustc_abi/src/callconv/reg.rs
@@ -35,6 +35,7 @@ impl Reg {
 
     reg_ctor!(f32, Float, 32);
     reg_ctor!(f64, Float, 64);
+    reg_ctor!(f128, Float, 128);
 }
 
 impl Reg {
diff --git a/compiler/rustc_target/src/callconv/sparc64.rs b/compiler/rustc_target/src/callconv/sparc64.rs
index fc732170dcb7..f55d03d89e8f 100644
--- a/compiler/rustc_target/src/callconv/sparc64.rs
+++ b/compiler/rustc_target/src/callconv/sparc64.rs
@@ -1,214 +1,203 @@
-// FIXME: This needs an audit for correctness and completeness.
-
 use rustc_abi::{
-    BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Scalar, Size, TyAbiInterface,
-    TyAndLayout,
+    Align, BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
+    TyAndLayout, Variants,
 };
 
 use crate::callconv::{ArgAbi, ArgAttribute, CastTarget, FnAbi, Uniform};
-use crate::spec::{Env, HasTargetSpec, Os};
+use crate::spec::{HasTargetSpec, Os};
 
-#[derive(Clone, Debug)]
-struct Sdata {
-    pub prefix: [Option; 8],
-    pub prefix_index: usize,
-    pub last_offset: Size,
-    pub has_float: bool,
-    pub arg_attribute: ArgAttribute,
+// NOTE: GCC and Clang/LLVM have disagreements that the ABI doesn't resolve, we match the
+// Clang/LLVM behavior in these cases.
+
+#[derive(Copy, Clone)]
+enum DoubleWord {
+    F64,
+    F128Start,
+    F128End,
+    Words([Word; 2]),
 }
 
-fn arg_scalar(cx: &C, scalar: &Scalar, offset: Size, mut data: Sdata) -> Sdata
-where
-    C: HasDataLayout,
-{
-    let dl = cx.data_layout();
-
-    if !matches!(scalar.primitive(), Primitive::Float(Float::F32 | Float::F64)) {
-        return data;
-    }
-
-    data.has_float = true;
-
-    if !data.last_offset.is_aligned(dl.f64_align) && data.last_offset < offset {
-        if data.prefix_index == data.prefix.len() {
-            return data;
-        }
-        data.prefix[data.prefix_index] = Some(Reg::i32());
-        data.prefix_index += 1;
-        data.last_offset = data.last_offset + Reg::i32().size;
-    }
-
-    for _ in 0..((offset - data.last_offset).bits() / 64)
-        .min((data.prefix.len() - data.prefix_index) as u64)
-    {
-        data.prefix[data.prefix_index] = Some(Reg::i64());
-        data.prefix_index += 1;
-        data.last_offset = data.last_offset + Reg::i64().size;
-    }
-
-    if data.last_offset < offset {
-        if data.prefix_index == data.prefix.len() {
-            return data;
-        }
-        data.prefix[data.prefix_index] = Some(Reg::i32());
-        data.prefix_index += 1;
-        data.last_offset = data.last_offset + Reg::i32().size;
-    }
-
-    if data.prefix_index == data.prefix.len() {
-        return data;
-    }
-
-    if scalar.primitive() == Primitive::Float(Float::F32) {
-        data.arg_attribute = ArgAttribute::InReg;
-        data.prefix[data.prefix_index] = Some(Reg::f32());
-        data.last_offset = offset + Reg::f32().size;
-    } else {
-        data.prefix[data.prefix_index] = Some(Reg::f64());
-        data.last_offset = offset + Reg::f64().size;
-    }
-    data.prefix_index += 1;
-    data
+#[derive(Copy, Clone)]
+enum Word {
+    F32,
+    Integer,
 }
 
-fn arg_scalar_pair(
+fn classify<'a, Ty, C>(
     cx: &C,
-    scalar1: &Scalar,
-    scalar2: &Scalar,
-    mut offset: Size,
-    mut data: Sdata,
-) -> Sdata
-where
-    C: HasDataLayout,
-{
-    data = arg_scalar(cx, scalar1, offset, data);
-    match (scalar1.primitive(), scalar2.primitive()) {
-        (Primitive::Float(Float::F32), _) => offset += Reg::f32().size,
-        (_, Primitive::Float(Float::F64)) => offset += Reg::f64().size,
-        (Primitive::Int(i, _signed), _) => offset += i.size(),
-        (Primitive::Pointer(_), _) => offset += Reg::i64().size,
-        _ => {}
-    }
-
-    if !offset.bytes().is_multiple_of(4)
-        && matches!(scalar2.primitive(), Primitive::Float(Float::F32 | Float::F64))
-    {
-        offset += Size::from_bytes(4 - (offset.bytes() % 4));
-    }
-    data = arg_scalar(cx, scalar2, offset, data);
-    data
-}
-
-fn parse_structure<'a, Ty, C>(
-    cx: &C,
-    layout: TyAndLayout<'a, Ty>,
-    mut data: Sdata,
-    mut offset: Size,
-) -> Sdata
-where
+    arg_layout: &TyAndLayout<'a, Ty>,
+    offset: Size,
+    double_words: &mut [DoubleWord; 4],
+) where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if let FieldsShape::Union(_) = layout.fields {
-        return data;
-    }
+    // If this function does not update the `double_words` array, the value will be passed via
+    // integer registers. The array is initialized with `DoubleWord::Words([Word::Integer; 2])`.
 
-    match layout.backend_repr {
-        BackendRepr::Scalar(scalar) => {
-            data = arg_scalar(cx, &scalar, offset, data);
-        }
-        BackendRepr::Memory { .. } => {
-            for i in 0..layout.fields.count() {
-                if offset < layout.fields.offset(i) {
-                    offset = layout.fields.offset(i);
+    match arg_layout.backend_repr {
+        BackendRepr::Scalar(scalar) => match scalar.primitive() {
+            Primitive::Float(float) => {
+                if offset.is_aligned(Ord::min(*float.align(cx), Align::EIGHT)) {
+                    let index = offset.bytes_usize() / 8;
+                    match float {
+                        Float::F128 => {
+                            double_words[index] = DoubleWord::F128Start;
+                            double_words[index + 1] = DoubleWord::F128End;
+                        }
+                        Float::F64 => {
+                            double_words[index] = DoubleWord::F64;
+                        }
+                        Float::F32 => match &mut double_words[index] {
+                            DoubleWord::Words(words) => {
+                                words[(offset.bytes_usize() % 8) / 4] = Word::F32;
+                            }
+                            _ => unreachable!(),
+                        },
+                        Float::F16 => {
+                            // Match LLVM by passing `f16` in integer registers.
+                        }
+                    }
+                } else {
+                    /* pass unaligned floats in integer registers */
                 }
-                data = parse_structure(cx, layout.field(cx, i), data.clone(), offset);
             }
-        }
-        _ => {
-            if let BackendRepr::ScalarPair(scalar1, scalar2) = &layout.backend_repr {
-                data = arg_scalar_pair(cx, scalar1, scalar2, offset, data);
+            Primitive::Int(_, _) | Primitive::Pointer(_) => { /* pass in integer registers */ }
+        },
+        BackendRepr::SimdVector { .. } => {}
+        BackendRepr::ScalableVector { .. } => {}
+        BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => match arg_layout.fields {
+            FieldsShape::Primitive => {
+                unreachable!("aggregates can't have `FieldsShape::Primitive`")
             }
-        }
+            FieldsShape::Union(_) => {
+                if !arg_layout.is_zst() {
+                    if arg_layout.is_transparent() {
+                        let non_1zst_elem = arg_layout.non_1zst_field(cx).expect("not exactly one non-1-ZST field in non-ZST repr(transparent) union").1;
+                        classify(cx, &non_1zst_elem, offset, double_words);
+                    }
+                }
+            }
+            FieldsShape::Array { .. } => {}
+            FieldsShape::Arbitrary { .. } => match arg_layout.variants {
+                Variants::Multiple { .. } => {}
+                Variants::Single { .. } | Variants::Empty => {
+                    // Match Clang by ignoring whether a struct is packed and just considering
+                    // whether individual fields are aligned. GCC currently uses only integer
+                    // registers when passing packed structs.
+                    for i in arg_layout.fields.index_by_increasing_offset() {
+                        classify(
+                            cx,
+                            &arg_layout.field(cx, i),
+                            offset + arg_layout.fields.offset(i),
+                            double_words,
+                        );
+                    }
+                }
+            },
+        },
     }
-
-    data
 }
 
-fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, in_registers_max: Size)
-where
+fn classify_arg<'a, Ty, C>(
+    cx: &C,
+    arg: &mut ArgAbi<'a, Ty>,
+    in_registers_max: Size,
+    total_double_word_count: &mut usize,
+) where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    // 64-bit SPARC allocates argument stack space in 64-bit chunks (double words), some of which
+    // are promoted to registers based on their position on the stack.
+
+    // Keep track of the total number of double words used by arguments so far. This allows padding
+    // arguments to be inserted where necessary to ensure that 16-aligned arguments are passed in an
+    // aligned set of registers.
+
+    let pad = !total_double_word_count.is_multiple_of(2) && arg.layout.align.abi.bytes() == 16;
+    // The number of double words used by this argument.
+    let double_word_count = arg.layout.size.bytes_usize().div_ceil(8);
+    // The number of double words before this argument, including any padding.
+    let start_double_word_count = *total_double_word_count + usize::from(pad);
+
     if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
         arg.make_indirect();
+        *total_double_word_count += 1;
         return;
     }
+
     if !arg.layout.is_aggregate() {
         arg.extend_integer_width_to(64);
+        *total_double_word_count = start_double_word_count + double_word_count;
         return;
     }
 
     let total = arg.layout.size;
     if total > in_registers_max {
         arg.make_indirect();
+        *total_double_word_count += 1;
         return;
     }
 
-    match arg.layout.fields {
-        FieldsShape::Primitive => unreachable!(),
-        FieldsShape::Array { .. } => {
-            // Arrays are passed indirectly
-            arg.make_indirect();
-            return;
+    *total_double_word_count = start_double_word_count + double_word_count;
+
+    const ARGUMENT_REGISTERS: usize = 8;
+
+    let mut double_words = [DoubleWord::Words([Word::Integer; 2]); ARGUMENT_REGISTERS / 2];
+    classify(cx, &arg.layout, Size::ZERO, &mut double_words);
+
+    let mut regs = [None; ARGUMENT_REGISTERS];
+    let mut i = 0;
+    let mut push = |reg| {
+        regs[i] = Some(reg);
+        i += 1;
+    };
+    let mut attrs = ArgAttribute::empty();
+
+    for (index, double_word) in double_words.into_iter().enumerate() {
+        if arg.layout.size.bytes_usize() <= index * 8 {
+            break;
         }
-        FieldsShape::Union(_) => {
-            // Unions and are always treated as a series of 64-bit integer chunks
-        }
-        FieldsShape::Arbitrary { .. } => {
-            // Structures with floating point numbers need special care.
-
-            let mut data = parse_structure(
-                cx,
-                arg.layout,
-                Sdata {
-                    prefix: [None; 8],
-                    prefix_index: 0,
-                    last_offset: Size::ZERO,
-                    has_float: false,
-                    arg_attribute: ArgAttribute::default(),
-                },
-                Size::ZERO,
-            );
-
-            if data.has_float {
-                // Structure { float, int, int } doesn't like to be handled like
-                // { float, long int }. Other way around it doesn't mind.
-                if data.last_offset < arg.layout.size
-                    && !data.last_offset.bytes().is_multiple_of(8)
-                    && data.prefix_index < data.prefix.len()
-                {
-                    data.prefix[data.prefix_index] = Some(Reg::i32());
-                    data.prefix_index += 1;
-                    data.last_offset += Reg::i32().size;
+        match double_word {
+            // `f128` must be aligned to be assigned a float register.
+            DoubleWord::F128Start if (start_double_word_count + index).is_multiple_of(2) => {
+                push(Reg::f128());
+            }
+            DoubleWord::F128Start => {
+                // Clang currently handles this case nonsensically, always returning a packed
+                // `struct { long double x; }` in an aligned quad floating-point register even when
+                // the `long double` isn't aligned on the stack, which also makes all future
+                // arguments get passed in the wrong registers. This passes the `f128` in integer
+                // registers when it is unaligned, same as with `f32` and `f64`.
+                push(Reg::i64());
+                push(Reg::i64());
+            }
+            DoubleWord::F128End => {} // Already handled by `F128Start`
+            DoubleWord::F64 => push(Reg::f64()),
+            DoubleWord::Words([Word::Integer, Word::Integer]) => push(Reg::i64()),
+            DoubleWord::Words(words) => {
+                attrs |= ArgAttribute::InReg;
+                for word in words {
+                    match word {
+                        Word::F32 => push(Reg::f32()),
+                        Word::Integer => push(Reg::i32()),
+                    }
                 }
-
-                let mut rest_size = arg.layout.size - data.last_offset;
-                if !rest_size.bytes().is_multiple_of(8) && data.prefix_index < data.prefix.len() {
-                    data.prefix[data.prefix_index] = Some(Reg::i32());
-                    rest_size = rest_size - Reg::i32().size;
-                }
-
-                arg.cast_to(
-                    CastTarget::prefixed(data.prefix, Uniform::new(Reg::i64(), rest_size))
-                        .with_attrs(data.arg_attribute.into()),
-                );
-                return;
             }
         }
     }
 
-    arg.cast_to(Uniform::new(Reg::i64(), total));
+    let cast_target = match regs {
+        [Some(reg), None, rest @ ..] => {
+            // Just a single register is needed for this value.
+            debug_assert!(rest.iter().all(|x| x.is_none()));
+            CastTarget::from(reg)
+        }
+        _ => CastTarget::prefixed(regs, Uniform::new(Reg::i8(), Size::ZERO)),
+    };
+
+    arg.cast_to_and_pad_i32(cast_target.with_attrs(attrs.into()), pad);
 }
 
 pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
@@ -217,23 +206,26 @@ where
     C: HasDataLayout + HasTargetSpec,
 {
     if !fn_abi.ret.is_ignore() && fn_abi.ret.layout.is_sized() {
-        classify_arg(cx, &mut fn_abi.ret, Size::from_bytes(32));
+        // A return value of 32 bytes or smaller is passed via registers.
+        classify_arg(cx, &mut fn_abi.ret, Size::from_bytes(32), &mut 0);
     }
 
+    // sparc64-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs.
+    let passes_zsts = matches!(cx.target_spec().os, Os::Linux);
+
+    let mut double_word_count = 0;
     for arg in fn_abi.args.iter_mut() {
         if !arg.layout.is_sized() {
             continue;
         }
         if arg.is_ignore() {
-            // sparc64-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs.
-            if cx.target_spec().os == Os::Linux
-                && matches!(cx.target_spec().env, Env::Gnu | Env::Musl | Env::Uclibc)
-                && arg.layout.is_zst()
-            {
+            if passes_zsts && arg.layout.is_zst() {
                 arg.make_indirect_from_ignore();
+                double_word_count += 1;
             }
-            return;
+            continue;
         }
-        classify_arg(cx, arg, Size::from_bytes(16));
+        // An argument of 16 bytes or smaller is passed via registers.
+        classify_arg(cx, arg, Size::from_bytes(16), &mut double_word_count);
     }
 }
diff --git a/tests/assembly-llvm/sparc-struct-abi.rs b/tests/assembly-llvm/sparc-struct-abi.rs
index c6e83b6f8dab..41ac2be09b65 100644
--- a/tests/assembly-llvm/sparc-struct-abi.rs
+++ b/tests/assembly-llvm/sparc-struct-abi.rs
@@ -9,6 +9,7 @@
 #![crate_type = "lib"]
 #![feature(no_core, lang_items)]
 #![no_core]
+#![feature(f128)]
 
 extern crate minicore;
 use minicore::*;
@@ -21,8 +22,33 @@ pub struct Franta {
     d: f32,
 }
 
+#[repr(C, packed)]
+struct Misaligned(i32, f64);
+
+#[repr(C)]
+struct AlignToMakeAssemblyShorter(T, f64);
+
+#[repr(C)]
+pub struct Floats(i32, f32, f64, f128);
+
+#[repr(C)]
+pub struct LessFloats(f32, i32, f64);
+
+#[repr(C)]
+pub struct NotMisaligned(i32, Misaligned);
+
+#[repr(C, align(16))]
+pub struct Align16(f64, i32, i32);
+
+impl Copy for Misaligned {}
+impl Copy for AlignToMakeAssemblyShorter {}
+impl Copy for Floats {}
+impl Copy for LessFloats {}
+impl Copy for NotMisaligned {}
+impl Copy for Align16 {}
+
 // NB: due to delay slots the `ld` following the call is actually executed before the call.
-#[no_mangle]
+#[unsafe(no_mangle)]
 pub unsafe extern "C" fn callee(arg: Franta) {
     // CHECK-LABEL: callee:
     // CHECK: st %f3, [[PLACE_D:.*]]
@@ -54,7 +80,7 @@ extern "C" {
     fn tail_call_avoidance_fn();
 }
 
-#[no_mangle]
+#[unsafe(no_mangle)]
 pub unsafe extern "C" fn caller() {
     // CHECK-LABEL: caller:
     // CHECK: ld [{{.*}}], %f0
@@ -62,7 +88,168 @@ pub unsafe extern "C" fn caller() {
     // CHECK: ld [{{.*}}], %f2
     // CHECK: ld [{{.*}}], %f3
     // CHECK: call opaque_callee
-    // CHECK: mov     3, %o2
+    // CHECK: mov 3, %o2
     opaque_callee(Franta { a: 1.0, b: 2.0, c: 3.0, d: 4.0 }, 3);
     tail_call_avoidance_fn();
 }
+
+// Check that misaligned floats aren't promoted to floating point registers.
+// CHECK-LABEL: misaligned_arg:
+#[unsafe(no_mangle)]
+extern "C" fn misaligned_arg(x: &mut AlignToMakeAssemblyShorter, value: Misaligned) {
+    // CHECK: srlx %o2, 32, %o2
+    // CHECK-NEXT: stx %o1, [%o0]
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: st %o2, [%o0+8]
+    x.0 = value;
+}
+
+// CHECK-LABEL: misaligned_ret:
+#[unsafe(no_mangle)]
+extern "C" fn misaligned_ret(x: &AlignToMakeAssemblyShorter) -> Misaligned {
+    // CHECK: ld [%o0+8], %o1
+    // CHECK-NEXT: ldx [%o0], %o0
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: sllx %o1, 32, %o1
+    x.0
+}
+
+// Check structs where 32 >= size > 16 are promoted to register only as an argument.
+// Also check that the various floating-point types are promoted to the correct registers.
+// CHECK-LABEL: floats_arg:
+#[unsafe(no_mangle)]
+extern "C" fn floats_arg(x: &mut Floats, value: Floats) {
+    // CHECK: ldx [%o1+24], %o2
+    // CHECK-NEXT: ldx [%o1+16], %o3
+    // CHECK-NEXT: ldx [%o1+8], %o4
+    // CHECK-NEXT: ldx [%o1], %o1
+    // CHECK-NEXT: stx %o2, [%o0+24]
+    // CHECK-NEXT: stx %o3, [%o0+16]
+    // CHECK-NEXT: stx %o4, [%o0+8]
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: stx %o1, [%o0]
+    *x = value;
+}
+
+// CHECK-LABEL: floats_ret:
+#[unsafe(no_mangle)]
+extern "C" fn floats_ret(x: &Floats) -> Floats {
+    // CHECK: ld [%o0+4], %f1
+    // CHECK-NEXT: ldd [%o0+8], %f2
+    // CHECK-NEXT: ldd [%o0+16], %f4
+    // CHECK-NEXT: ld [%o0], %o1
+    // CHECK-NEXT: ldd [%o0+24], %f6
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: sllx %o1, 32, %o0
+    *x
+}
+
+// Check float promotion when passing as an argument with a struct where size <= 16.
+// CHECK-LABEL: less_floats_arg:
+#[unsafe(no_mangle)]
+extern "C" fn less_floats_arg(x: &mut LessFloats, value: LessFloats) {
+    // CHECK: st %f2, [%o0]
+    // CHECK-NEXT: st %o1, [%o0+4]
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: std %f4, [%o0+8]
+    *x = value;
+}
+
+// CHECK-LABEL: less_floats_ret:
+#[unsafe(no_mangle)]
+extern "C" fn less_floats_ret(x: &LessFloats) -> LessFloats {
+    // CHECK: ld [%o0], %f0
+    // CHECK-NEXT: ldd [%o0+8], %f2
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: ld [%o0+4], %o0
+    *x
+}
+
+// Check fields are promoted if they are aligned in the overall structure.
+// This matches Clang's behaviour but not GCC's.
+// CHECK-LABEL: not_misaligned_arg:
+#[unsafe(no_mangle)]
+extern "C" fn not_misaligned_arg(
+    x: &mut AlignToMakeAssemblyShorter,
+    value: NotMisaligned,
+) {
+    // CHECK: stx %o1, [%o0]
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: std %f4, [%o0+8]
+    x.0 = value;
+}
+
+// CHECK-LABEL: not_misaligned_ret:
+#[unsafe(no_mangle)]
+extern "C" fn not_misaligned_ret(x: &AlignToMakeAssemblyShorter) -> NotMisaligned {
+    // CHECK: ldx [%o0], %o1
+    // CHECK-NEXT: ldd [%o0+8], %f2
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: mov %o1, %o0
+    x.0
+}
+
+// Check that 16-aligned structs are allocated the correct registers.
+// CHECK-LABEL: align_16_arg:
+#[unsafe(no_mangle)]
+extern "C" fn align_16_arg(x: &mut Align16, value: Align16) {
+    // CHECK: std %f4, [%o0]
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: stx %o3, [%o0+8]
+    *x = value;
+}
+
+// CHECK-LABEL: align_16_ret:
+#[unsafe(no_mangle)]
+extern "C" fn align_16_ret(x: &Align16) -> Align16 {
+    // CHECK: ldd [%o0], %f0
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: ldx [%o0+8], %o1
+    *x
+}
+
+// Check ZST args don't prevent further arguments from being processed.
+// CHECK-LABEL: zst_arg:
+#[unsafe(no_mangle)]
+extern "C" fn zst_arg(_: (), value: LessFloats, x: &mut LessFloats) {
+    // CHECK: st %f0, [%o2]
+    // CHECK-NEXT: st %o0, [%o2+4]
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: std %f2, [%o2+8]
+    *x = value;
+}
+
+#[repr(C)]
+struct I32F32Input {
+    a: i32,
+    b: f32,
+}
+
+#[repr(C)]
+struct I32F32Output {
+    b: f32,
+    a: i32,
+}
+
+// The clang/LLVM implementation mentions that this case requires special handling.
+// CHECK-LABEL: i32_f32:
+#[unsafe(no_mangle)]
+extern "C" fn i32_f32(input: I32F32Input) -> I32F32Output {
+    // CHECK: srlx %o0, 32, %o0
+    // CHECK-NEXT: fmovs %f1, %f0
+    // CHECK-NEXT: retl
+    // CHECK-NEXT: nop
+    I32F32Output { a: input.a, b: input.b }
+}
+
+#[repr(C)]
+pub struct C {
+    a: f64,
+    b: f32,
+}
+
+// regression test for https://github.com/rust-lang/rust/issues/147883.
+#[unsafe(no_mangle)]
+pub extern "C" fn foo(c: C) -> C {
+    c
+}
diff --git a/tests/codegen-llvm/cast-target-abi.rs b/tests/codegen-llvm/cast-target-abi.rs
index 101e73e33c91..6f1ab4572ee0 100644
--- a/tests/codegen-llvm/cast-target-abi.rs
+++ b/tests/codegen-llvm/cast-target-abi.rs
@@ -119,7 +119,7 @@ pub extern "C" fn returns_twou16s() -> TwoU16s {
 // aarch64-SAME:     ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
 // loongarch64-SAME: ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
 // powerpc64-SAME:   ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
-// sparc64-SAME:     ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
+// sparc64-SAME:     ([[ABI_TYPE:{ i64, i64 }]] {{.*}}[[ABI_VALUE:%.+]])
 // x86_64-SAME:      ([[ABI_TYPE:{ i64, i16 }]] {{.*}}[[ABI_VALUE:%.+]])
 #[no_mangle]
 #[inline(never)]
@@ -148,7 +148,7 @@ pub extern "C" fn returns_fiveu16s() -> FiveU16s {
 
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i64 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i16 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // aarch64:     ret [[ABI_TYPE]] [[ABI_VALUE]]
@@ -217,7 +217,7 @@ pub extern "C" fn returns_doubledouble() -> DoubleDouble {
 // aarch64-SAME:     ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
 // loongarch64-SAME: ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
 // powerpc64-SAME:   ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
-// sparc64-SAME:     ([[ABI_TYPE:\[2 x i64\]]] {{.*}}[[ABI_VALUE:%.+]])
+// sparc64-SAME:     ([[ABI_TYPE:{ i64, i64 }]] {{.*}}[[ABI_VALUE:%.+]])
 // x86_64-SAME:      ([[ABI_TYPE:{ i64, i32 }]] {{.*}}[[ABI_VALUE:%.+]])
 #[no_mangle]
 #[inline(never)]
@@ -246,7 +246,7 @@ pub extern "C" fn returns_three32s() -> Three32s {
 
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i64 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i32 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // aarch64:     ret [[ABI_TYPE]] [[ABI_VALUE]]
@@ -399,7 +399,7 @@ pub fn call_fiveu16s() {
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // powerpc64:   [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i64 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i16 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // CHECK: call void @receives_fiveu16s([[ABI_TYPE]] [[ABI_VALUE]])
@@ -424,7 +424,7 @@ pub fn return_fiveu16s() -> FiveU16s {
 
     // aarch64:     [[ABI_VALUE:%.+]] = call [[ABI_TYPE:\[2 x i64\]]] @returns_fiveu16s()
     // loongarch64: [[ABI_VALUE:%.+]] = call [[ABI_TYPE:\[2 x i64\]]] @returns_fiveu16s()
-    // sparc64:     [[ABI_VALUE:%.+]] = call [[ABI_TYPE:\[2 x i64\]]] @returns_fiveu16s()
+    // sparc64:     [[ABI_VALUE:%.+]] = call [[ABI_TYPE:{ i64, i64 }]] @returns_fiveu16s()
     // x86_64:      [[ABI_VALUE:%.+]] = call [[ABI_TYPE:{ i64, i16 }]] @returns_fiveu16s()
 
     // aarch64:     store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
@@ -595,7 +595,7 @@ pub fn call_three32s() {
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // powerpc64:   [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i64 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ i64, i32 }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // CHECK: call void @receives_three32s([[ABI_TYPE]] [[ABI_VALUE]])
@@ -619,7 +619,7 @@ pub fn return_three32s() -> Three32s {
 
     // aarch64:     [[ABI_VALUE:%.+]] = call [[ABI_TYPE:\[2 x i64\]]] @returns_three32s()
     // loongarch64: [[ABI_VALUE:%.+]] = call [[ABI_TYPE:\[2 x i64\]]] @returns_three32s()
-    // sparc64:     [[ABI_VALUE:%.+]] = call [[ABI_TYPE:\[2 x i64\]]] @returns_three32s()
+    // sparc64:     [[ABI_VALUE:%.+]] = call [[ABI_TYPE:{ i64, i64 }]] @returns_three32s()
     // x86_64:      [[ABI_VALUE:%.+]] = call [[ABI_TYPE:{ i64, i32 }]] @returns_three32s()
 
     // aarch64:     store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]

From b23d30885308fadf851af7fc3134d129731b6417 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Wed, 14 Jan 2026 22:19:12 +0100
Subject: [PATCH 1552/1843] Support structs in type info reflection

---
 .../src/const_eval/type_info.rs               | 177 ++++++++++++++----
 compiler/rustc_span/src/symbol.rs             |   2 +
 library/core/src/mem/type_info.rs             |  15 ++
 library/coretests/tests/mem/type_info.rs      |  51 +++++
 .../missing-trait-bounds/issue-69725.stderr   |   8 +-
 tests/ui/reflection/dump.bit32.run.stdout     |  55 +++++-
 tests/ui/reflection/dump.bit64.run.stdout     |  55 +++++-
 tests/ui/reflection/dump.rs                   |   9 +-
 8 files changed, 333 insertions(+), 39 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index f8881f0968bb..5b96856b253f 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -1,18 +1,36 @@
-use rustc_abi::FieldIdx;
+use std::borrow::Cow;
+
+use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_ast::Mutability;
 use rustc_hir::LangItem;
 use rustc_middle::span_bug;
 use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{self, Const, ScalarInt, Ty};
+use rustc_middle::ty::{self, AdtDef, AdtKind, Const, GenericArgs, ScalarInt, Ty, VariantDef};
 use rustc_span::{Symbol, sym};
 
 use crate::const_eval::CompileTimeMachine;
 use crate::interpret::{
-    CtfeProvenance, Immediate, InterpCx, InterpResult, MPlaceTy, MemoryKind, Scalar, Writeable,
-    interp_ok,
+    CtfeProvenance, Immediate, InterpCx, InterpResult, MPlaceTy, MemoryKind, Projectable, Scalar,
+    Writeable, interp_ok,
 };
 
 impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
+    /// Equivalent to `project_downcast`, but identifies the variant by name instead of index.
+    fn downcast<'a>(
+        &self,
+        place: &(impl Writeable<'tcx, CtfeProvenance> + 'a),
+        name: Symbol,
+    ) -> InterpResult<'tcx, (VariantIdx, impl Writeable<'tcx, CtfeProvenance> + 'a)> {
+        let variants = place.layout().ty.ty_adt_def().unwrap().variants();
+        let variant_idx = variants
+            .iter_enumerated()
+            .find(|(_idx, var)| var.name == name)
+            .unwrap_or_else(|| panic!("got {name} but expected one of {variants:#?}"))
+            .0;
+
+        interp_ok((variant_idx, self.project_downcast(place, variant_idx)?))
+    }
+
     /// Writes a `core::mem::type_info::TypeInfo` for a given type, `ty` to the given place.
     pub(crate) fn write_type_info(
         &mut self,
@@ -26,22 +44,13 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         // Fill all fields of the `TypeInfo` struct.
         for (idx, field) in ty_struct.fields.iter_enumerated() {
             let field_dest = self.project_field(dest, idx)?;
-            let downcast = |name: Symbol| {
-                let variants = field_dest.layout().ty.ty_adt_def().unwrap().variants();
-                let variant_id = variants
-                    .iter_enumerated()
-                    .find(|(_idx, var)| var.name == name)
-                    .unwrap_or_else(|| panic!("got {name} but expected one of {variants:#?}"))
-                    .0;
-
-                interp_ok((variant_id, self.project_downcast(&field_dest, variant_id)?))
-            };
             let ptr_bit_width = || self.tcx.data_layout.pointer_size().bits();
             match field.name {
                 sym::kind => {
                     let variant_index = match ty.kind() {
                         ty::Tuple(fields) => {
-                            let (variant, variant_place) = downcast(sym::Tuple)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::Tuple)?;
                             // project to the single tuple variant field of `type_info::Tuple` struct type
                             let tuple_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
                             assert_eq!(
@@ -59,7 +68,8 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Array(ty, len) => {
-                            let (variant, variant_place) = downcast(sym::Array)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::Array)?;
                             let array_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
 
                             self.write_array_type_info(array_place, *ty, *len)?;
@@ -67,23 +77,38 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Slice(ty) => {
-                            let (variant, variant_place) = downcast(sym::Slice)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::Slice)?;
                             let slice_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
 
                             self.write_slice_type_info(slice_place, *ty)?;
 
                             variant
                         }
+                        ty::Adt(adt_def, generics) => {
+                            // TODO(type_info): Handle enum and union
+                            if !adt_def.is_struct() {
+                                self.downcast(&field_dest, sym::Other)?.0
+                            } else {
+                                let (variant, variant_place) =
+                                    self.downcast(&field_dest, sym::Struct)?;
+                                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                                self.write_adt_type_info(place, (ty, *adt_def), generics)?;
+                                variant
+                            }
+                        }
                         ty::Bool => {
-                            let (variant, _variant_place) = downcast(sym::Bool)?;
+                            let (variant, _variant_place) =
+                                self.downcast(&field_dest, sym::Bool)?;
                             variant
                         }
                         ty::Char => {
-                            let (variant, _variant_place) = downcast(sym::Char)?;
+                            let (variant, _variant_place) =
+                                self.downcast(&field_dest, sym::Char)?;
                             variant
                         }
                         ty::Int(int_ty) => {
-                            let (variant, variant_place) = downcast(sym::Int)?;
+                            let (variant, variant_place) = self.downcast(&field_dest, sym::Int)?;
                             let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
                             self.write_int_type_info(
                                 place,
@@ -93,7 +118,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Uint(uint_ty) => {
-                            let (variant, variant_place) = downcast(sym::Int)?;
+                            let (variant, variant_place) = self.downcast(&field_dest, sym::Int)?;
                             let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
                             self.write_int_type_info(
                                 place,
@@ -103,17 +128,19 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Float(float_ty) => {
-                            let (variant, variant_place) = downcast(sym::Float)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::Float)?;
                             let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
                             self.write_float_type_info(place, float_ty.bit_width())?;
                             variant
                         }
                         ty::Str => {
-                            let (variant, _variant_place) = downcast(sym::Str)?;
+                            let (variant, _variant_place) = self.downcast(&field_dest, sym::Str)?;
                             variant
                         }
                         ty::Ref(_, ty, mutability) => {
-                            let (variant, variant_place) = downcast(sym::Reference)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::Reference)?;
                             let reference_place =
                                 self.project_field(&variant_place, FieldIdx::ZERO)?;
                             self.write_reference_type_info(reference_place, *ty, *mutability)?;
@@ -121,7 +148,8 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::RawPtr(ty, mutability) => {
-                            let (variant, variant_place) = downcast(sym::Pointer)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::Pointer)?;
                             let pointer_place =
                                 self.project_field(&variant_place, FieldIdx::ZERO)?;
 
@@ -130,13 +158,13 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Dynamic(predicates, region) => {
-                            let (variant, variant_place) = downcast(sym::DynTrait)?;
+                            let (variant, variant_place) =
+                                self.downcast(&field_dest, sym::DynTrait)?;
                             let dyn_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
                             self.write_dyn_trait_type_info(dyn_place, *predicates, *region)?;
                             variant
                         }
-                        ty::Adt(_, _)
-                        | ty::Foreign(_)
+                        ty::Foreign(_)
                         | ty::Pat(_, _)
                         | ty::FnDef(..)
                         | ty::FnPtr(..)
@@ -151,14 +179,14 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                         | ty::Bound(..)
                         | ty::Placeholder(_)
                         | ty::Infer(..)
-                        | ty::Error(_) => downcast(sym::Other)?.0,
+                        | ty::Error(_) => self.downcast(&field_dest, sym::Other)?.0,
                     };
                     self.write_discriminant(variant_index, &field_dest)?
                 }
                 sym::size => {
                     let layout = self.layout_of(ty)?;
                     let variant_index = if layout.is_sized() {
-                        let (variant, variant_place) = downcast(sym::Some)?;
+                        let (variant, variant_place) = self.downcast(&field_dest, sym::Some)?;
                         let size_field_place =
                             self.project_field(&variant_place, FieldIdx::ZERO)?;
                         self.write_scalar(
@@ -168,7 +196,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                         )?;
                         variant
                     } else {
-                        downcast(sym::None)?.0
+                        self.downcast(&field_dest, sym::None)?.0
                     };
                     self.write_discriminant(variant_index, &field_dest)?;
                 }
@@ -204,7 +232,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
 
         while let Some((i, place)) = fields_places.next(self)? {
             let field_ty = fields[i as usize];
-            self.write_field(field_ty, place, tuple_layout, i)?;
+            self.write_field(field_ty, place, tuple_layout, None, i)?;
         }
 
         let fields_place = fields_place.map_provenance(CtfeProvenance::as_immutable);
@@ -219,6 +247,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         field_ty: Ty<'tcx>,
         place: MPlaceTy<'tcx>,
         layout: TyAndLayout<'tcx>,
+        name: Option,
         idx: u64,
     ) -> InterpResult<'tcx> {
         for (field_idx, field_ty_field) in
@@ -226,6 +255,15 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         {
             let field_place = self.project_field(&place, field_idx)?;
             match field_ty_field.name {
+                sym::name => {
+                    let name = match name.as_ref() {
+                        Some(name) => Cow::Borrowed(name.as_str()),
+                        None => Cow::Owned(idx.to_string()), // For tuples
+                    };
+                    let name_place = self.allocate_str_dedup(&name)?;
+                    let ptr = self.mplace_to_ref(&name_place)?;
+                    self.write_immediate(*ptr, &field_place)?
+                }
                 sym::ty => self.write_type_id(field_ty, &field_place)?,
                 sym::offset => {
                     let offset = layout.fields.offset(idx as usize);
@@ -287,6 +325,81 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
+    // FIXME(type_info): No semver considerations for now
+    pub(crate) fn write_adt_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        adt: (Ty<'tcx>, AdtDef<'tcx>),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (adt_ty, adt_def) = adt;
+        match adt_def.adt_kind() {
+            AdtKind::Struct => self.write_struct_type_info(
+                place,
+                (adt_ty, adt_def.variant(VariantIdx::ZERO)),
+                generics,
+            ),
+            AdtKind::Union => todo!(),
+            AdtKind::Enum => todo!(),
+        }
+    }
+
+    pub(crate) fn write_struct_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        struct_: (Ty<'tcx>, &'tcx VariantDef),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (struct_ty, struct_def) = struct_;
+        let struct_layout = self.layout_of(struct_ty)?;
+
+        for (field_idx, field) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+
+            match field.name {
+                sym::fields => {
+                    let fields_slice_place = field_place;
+                    let field_type = fields_slice_place
+                        .layout()
+                        .ty
+                        .builtin_deref(false)
+                        .unwrap()
+                        .sequence_element_type(self.tcx.tcx);
+                    let fields_layout = self.layout_of(Ty::new_array(
+                        self.tcx.tcx,
+                        field_type,
+                        struct_def.fields.len() as u64,
+                    ))?;
+                    let fields_place = self.allocate(fields_layout, MemoryKind::Stack)?;
+                    let mut fields_places = self.project_array_fields(&fields_place)?;
+
+                    for field_def in &struct_def.fields {
+                        let (i, place) = fields_places.next(self)?.unwrap();
+                        let field_ty = field_def.ty(*self.tcx, generics);
+                        self.write_field(field_ty, place, struct_layout, Some(field_def.name), i)?;
+                    }
+
+                    let fields_place = fields_place.map_provenance(CtfeProvenance::as_immutable);
+                    let ptr = Immediate::new_slice(
+                        fields_place.ptr(),
+                        struct_def.fields.len() as u64,
+                        self,
+                    );
+                    self.write_immediate(ptr, &fields_slice_place)?
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = struct_def.is_field_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
+            }
+        }
+
+        interp_ok(())
+    }
+
     fn write_int_type_info(
         &mut self,
         place: impl Writeable<'tcx, CtfeProvenance>,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 6aa2eae556e2..271124a4c372 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -372,6 +372,7 @@ symbols! {
         Stdin,
         Str,
         String,
+        Struct,
         StructuralPartialEq,
         SubdiagMessage,
         Subdiagnostic,
@@ -1086,6 +1087,7 @@ symbols! {
         ffi_returns_twice,
         field,
         field_init_shorthand,
+        fields,
         file,
         file_options,
         flags,
diff --git a/library/core/src/mem/type_info.rs b/library/core/src/mem/type_info.rs
index 8b30803c97c9..0e87b9d9cbb7 100644
--- a/library/core/src/mem/type_info.rs
+++ b/library/core/src/mem/type_info.rs
@@ -49,6 +49,8 @@ pub enum TypeKind {
     Slice(Slice),
     /// Dynamic Traits.
     DynTrait(DynTrait),
+    /// Structs.
+    Struct(Struct),
     /// Primitive boolean type.
     Bool(Bool),
     /// Primitive character type.
@@ -81,6 +83,8 @@ pub struct Tuple {
 #[non_exhaustive]
 #[unstable(feature = "type_info", issue = "146922")]
 pub struct Field {
+    /// The name of the field.
+    pub name: &'static str,
     /// The field's type.
     pub ty: TypeId,
     /// Offset in bytes from the parent type
@@ -137,6 +141,17 @@ pub struct Trait {
     pub is_auto: bool,
 }
 
+/// Compile-time type information about arrays.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct Struct {
+    /// All fields of the struct.
+    pub fields: &'static [Field],
+    /// Whether the struct field list is non-exhaustive.
+    pub non_exhaustive: bool,
+}
+
 /// Compile-time type information about `bool`.
 #[derive(Debug)]
 #[non_exhaustive]
diff --git a/library/coretests/tests/mem/type_info.rs b/library/coretests/tests/mem/type_info.rs
index 87f2d5dd8289..03ff5f55c4f7 100644
--- a/library/coretests/tests/mem/type_info.rs
+++ b/library/coretests/tests/mem/type_info.rs
@@ -1,4 +1,7 @@
+#![allow(dead_code)]
+
 use std::any::{Any, TypeId};
+use std::mem::offset_of;
 use std::mem::type_info::{Type, TypeKind};
 
 #[test]
@@ -66,6 +69,54 @@ fn test_tuples() {
     }
 }
 
+#[test]
+fn test_structs() {
+    use TypeKind::*;
+
+    const {
+        struct TestStruct {
+            first: u8,
+            second: u16,
+            reference: &'static u16,
+        }
+
+        let Type { kind: Struct(ty), size, .. } = Type::of::() else { panic!() };
+        assert!(size == Some(size_of::()));
+        assert!(!ty.non_exhaustive);
+        assert!(ty.fields.len() == 3);
+        assert!(ty.fields[0].name == "first");
+        assert!(ty.fields[0].ty == TypeId::of::());
+        assert!(ty.fields[0].offset == offset_of!(TestStruct, first));
+        assert!(ty.fields[1].name == "second");
+        assert!(ty.fields[1].ty == TypeId::of::());
+        assert!(ty.fields[1].offset == offset_of!(TestStruct, second));
+        assert!(ty.fields[2].name == "reference");
+        assert!(ty.fields[2].ty != TypeId::of::<&'static u16>()); // FIXME(type_info): should be ==
+        assert!(ty.fields[2].offset == offset_of!(TestStruct, reference));
+    }
+
+    const {
+        #[non_exhaustive]
+        struct NonExhaustive {
+            a: u8,
+        }
+
+        let Type { kind: Struct(ty), .. } = Type::of::() else { panic!() };
+        assert!(ty.non_exhaustive);
+    }
+
+    const {
+        struct TupleStruct(u8, u16);
+
+        let Type { kind: Struct(ty), .. } = Type::of::() else { panic!() };
+        assert!(ty.fields.len() == 2);
+        assert!(ty.fields[0].name == "0");
+        assert!(ty.fields[0].ty == TypeId::of::());
+        assert!(ty.fields[1].name == "1");
+        assert!(ty.fields[1].ty == TypeId::of::());
+    }
+}
+
 #[test]
 fn test_primitives() {
     use TypeKind::*;
diff --git a/tests/ui/missing-trait-bounds/issue-69725.stderr b/tests/ui/missing-trait-bounds/issue-69725.stderr
index f483ea849b0e..20d2213f4f5f 100644
--- a/tests/ui/missing-trait-bounds/issue-69725.stderr
+++ b/tests/ui/missing-trait-bounds/issue-69725.stderr
@@ -1,17 +1,17 @@
-error[E0599]: the method `clone` exists for struct `Struct`, but its trait bounds were not satisfied
+error[E0599]: the method `clone` exists for struct `issue_69725::Struct`, but its trait bounds were not satisfied
   --> $DIR/issue-69725.rs:9:32
    |
 LL |     let _ = Struct::::new().clone();
-   |                                ^^^^^ method cannot be called on `Struct` due to unsatisfied trait bounds
+   |                                ^^^^^ method cannot be called on `issue_69725::Struct` due to unsatisfied trait bounds
    |
   ::: $DIR/auxiliary/issue-69725.rs:2:1
    |
 LL | pub struct Struct(A);
-   | -------------------- doesn't satisfy `Struct: Clone`
+   | -------------------- doesn't satisfy `issue_69725::Struct: Clone`
    |
    = note: the following trait bounds were not satisfied:
            `A: Clone`
-           which is required by `Struct: Clone`
+           which is required by `issue_69725::Struct: Clone`
 help: consider restricting the type parameter to satisfy the trait bound
    |
 LL | fn crash() where A: Clone {
diff --git a/tests/ui/reflection/dump.bit32.run.stdout b/tests/ui/reflection/dump.bit32.run.stdout
index d15b46509ff2..feda3401e462 100644
--- a/tests/ui/reflection/dump.bit32.run.stdout
+++ b/tests/ui/reflection/dump.bit32.run.stdout
@@ -3,14 +3,17 @@ Type {
         Tuple {
             fields: [
                 Field {
+                    name: "0",
                     ty: TypeId(0x0596b48cc04376e64d5c788c2aa46bdb),
                     offset: 0,
                 },
                 Field {
+                    name: "1",
                     ty: TypeId(0x0596b48cc04376e64d5c788c2aa46bdb),
                     offset: 1,
                 },
                 Field {
+                    name: "2",
                     ty: TypeId(0x41223169ff28813ba79b7268a2a968d9),
                     offset: 2,
                 },
@@ -143,7 +146,18 @@ Type {
     ),
 }
 Type {
-    kind: Other,
+    kind: Struct(
+        Struct {
+            fields: [
+                Field {
+                    name: "a",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
     size: Some(
         4,
     ),
@@ -154,6 +168,45 @@ Type {
         12,
     ),
 }
+Type {
+    kind: Struct(
+        Struct {
+            fields: [
+                Field {
+                    name: "a",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: true,
+        },
+    ),
+    size: Some(
+        4,
+    ),
+}
+Type {
+    kind: Struct(
+        Struct {
+            fields: [
+                Field {
+                    name: "0",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 8,
+                },
+                Field {
+                    name: "1",
+                    ty: TypeId(0x9ed91be891e304132cb86891e578f4a5),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
+    size: Some(
+        12,
+    ),
+}
 Type {
     kind: Reference(
         Reference {
diff --git a/tests/ui/reflection/dump.bit64.run.stdout b/tests/ui/reflection/dump.bit64.run.stdout
index efae22653951..d9fef71f619c 100644
--- a/tests/ui/reflection/dump.bit64.run.stdout
+++ b/tests/ui/reflection/dump.bit64.run.stdout
@@ -3,14 +3,17 @@ Type {
         Tuple {
             fields: [
                 Field {
+                    name: "0",
                     ty: TypeId(0x0596b48cc04376e64d5c788c2aa46bdb),
                     offset: 0,
                 },
                 Field {
+                    name: "1",
                     ty: TypeId(0x0596b48cc04376e64d5c788c2aa46bdb),
                     offset: 1,
                 },
                 Field {
+                    name: "2",
                     ty: TypeId(0x41223169ff28813ba79b7268a2a968d9),
                     offset: 2,
                 },
@@ -143,7 +146,18 @@ Type {
     ),
 }
 Type {
-    kind: Other,
+    kind: Struct(
+        Struct {
+            fields: [
+                Field {
+                    name: "a",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
     size: Some(
         4,
     ),
@@ -154,6 +168,45 @@ Type {
         24,
     ),
 }
+Type {
+    kind: Struct(
+        Struct {
+            fields: [
+                Field {
+                    name: "a",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: true,
+        },
+    ),
+    size: Some(
+        4,
+    ),
+}
+Type {
+    kind: Struct(
+        Struct {
+            fields: [
+                Field {
+                    name: "0",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 8,
+                },
+                Field {
+                    name: "1",
+                    ty: TypeId(0x9ed91be891e304132cb86891e578f4a5),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
+    size: Some(
+        16,
+    ),
+}
 Type {
     kind: Reference(
         Reference {
diff --git a/tests/ui/reflection/dump.rs b/tests/ui/reflection/dump.rs
index d42216a62fdc..fefaad325aea 100644
--- a/tests/ui/reflection/dump.rs
+++ b/tests/ui/reflection/dump.rs
@@ -14,6 +14,13 @@ struct Foo {
     a: u32,
 }
 
+#[non_exhaustive]
+struct NonExhaustiveStruct {
+    a: u32,
+}
+
+struct TupleStruct(u32, u64);
+
 enum Bar {
     Some(u32),
     None,
@@ -37,7 +44,7 @@ fn main() {
         [u8; 2],
         i8, i32, i64, i128, isize,
         u8, u32, u64, u128, usize,
-        Foo, Bar,
+        Foo, Bar, NonExhaustiveStruct, TupleStruct,
         &Unsized, &str, &[u8],
         str, [u8],
         &u8, &mut u8,

From 870fd9070b5d98422107b8c748f7c7d084d4d037 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Mon, 19 Jan 2026 01:07:51 +0100
Subject: [PATCH 1553/1843] Add generics info for structs in type info

---
 .../src/const_eval/type_info.rs               | 153 ++++++++++++++----
 compiler/rustc_span/src/symbol.rs             |   4 +
 library/core/src/mem/type_info.rs             |  41 +++++
 library/coretests/tests/mem/type_info.rs      |  20 ++-
 tests/ui/generics/wrong-number-of-args.rs     |   2 +-
 tests/ui/generics/wrong-number-of-args.stderr |   2 +-
 tests/ui/reflection/dump.bit32.run.stdout     |  50 ++++++
 tests/ui/reflection/dump.bit64.run.stdout     |  50 ++++++
 tests/ui/reflection/dump.rs                   |   8 +-
 .../resolve/resolve-assoc-suggestions.stderr  |  10 ++
 10 files changed, 304 insertions(+), 36 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 5b96856b253f..9fb01b1f5c5c 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -3,9 +3,12 @@ use std::borrow::Cow;
 use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_ast::Mutability;
 use rustc_hir::LangItem;
-use rustc_middle::span_bug;
 use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{self, AdtDef, AdtKind, Const, GenericArgs, ScalarInt, Ty, VariantDef};
+use rustc_middle::ty::{
+    self, AdtDef, AdtKind, Const, ConstKind, GenericArgKind, GenericArgs, Region, ScalarInt, Ty,
+    VariantDef,
+};
+use rustc_middle::{bug, span_bug};
 use rustc_span::{Symbol, sym};
 
 use crate::const_eval::CompileTimeMachine;
@@ -31,6 +34,37 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok((variant_idx, self.project_downcast(place, variant_idx)?))
     }
 
+    // A general method to write an array to a static slice place.
+    fn allocate_fill_and_write_slice_ptr(
+        &mut self,
+        slice_place: impl Writeable<'tcx, CtfeProvenance>,
+        len: u64,
+        writer: impl Fn(&mut Self, /* index */ u64, MPlaceTy<'tcx>) -> InterpResult<'tcx>,
+    ) -> InterpResult<'tcx> {
+        // Array element type
+        let field_ty = slice_place
+            .layout()
+            .ty
+            .builtin_deref(false)
+            .unwrap()
+            .sequence_element_type(self.tcx.tcx);
+
+        // Allocate an array
+        let array_layout = self.layout_of(Ty::new_array(self.tcx.tcx, field_ty, len))?;
+        let array_place = self.allocate(array_layout, MemoryKind::Stack)?;
+
+        // Fill the array fields
+        let mut field_places = self.project_array_fields(&array_place)?;
+        while let Some((i, place)) = field_places.next(self)? {
+            writer(self, i, place)?;
+        }
+
+        // Write the slice pointing to the array
+        let array_place = array_place.map_provenance(CtfeProvenance::as_immutable);
+        let ptr = Immediate::new_slice(array_place.ptr(), len, self);
+        self.write_immediate(ptr, &slice_place)
+    }
+
     /// Writes a `core::mem::type_info::TypeInfo` for a given type, `ty` to the given place.
     pub(crate) fn write_type_info(
         &mut self,
@@ -207,6 +241,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
+    // TODO(type_info): Remove this method, use `allocate_fill_and_write_slice_ptr` as it's more general.
     pub(crate) fn write_tuple_fields(
         &mut self,
         tuple_place: impl Writeable<'tcx, CtfeProvenance>,
@@ -359,36 +394,16 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
             let field_place = self.project_field(&place, field_idx)?;
 
             match field.name {
-                sym::fields => {
-                    let fields_slice_place = field_place;
-                    let field_type = fields_slice_place
-                        .layout()
-                        .ty
-                        .builtin_deref(false)
-                        .unwrap()
-                        .sequence_element_type(self.tcx.tcx);
-                    let fields_layout = self.layout_of(Ty::new_array(
-                        self.tcx.tcx,
-                        field_type,
-                        struct_def.fields.len() as u64,
-                    ))?;
-                    let fields_place = self.allocate(fields_layout, MemoryKind::Stack)?;
-                    let mut fields_places = self.project_array_fields(&fields_place)?;
-
-                    for field_def in &struct_def.fields {
-                        let (i, place) = fields_places.next(self)?.unwrap();
-                        let field_ty = field_def.ty(*self.tcx, generics);
-                        self.write_field(field_ty, place, struct_layout, Some(field_def.name), i)?;
-                    }
-
-                    let fields_place = fields_place.map_provenance(CtfeProvenance::as_immutable);
-                    let ptr = Immediate::new_slice(
-                        fields_place.ptr(),
-                        struct_def.fields.len() as u64,
-                        self,
-                    );
-                    self.write_immediate(ptr, &fields_slice_place)?
-                }
+                sym::generics => self.write_generics(field_place, generics)?,
+                sym::fields => self.allocate_fill_and_write_slice_ptr(
+                    field_place,
+                    struct_def.fields.len() as u64,
+                    |this, i, place| {
+                        let field_def = &struct_def.fields[FieldIdx::from_usize(i as usize)];
+                        let field_ty = field_def.ty(*this.tcx, generics);
+                        this.write_field(field_ty, place, struct_layout, Some(field_def.name), i)
+                    },
+                )?,
                 sym::non_exhaustive => {
                     let is_non_exhaustive = struct_def.is_field_list_non_exhaustive();
                     self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
@@ -400,6 +415,80 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
+    fn write_generics(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        self.allocate_fill_and_write_slice_ptr(place, generics.len() as u64, |this, i, place| {
+            match generics[i as usize].kind() {
+                GenericArgKind::Lifetime(region) => this.write_generic_lifetime(region, place),
+                GenericArgKind::Type(ty) => this.write_generic_type(ty, place),
+                GenericArgKind::Const(c) => this.write_generic_const(c, place),
+            }
+        })
+    }
+
+    fn write_generic_lifetime(
+        &mut self,
+        _region: Region<'tcx>,
+        place: MPlaceTy<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (variant_idx, _) = self.downcast(&place, sym::Lifetime)?;
+        self.write_discriminant(variant_idx, &place)?;
+        interp_ok(())
+    }
+
+    fn write_generic_type(&mut self, ty: Ty<'tcx>, place: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
+        let (variant_idx, variant_place) = self.downcast(&place, sym::Type)?;
+        let generic_type_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+
+        for (field_idx, field_def) in generic_type_place
+            .layout()
+            .ty
+            .ty_adt_def()
+            .unwrap()
+            .non_enum_variant()
+            .fields
+            .iter_enumerated()
+        {
+            let field_place = self.project_field(&generic_type_place, field_idx)?;
+            match field_def.name {
+                sym::ty => self.write_type_id(ty, &field_place)?,
+                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
+            }
+        }
+
+        self.write_discriminant(variant_idx, &place)?;
+        interp_ok(())
+    }
+
+    fn write_generic_const(&mut self, c: Const<'tcx>, place: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
+        let ConstKind::Value(c) = c.kind() else { bug!("expected a computed const, got {c:?}") };
+
+        let (variant_idx, variant_place) = self.downcast(&place, sym::Const)?;
+        let const_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+
+        for (field_idx, field_def) in const_place
+            .layout()
+            .ty
+            .ty_adt_def()
+            .unwrap()
+            .non_enum_variant()
+            .fields
+            .iter_enumerated()
+        {
+            let field_place = self.project_field(&const_place, field_idx)?;
+            match field_def.name {
+                sym::ty => self.write_type_id(c.ty, &field_place)?,
+                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
+            }
+        }
+
+        self.write_discriminant(variant_idx, &place)?;
+        interp_ok(())
+    }
+
     fn write_int_type_info(
         &mut self,
         place: impl Writeable<'tcx, CtfeProvenance>,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 271124a4c372..70f5c1d0c472 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -211,6 +211,7 @@ symbols! {
         CoercePointeeValidated,
         CoerceUnsized,
         Command,
+        Const,
         ConstParamTy,
         ConstParamTy_,
         Context,
@@ -292,6 +293,7 @@ symbols! {
         IteratorMap,
         Layout,
         Left,
+        Lifetime,
         LinkedList,
         LintDiagnostic,
         LintPass,
@@ -396,6 +398,7 @@ symbols! {
         Ty,
         TyCtxt,
         TyKind,
+        Type,
         Unknown,
         Unsize,
         UnsizedConstParamTy,
@@ -1176,6 +1179,7 @@ symbols! {
         generic_const_parameter_types,
         generic_param_attrs,
         generic_pattern_types,
+        generics,
         get_context,
         global_alloc_ty,
         global_allocator,
diff --git a/library/core/src/mem/type_info.rs b/library/core/src/mem/type_info.rs
index 0e87b9d9cbb7..8b11e5bd7035 100644
--- a/library/core/src/mem/type_info.rs
+++ b/library/core/src/mem/type_info.rs
@@ -146,12 +146,53 @@ pub struct Trait {
 #[non_exhaustive]
 #[unstable(feature = "type_info", issue = "146922")]
 pub struct Struct {
+    /// Instantiated generics of the struct.
+    pub generics: &'static [Generic],
     /// All fields of the struct.
     pub fields: &'static [Field],
     /// Whether the struct field list is non-exhaustive.
     pub non_exhaustive: bool,
 }
 
+/// Compile-time type information about instantiated generics of structs, enum and union variants.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub enum Generic {
+    /// Lifetimes.
+    Lifetime(Lifetime),
+    /// Types.
+    Type(GenericType),
+    /// Const parameters.
+    Const(Const),
+}
+
+/// Compile-time type information about generic lifetimes.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct Lifetime {
+    // No additional information to provide for now.
+}
+
+/// Compile-time type information about instantiated generic types.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct GenericType {
+    /// The type itself.
+    pub ty: TypeId,
+}
+
+/// Compile-time type information about generic const parameters.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct Const {
+    /// The const's type.
+    pub ty: TypeId,
+}
+
 /// Compile-time type information about `bool`.
 #[derive(Debug)]
 #[non_exhaustive]
diff --git a/library/coretests/tests/mem/type_info.rs b/library/coretests/tests/mem/type_info.rs
index 03ff5f55c4f7..9daf31029af6 100644
--- a/library/coretests/tests/mem/type_info.rs
+++ b/library/coretests/tests/mem/type_info.rs
@@ -2,7 +2,7 @@
 
 use std::any::{Any, TypeId};
 use std::mem::offset_of;
-use std::mem::type_info::{Type, TypeKind};
+use std::mem::type_info::{Const, Generic, GenericType, Type, TypeKind};
 
 #[test]
 fn test_arrays() {
@@ -115,6 +115,24 @@ fn test_structs() {
         assert!(ty.fields[1].name == "1");
         assert!(ty.fields[1].ty == TypeId::of::());
     }
+
+    const {
+        struct Generics<'a, T, const C: u64> {
+            a: &'a T,
+        }
+
+        let Type { kind: Struct(ty), .. } = Type::of::>() else {
+            panic!()
+        };
+        assert!(ty.fields.len() == 1);
+        assert!(ty.generics.len() == 3);
+
+        let Generic::Lifetime(_) = ty.generics[0] else { panic!() };
+        let Generic::Type(GenericType { ty: generic_ty, .. }) = ty.generics[1] else { panic!() };
+        assert!(generic_ty == TypeId::of::());
+        let Generic::Const(Const { ty: const_ty, .. }) = ty.generics[2] else { panic!() };
+        assert!(const_ty == TypeId::of::());
+    }
 }
 
 #[test]
diff --git a/tests/ui/generics/wrong-number-of-args.rs b/tests/ui/generics/wrong-number-of-args.rs
index 8bc384a3d817..9af16567cd75 100644
--- a/tests/ui/generics/wrong-number-of-args.rs
+++ b/tests/ui/generics/wrong-number-of-args.rs
@@ -127,7 +127,7 @@ mod r#trait {
     //~| HELP remove
 
     type D = Box;
-    //~^ ERROR missing generics for trait `GenericType`
+    //~^ ERROR missing generics for trait `r#trait::GenericType`
     //~| HELP add missing
 
     type E = Box>;
diff --git a/tests/ui/generics/wrong-number-of-args.stderr b/tests/ui/generics/wrong-number-of-args.stderr
index bedeeb812fc9..554d017d67e3 100644
--- a/tests/ui/generics/wrong-number-of-args.stderr
+++ b/tests/ui/generics/wrong-number-of-args.stderr
@@ -469,7 +469,7 @@ note: trait defined here, with 1 lifetime parameter: `'a`
 LL |     trait GenericLifetime<'a> {
    |           ^^^^^^^^^^^^^^^ --
 
-error[E0107]: missing generics for trait `GenericType`
+error[E0107]: missing generics for trait `r#trait::GenericType`
   --> $DIR/wrong-number-of-args.rs:129:22
    |
 LL |     type D = Box;
diff --git a/tests/ui/reflection/dump.bit32.run.stdout b/tests/ui/reflection/dump.bit32.run.stdout
index feda3401e462..e80cea8ece2f 100644
--- a/tests/ui/reflection/dump.bit32.run.stdout
+++ b/tests/ui/reflection/dump.bit32.run.stdout
@@ -148,6 +148,7 @@ Type {
 Type {
     kind: Struct(
         Struct {
+            generics: [],
             fields: [
                 Field {
                     name: "a",
@@ -171,6 +172,7 @@ Type {
 Type {
     kind: Struct(
         Struct {
+            generics: [],
             fields: [
                 Field {
                     name: "a",
@@ -188,6 +190,7 @@ Type {
 Type {
     kind: Struct(
         Struct {
+            generics: [],
             fields: [
                 Field {
                     name: "0",
@@ -207,6 +210,53 @@ Type {
         12,
     ),
 }
+Type {
+    kind: Struct(
+        Struct {
+            generics: [
+                Lifetime(
+                    Lifetime,
+                ),
+                Type(
+                    GenericType {
+                        ty: TypeId(0x56ced5e4a15bd89050bb9674fa2df013),
+                    },
+                ),
+                Type(
+                    GenericType {
+                        ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    },
+                ),
+                Const(
+                    Const {
+                        ty: TypeId(0x9ed91be891e304132cb86891e578f4a5),
+                    },
+                ),
+            ],
+            fields: [
+                Field {
+                    name: "a",
+                    ty: TypeId(0x56ced5e4a15bd89050bb9674fa2df013),
+                    offset: 4,
+                },
+                Field {
+                    name: "b",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 8,
+                },
+                Field {
+                    name: "l",
+                    ty: TypeId(0x5d686ae9be5f6232dca1f88c0b941fd9),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
+    size: Some(
+        12,
+    ),
+}
 Type {
     kind: Reference(
         Reference {
diff --git a/tests/ui/reflection/dump.bit64.run.stdout b/tests/ui/reflection/dump.bit64.run.stdout
index d9fef71f619c..812a918e519d 100644
--- a/tests/ui/reflection/dump.bit64.run.stdout
+++ b/tests/ui/reflection/dump.bit64.run.stdout
@@ -148,6 +148,7 @@ Type {
 Type {
     kind: Struct(
         Struct {
+            generics: [],
             fields: [
                 Field {
                     name: "a",
@@ -171,6 +172,7 @@ Type {
 Type {
     kind: Struct(
         Struct {
+            generics: [],
             fields: [
                 Field {
                     name: "a",
@@ -188,6 +190,7 @@ Type {
 Type {
     kind: Struct(
         Struct {
+            generics: [],
             fields: [
                 Field {
                     name: "0",
@@ -207,6 +210,53 @@ Type {
         16,
     ),
 }
+Type {
+    kind: Struct(
+        Struct {
+            generics: [
+                Lifetime(
+                    Lifetime,
+                ),
+                Type(
+                    GenericType {
+                        ty: TypeId(0x56ced5e4a15bd89050bb9674fa2df013),
+                    },
+                ),
+                Type(
+                    GenericType {
+                        ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    },
+                ),
+                Const(
+                    Const {
+                        ty: TypeId(0x9ed91be891e304132cb86891e578f4a5),
+                    },
+                ),
+            ],
+            fields: [
+                Field {
+                    name: "a",
+                    ty: TypeId(0x56ced5e4a15bd89050bb9674fa2df013),
+                    offset: 8,
+                },
+                Field {
+                    name: "b",
+                    ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                    offset: 12,
+                },
+                Field {
+                    name: "l",
+                    ty: TypeId(0x5d686ae9be5f6232dca1f88c0b941fd9),
+                    offset: 0,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
+    size: Some(
+        16,
+    ),
+}
 Type {
     kind: Reference(
         Reference {
diff --git a/tests/ui/reflection/dump.rs b/tests/ui/reflection/dump.rs
index fefaad325aea..d0da0582fb51 100644
--- a/tests/ui/reflection/dump.rs
+++ b/tests/ui/reflection/dump.rs
@@ -27,6 +27,12 @@ enum Bar {
     Foomp { a: (), b: &'static str },
 }
 
+struct Generics<'a, A, B, const C: u64> {
+    a: A,
+    b: B,
+    l: &'a (),
+}
+
 struct Unsized {
     x: u16,
     s: str,
@@ -44,7 +50,7 @@ fn main() {
         [u8; 2],
         i8, i32, i64, i128, isize,
         u8, u32, u64, u128, usize,
-        Foo, Bar, NonExhaustiveStruct, TupleStruct,
+        Foo, Bar, NonExhaustiveStruct, TupleStruct, Generics,
         &Unsized, &str, &[u8],
         str, [u8],
         &u8, &mut u8,
diff --git a/tests/ui/resolve/resolve-assoc-suggestions.stderr b/tests/ui/resolve/resolve-assoc-suggestions.stderr
index 7d94fb5ca35f..e6311962884f 100644
--- a/tests/ui/resolve/resolve-assoc-suggestions.stderr
+++ b/tests/ui/resolve/resolve-assoc-suggestions.stderr
@@ -41,12 +41,22 @@ error[E0531]: cannot find tuple struct or tuple variant `Type` in this scope
    |
 LL |         let Type(..);
    |             ^^^^ not found in this scope
+   |
+help: consider importing this tuple variant
+   |
+LL + use std::mem::type_info::Generic::Type;
+   |
 
 error[E0425]: cannot find value `Type` in this scope
   --> $DIR/resolve-assoc-suggestions.rs:27:9
    |
 LL |         Type;
    |         ^^^^ not found in this scope
+   |
+help: consider importing this tuple variant
+   |
+LL + use std::mem::type_info::Generic::Type;
+   |
 
 error[E0425]: cannot find type `method` in this scope
   --> $DIR/resolve-assoc-suggestions.rs:30:16

From e9037882c17d7ebd017c7403c934a5a5ef2d8488 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Tue, 20 Jan 2026 08:46:33 +0100
Subject: [PATCH 1554/1843] Support enums in type info reflection

---
 .../src/const_eval/type_info.rs               | 142 ++++++++++++++----
 compiler/rustc_span/src/symbol.rs             |   2 +
 library/core/src/mem/type_info.rs             |  28 ++++
 library/coretests/tests/mem/type_info.rs      |  42 ++++++
 tests/ui/privacy/issue-79593.rs               |   6 +-
 tests/ui/privacy/issue-79593.stderr           |   6 +-
 tests/ui/reflection/dump.bit32.run.stdout     |  41 ++++-
 tests/ui/reflection/dump.bit64.run.stdout     |  41 ++++-
 tests/ui/reflection/dump.rs                   |   6 +-
 9 files changed, 279 insertions(+), 35 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 9fb01b1f5c5c..701adc7f732b 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -120,15 +120,11 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Adt(adt_def, generics) => {
-                            // TODO(type_info): Handle enum and union
-                            if !adt_def.is_struct() {
+                            // TODO(type_info): Handle union
+                            if !adt_def.is_struct() && !adt_def.is_enum() {
                                 self.downcast(&field_dest, sym::Other)?.0
                             } else {
-                                let (variant, variant_place) =
-                                    self.downcast(&field_dest, sym::Struct)?;
-                                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
-                                self.write_adt_type_info(place, (ty, *adt_def), generics)?;
-                                variant
+                                self.write_adt_type_info(&field_dest, (ty, *adt_def), generics)?
                             }
                         }
                         ty::Bool => {
@@ -277,6 +273,25 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         self.write_immediate(ptr, &fields_slice_place)
     }
 
+    // Write fields for struct, enum variants
+    fn write_variant_fields(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        variant_def: &'tcx VariantDef,
+        variant_layout: TyAndLayout<'tcx>,
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        self.allocate_fill_and_write_slice_ptr(
+            place,
+            variant_def.fields.len() as u64,
+            |this, i, place| {
+                let field_def = &variant_def.fields[FieldIdx::from_usize(i as usize)];
+                let field_ty = field_def.ty(*this.tcx, generics);
+                this.write_field(field_ty, place, variant_layout, Some(field_def.name), i)
+            },
+        )
+    }
+
     fn write_field(
         &mut self,
         field_ty: Ty<'tcx>,
@@ -363,20 +378,31 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
     // FIXME(type_info): No semver considerations for now
     pub(crate) fn write_adt_type_info(
         &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
+        place: &impl Writeable<'tcx, CtfeProvenance>,
         adt: (Ty<'tcx>, AdtDef<'tcx>),
         generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, VariantIdx> {
         let (adt_ty, adt_def) = adt;
-        match adt_def.adt_kind() {
-            AdtKind::Struct => self.write_struct_type_info(
-                place,
-                (adt_ty, adt_def.variant(VariantIdx::ZERO)),
-                generics,
-            ),
+        let variant_idx = match adt_def.adt_kind() {
+            AdtKind::Struct => {
+                let (variant, variant_place) = self.downcast(place, sym::Struct)?;
+                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                self.write_struct_type_info(
+                    place,
+                    (adt_ty, adt_def.variant(VariantIdx::ZERO)),
+                    generics,
+                )?;
+                variant
+            }
+            AdtKind::Enum => {
+                let (variant, variant_place) = self.downcast(place, sym::Enum)?;
+                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                self.write_enum_type_info(place, adt, generics)?;
+                variant
+            }
             AdtKind::Union => todo!(),
-            AdtKind::Enum => todo!(),
-        }
+        };
+        interp_ok(variant_idx)
     }
 
     pub(crate) fn write_struct_type_info(
@@ -395,15 +421,9 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
 
             match field.name {
                 sym::generics => self.write_generics(field_place, generics)?,
-                sym::fields => self.allocate_fill_and_write_slice_ptr(
-                    field_place,
-                    struct_def.fields.len() as u64,
-                    |this, i, place| {
-                        let field_def = &struct_def.fields[FieldIdx::from_usize(i as usize)];
-                        let field_ty = field_def.ty(*this.tcx, generics);
-                        this.write_field(field_ty, place, struct_layout, Some(field_def.name), i)
-                    },
-                )?,
+                sym::fields => {
+                    self.write_variant_fields(field_place, struct_def, struct_layout, generics)?
+                }
                 sym::non_exhaustive => {
                     let is_non_exhaustive = struct_def.is_field_list_non_exhaustive();
                     self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
@@ -415,6 +435,76 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
+    pub(crate) fn write_enum_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        enum_: (Ty<'tcx>, AdtDef<'tcx>),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (enum_ty, enum_def) = enum_;
+        let enum_layout = self.layout_of(enum_ty)?;
+
+        for (field_idx, field) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+
+            match field.name {
+                sym::generics => self.write_generics(field_place, generics)?,
+                sym::variants => {
+                    self.allocate_fill_and_write_slice_ptr(
+                        field_place,
+                        enum_def.variants().len() as u64,
+                        |this, i, place| {
+                            let variant_idx = VariantIdx::from_usize(i as usize);
+                            let variant_def = &enum_def.variants()[variant_idx];
+                            let variant_layout = enum_layout.for_variant(this, variant_idx);
+                            this.write_enum_variant(place, (variant_layout, &variant_def), generics)
+                        },
+                    )?;
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = enum_def.is_variant_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
+            }
+        }
+
+        interp_ok(())
+    }
+
+    fn write_enum_variant(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        variant: (TyAndLayout<'tcx>, &'tcx VariantDef),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (variant_layout, variant_def) = variant;
+
+        for (field_idx, field_def) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+            match field_def.name {
+                sym::name => {
+                    let name_place = self.allocate_str_dedup(variant_def.name.as_str())?;
+                    let ptr = self.mplace_to_ref(&name_place)?;
+                    self.write_immediate(*ptr, &field_place)?
+                }
+                sym::fields => {
+                    self.write_variant_fields(field_place, &variant_def, variant_layout, generics)?
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = variant_def.is_field_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
+            }
+        }
+        interp_ok(())
+    }
+
     fn write_generics(
         &mut self,
         place: impl Writeable<'tcx, CtfeProvenance>,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 70f5c1d0c472..a3576b9a4437 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -235,6 +235,7 @@ symbols! {
         DynTrait,
         Encodable,
         Encoder,
+        Enum,
         Enumerate,
         Eq,
         Equal,
@@ -2480,6 +2481,7 @@ symbols! {
         values,
         var,
         variant_count,
+        variants,
         vec,
         vec_as_mut_slice,
         vec_as_slice,
diff --git a/library/core/src/mem/type_info.rs b/library/core/src/mem/type_info.rs
index 8b11e5bd7035..b66a836c4bd4 100644
--- a/library/core/src/mem/type_info.rs
+++ b/library/core/src/mem/type_info.rs
@@ -51,6 +51,8 @@ pub enum TypeKind {
     DynTrait(DynTrait),
     /// Structs.
     Struct(Struct),
+    /// Enums.
+    Enum(Enum),
     /// Primitive boolean type.
     Bool(Bool),
     /// Primitive character type.
@@ -154,6 +156,32 @@ pub struct Struct {
     pub non_exhaustive: bool,
 }
 
+/// Compile-time type information about enums.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct Enum {
+    /// Instantiated generics of the enum.
+    pub generics: &'static [Generic],
+    /// All variants of the enum.
+    pub variants: &'static [Variant],
+    /// Whether the enum variant list is non-exhaustive.
+    pub non_exhaustive: bool,
+}
+
+/// Compile-time type information about variants of enums.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct Variant {
+    /// The name of the variant.
+    pub name: &'static str,
+    /// All fields of the variant.
+    pub fields: &'static [Field],
+    /// Whether the enum variant fields is non-exhaustive.
+    pub non_exhaustive: bool,
+}
+
 /// Compile-time type information about instantiated generics of structs, enum and union variants.
 #[derive(Debug)]
 #[non_exhaustive]
diff --git a/library/coretests/tests/mem/type_info.rs b/library/coretests/tests/mem/type_info.rs
index 9daf31029af6..09e3a50d374c 100644
--- a/library/coretests/tests/mem/type_info.rs
+++ b/library/coretests/tests/mem/type_info.rs
@@ -135,6 +135,48 @@ fn test_structs() {
     }
 }
 
+#[test]
+fn test_enums() {
+    use TypeKind::*;
+
+    const {
+        enum E {
+            Some(u32),
+            None,
+            #[non_exhaustive]
+            Foomp {
+                a: (),
+                b: &'static str,
+            },
+        }
+
+        let Type { kind: Enum(ty), size, .. } = Type::of::() else { panic!() };
+        assert!(size == Some(size_of::()));
+        assert!(ty.variants.len() == 3);
+
+        assert!(ty.variants[0].name == "Some");
+        assert!(!ty.variants[0].non_exhaustive);
+        assert!(ty.variants[0].fields.len() == 1);
+
+        assert!(ty.variants[1].name == "None");
+        assert!(!ty.variants[1].non_exhaustive);
+        assert!(ty.variants[1].fields.len() == 0);
+
+        assert!(ty.variants[2].name == "Foomp");
+        assert!(ty.variants[2].non_exhaustive);
+        assert!(ty.variants[2].fields.len() == 2);
+    }
+
+    const {
+        let Type { kind: Enum(ty), size, .. } = Type::of::>() else { panic!() };
+        assert!(size == Some(size_of::>()));
+        assert!(ty.variants.len() == 2);
+        assert!(ty.generics.len() == 1);
+        let Generic::Type(GenericType { ty: generic_ty, .. }) = ty.generics[0] else { panic!() };
+        assert!(generic_ty == TypeId::of::());
+    }
+}
+
 #[test]
 fn test_primitives() {
     use TypeKind::*;
diff --git a/tests/ui/privacy/issue-79593.rs b/tests/ui/privacy/issue-79593.rs
index 39c222f7c341..663dd67fcdb1 100644
--- a/tests/ui/privacy/issue-79593.rs
+++ b/tests/ui/privacy/issue-79593.rs
@@ -10,7 +10,7 @@ mod foo {
         Pub {};
         //~^ ERROR missing field `private` in initializer of `Pub`
         Enum::Variant { x: () };
-        //~^ ERROR missing field `y` in initializer of `Enum`
+        //~^ ERROR missing field `y` in initializer of `foo::Enum`
     }
 }
 
@@ -21,9 +21,9 @@ fn correct() {
 
 fn wrong() {
     foo::Enum::Variant { x: () };
-    //~^ ERROR missing field `y` in initializer of `Enum`
+    //~^ ERROR missing field `y` in initializer of `foo::Enum`
     foo::Enum::Variant { };
-    //~^ ERROR missing fields `x` and `y` in initializer of `Enum`
+    //~^ ERROR missing fields `x` and `y` in initializer of `foo::Enum`
 }
 
 fn main() {}
diff --git a/tests/ui/privacy/issue-79593.stderr b/tests/ui/privacy/issue-79593.stderr
index 5bb69836f609..74fea1a3ab76 100644
--- a/tests/ui/privacy/issue-79593.stderr
+++ b/tests/ui/privacy/issue-79593.stderr
@@ -4,7 +4,7 @@ error[E0063]: missing field `private` in initializer of `Pub`
 LL |         Pub {};
    |         ^^^ missing `private`
 
-error[E0063]: missing field `y` in initializer of `Enum`
+error[E0063]: missing field `y` in initializer of `foo::Enum`
   --> $DIR/issue-79593.rs:12:9
    |
 LL |         Enum::Variant { x: () };
@@ -18,13 +18,13 @@ LL |     foo::Pub {};
    |
    = note: private field `private` that was not provided
 
-error[E0063]: missing field `y` in initializer of `Enum`
+error[E0063]: missing field `y` in initializer of `foo::Enum`
   --> $DIR/issue-79593.rs:23:5
    |
 LL |     foo::Enum::Variant { x: () };
    |     ^^^^^^^^^^^^^^^^^^ missing `y`
 
-error[E0063]: missing fields `x` and `y` in initializer of `Enum`
+error[E0063]: missing fields `x` and `y` in initializer of `foo::Enum`
   --> $DIR/issue-79593.rs:25:5
    |
 LL |     foo::Enum::Variant { };
diff --git a/tests/ui/reflection/dump.bit32.run.stdout b/tests/ui/reflection/dump.bit32.run.stdout
index e80cea8ece2f..a6ad84f5fef1 100644
--- a/tests/ui/reflection/dump.bit32.run.stdout
+++ b/tests/ui/reflection/dump.bit32.run.stdout
@@ -164,7 +164,46 @@ Type {
     ),
 }
 Type {
-    kind: Other,
+    kind: Enum(
+        Enum {
+            generics: [],
+            variants: [
+                Variant {
+                    name: "Some",
+                    fields: [
+                        Field {
+                            name: "0",
+                            ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                            offset: 4,
+                        },
+                    ],
+                    non_exhaustive: false,
+                },
+                Variant {
+                    name: "None",
+                    fields: [],
+                    non_exhaustive: false,
+                },
+                Variant {
+                    name: "Foomp",
+                    fields: [
+                        Field {
+                            name: "a",
+                            ty: TypeId(0x41223169ff28813ba79b7268a2a968d9),
+                            offset: 4,
+                        },
+                        Field {
+                            name: "b",
+                            ty: TypeId(0xb98b1b7157a6417863eb502cd6cb5d6d),
+                            offset: 4,
+                        },
+                    ],
+                    non_exhaustive: true,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
     size: Some(
         12,
     ),
diff --git a/tests/ui/reflection/dump.bit64.run.stdout b/tests/ui/reflection/dump.bit64.run.stdout
index 812a918e519d..45f0a2e2f063 100644
--- a/tests/ui/reflection/dump.bit64.run.stdout
+++ b/tests/ui/reflection/dump.bit64.run.stdout
@@ -164,7 +164,46 @@ Type {
     ),
 }
 Type {
-    kind: Other,
+    kind: Enum(
+        Enum {
+            generics: [],
+            variants: [
+                Variant {
+                    name: "Some",
+                    fields: [
+                        Field {
+                            name: "0",
+                            ty: TypeId(0x1378bb1c0a0202683eb65e7c11f2e4d7),
+                            offset: 4,
+                        },
+                    ],
+                    non_exhaustive: false,
+                },
+                Variant {
+                    name: "None",
+                    fields: [],
+                    non_exhaustive: false,
+                },
+                Variant {
+                    name: "Foomp",
+                    fields: [
+                        Field {
+                            name: "a",
+                            ty: TypeId(0x41223169ff28813ba79b7268a2a968d9),
+                            offset: 4,
+                        },
+                        Field {
+                            name: "b",
+                            ty: TypeId(0xb98b1b7157a6417863eb502cd6cb5d6d),
+                            offset: 8,
+                        },
+                    ],
+                    non_exhaustive: true,
+                },
+            ],
+            non_exhaustive: false,
+        },
+    ),
     size: Some(
         24,
     ),
diff --git a/tests/ui/reflection/dump.rs b/tests/ui/reflection/dump.rs
index d0da0582fb51..755f0c900e33 100644
--- a/tests/ui/reflection/dump.rs
+++ b/tests/ui/reflection/dump.rs
@@ -24,7 +24,11 @@ struct TupleStruct(u32, u64);
 enum Bar {
     Some(u32),
     None,
-    Foomp { a: (), b: &'static str },
+    #[non_exhaustive]
+    Foomp {
+        a: (),
+        b: &'static str,
+    },
 }
 
 struct Generics<'a, A, B, const C: u64> {

From 98e0c34f7f5ba6d68a6e3506d1569939fbf14627 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Thu, 5 Feb 2026 19:28:55 +0100
Subject: [PATCH 1555/1843] Support unions in type info reflection

---
 .../src/const_eval/type_info.rs               | 48 ++++++++++++++++---
 compiler/rustc_span/src/symbol.rs             |  1 +
 library/core/src/mem/type_info.rs             | 13 +++++
 library/coretests/tests/mem/type_info.rs      | 41 ++++++++++++++++
 tests/ui/reflection/dump.bit32.run.stdout     | 22 +++++++++
 tests/ui/reflection/dump.bit64.run.stdout     | 22 +++++++++
 tests/ui/reflection/dump.rs                   |  7 ++-
 7 files changed, 146 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 701adc7f732b..0529d452626b 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -120,12 +120,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                             variant
                         }
                         ty::Adt(adt_def, generics) => {
-                            // TODO(type_info): Handle union
-                            if !adt_def.is_struct() && !adt_def.is_enum() {
-                                self.downcast(&field_dest, sym::Other)?.0
-                            } else {
-                                self.write_adt_type_info(&field_dest, (ty, *adt_def), generics)?
-                            }
+                            self.write_adt_type_info(&field_dest, (ty, *adt_def), generics)?
                         }
                         ty::Bool => {
                             let (variant, _variant_place) =
@@ -394,13 +389,22 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                 )?;
                 variant
             }
+            AdtKind::Union => {
+                let (variant, variant_place) = self.downcast(place, sym::Union)?;
+                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                self.write_union_type_info(
+                    place,
+                    (adt_ty, adt_def.variant(VariantIdx::ZERO)),
+                    generics,
+                )?;
+                variant
+            }
             AdtKind::Enum => {
                 let (variant, variant_place) = self.downcast(place, sym::Enum)?;
                 let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
                 self.write_enum_type_info(place, adt, generics)?;
                 variant
             }
-            AdtKind::Union => todo!(),
         };
         interp_ok(variant_idx)
     }
@@ -435,6 +439,36 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
+    pub(crate) fn write_union_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        union_: (Ty<'tcx>, &'tcx VariantDef),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (union_ty, union_def) = union_;
+        let union_layout = self.layout_of(union_ty)?;
+
+        for (field_idx, field) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+
+            match field.name {
+                sym::generics => self.write_generics(field_place, generics)?,
+                sym::fields => {
+                    self.write_variant_fields(field_place, union_def, union_layout, generics)?
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = union_def.is_field_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
+            }
+        }
+
+        interp_ok(())
+    }
+
     pub(crate) fn write_enum_type_info(
         &mut self,
         place: impl Writeable<'tcx, CtfeProvenance>,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index a3576b9a4437..1a099f0cd70f 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -400,6 +400,7 @@ symbols! {
         TyCtxt,
         TyKind,
         Type,
+        Union,
         Unknown,
         Unsize,
         UnsizedConstParamTy,
diff --git a/library/core/src/mem/type_info.rs b/library/core/src/mem/type_info.rs
index b66a836c4bd4..c2b2cf2f270d 100644
--- a/library/core/src/mem/type_info.rs
+++ b/library/core/src/mem/type_info.rs
@@ -53,6 +53,8 @@ pub enum TypeKind {
     Struct(Struct),
     /// Enums.
     Enum(Enum),
+    /// Unions.
+    Union(Union),
     /// Primitive boolean type.
     Bool(Bool),
     /// Primitive character type.
@@ -156,6 +158,17 @@ pub struct Struct {
     pub non_exhaustive: bool,
 }
 
+/// Compile-time type information about unions.
+#[derive(Debug)]
+#[non_exhaustive]
+#[unstable(feature = "type_info", issue = "146922")]
+pub struct Union {
+    /// Instantiated generics of the union.
+    pub generics: &'static [Generic],
+    /// All fields of the union.
+    pub fields: &'static [Field],
+}
+
 /// Compile-time type information about enums.
 #[derive(Debug)]
 #[non_exhaustive]
diff --git a/library/coretests/tests/mem/type_info.rs b/library/coretests/tests/mem/type_info.rs
index 09e3a50d374c..808ef68783af 100644
--- a/library/coretests/tests/mem/type_info.rs
+++ b/library/coretests/tests/mem/type_info.rs
@@ -135,6 +135,47 @@ fn test_structs() {
     }
 }
 
+#[test]
+fn test_unions() {
+    use TypeKind::*;
+
+    const {
+        union TestUnion {
+            first: i16,
+            second: u16,
+        }
+
+        let Type { kind: Union(ty), size, .. } = Type::of::() else { panic!() };
+        assert!(size == Some(size_of::()));
+        assert!(ty.fields.len() == 2);
+        assert!(ty.fields[0].name == "first");
+        assert!(ty.fields[0].offset == offset_of!(TestUnion, first));
+        assert!(ty.fields[1].name == "second");
+        assert!(ty.fields[1].offset == offset_of!(TestUnion, second));
+    }
+
+    const {
+        union Generics<'a, T: Copy, const C: u64> {
+            a: T,
+            z: &'a (),
+        }
+
+        let Type { kind: Union(ty), .. } = Type::of::>() else {
+            panic!()
+        };
+        assert!(ty.fields.len() == 2);
+        assert!(ty.fields[0].offset == offset_of!(Generics<'static, i32, 1_u64>, a));
+        assert!(ty.fields[1].offset == offset_of!(Generics<'static, i32, 1_u64>, z));
+
+        assert!(ty.generics.len() == 3);
+        let Generic::Lifetime(_) = ty.generics[0] else { panic!() };
+        let Generic::Type(GenericType { ty: generic_ty, .. }) = ty.generics[1] else { panic!() };
+        assert!(generic_ty == TypeId::of::());
+        let Generic::Const(Const { ty: const_ty, .. }) = ty.generics[2] else { panic!() };
+        assert!(const_ty == TypeId::of::());
+    }
+}
+
 #[test]
 fn test_enums() {
     use TypeKind::*;
diff --git a/tests/ui/reflection/dump.bit32.run.stdout b/tests/ui/reflection/dump.bit32.run.stdout
index a6ad84f5fef1..24a467775725 100644
--- a/tests/ui/reflection/dump.bit32.run.stdout
+++ b/tests/ui/reflection/dump.bit32.run.stdout
@@ -296,6 +296,28 @@ Type {
         12,
     ),
 }
+Type {
+    kind: Union(
+        Union {
+            generics: [],
+            fields: [
+                Field {
+                    name: "first",
+                    ty: TypeId(0xdeb66dd04fa9db03298cc77926096aae),
+                    offset: 0,
+                },
+                Field {
+                    name: "second",
+                    ty: TypeId(0xc50c4a8d8e150aa67101203f1fab1cd7),
+                    offset: 0,
+                },
+            ],
+        },
+    ),
+    size: Some(
+        2,
+    ),
+}
 Type {
     kind: Reference(
         Reference {
diff --git a/tests/ui/reflection/dump.bit64.run.stdout b/tests/ui/reflection/dump.bit64.run.stdout
index 45f0a2e2f063..15f40d944173 100644
--- a/tests/ui/reflection/dump.bit64.run.stdout
+++ b/tests/ui/reflection/dump.bit64.run.stdout
@@ -296,6 +296,28 @@ Type {
         16,
     ),
 }
+Type {
+    kind: Union(
+        Union {
+            generics: [],
+            fields: [
+                Field {
+                    name: "first",
+                    ty: TypeId(0xdeb66dd04fa9db03298cc77926096aae),
+                    offset: 0,
+                },
+                Field {
+                    name: "second",
+                    ty: TypeId(0xc50c4a8d8e150aa67101203f1fab1cd7),
+                    offset: 0,
+                },
+            ],
+        },
+    ),
+    size: Some(
+        2,
+    ),
+}
 Type {
     kind: Reference(
         Reference {
diff --git a/tests/ui/reflection/dump.rs b/tests/ui/reflection/dump.rs
index 755f0c900e33..57669ca85f30 100644
--- a/tests/ui/reflection/dump.rs
+++ b/tests/ui/reflection/dump.rs
@@ -42,6 +42,11 @@ struct Unsized {
     s: str,
 }
 
+union Union {
+    first: i16,
+    second: u16,
+}
+
 macro_rules! dump_types {
     ($($ty:ty),+ $(,)?) => {
         $(println!("{:#?}", const { Type::of::<$ty>() });)+
@@ -54,7 +59,7 @@ fn main() {
         [u8; 2],
         i8, i32, i64, i128, isize,
         u8, u32, u64, u128, usize,
-        Foo, Bar, NonExhaustiveStruct, TupleStruct, Generics,
+        Foo, Bar, NonExhaustiveStruct, TupleStruct, Generics, Union,
         &Unsized, &str, &[u8],
         str, [u8],
         &u8, &mut u8,

From b410cb01fe86d992a97f6316a7265716d0e3f851 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Sat, 7 Feb 2026 19:03:34 +0100
Subject: [PATCH 1556/1843] Simplify the writing of tuple type info

---
 .../src/const_eval/type_info.rs               | 56 +++++++------------
 1 file changed, 19 insertions(+), 37 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 0529d452626b..97206e138697 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -98,7 +98,7 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                                     .fields
                                     .len()
                             );
-                            self.write_tuple_fields(tuple_place, fields, ty)?;
+                            self.write_tuple_type_info(tuple_place, fields, ty)?;
                             variant
                         }
                         ty::Array(ty, len) => {
@@ -232,42 +232,6 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
-    // TODO(type_info): Remove this method, use `allocate_fill_and_write_slice_ptr` as it's more general.
-    pub(crate) fn write_tuple_fields(
-        &mut self,
-        tuple_place: impl Writeable<'tcx, CtfeProvenance>,
-        fields: &[Ty<'tcx>],
-        tuple_ty: Ty<'tcx>,
-    ) -> InterpResult<'tcx> {
-        // project into the `type_info::Tuple::fields` field
-        let fields_slice_place = self.project_field(&tuple_place, FieldIdx::ZERO)?;
-        // get the `type_info::Field` type from `fields: &[Field]`
-        let field_type = fields_slice_place
-            .layout()
-            .ty
-            .builtin_deref(false)
-            .unwrap()
-            .sequence_element_type(self.tcx.tcx);
-        // Create an array with as many elements as the number of fields in the inspected tuple
-        let fields_layout =
-            self.layout_of(Ty::new_array(self.tcx.tcx, field_type, fields.len() as u64))?;
-        let fields_place = self.allocate(fields_layout, MemoryKind::Stack)?;
-        let mut fields_places = self.project_array_fields(&fields_place)?;
-
-        let tuple_layout = self.layout_of(tuple_ty)?;
-
-        while let Some((i, place)) = fields_places.next(self)? {
-            let field_ty = fields[i as usize];
-            self.write_field(field_ty, place, tuple_layout, None, i)?;
-        }
-
-        let fields_place = fields_place.map_provenance(CtfeProvenance::as_immutable);
-
-        let ptr = Immediate::new_slice(fields_place.ptr(), fields.len() as u64, self);
-
-        self.write_immediate(ptr, &fields_slice_place)
-    }
-
     // Write fields for struct, enum variants
     fn write_variant_fields(
         &mut self,
@@ -325,6 +289,24 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
+    pub(crate) fn write_tuple_type_info(
+        &mut self,
+        tuple_place: impl Writeable<'tcx, CtfeProvenance>,
+        fields: &[Ty<'tcx>],
+        tuple_ty: Ty<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let tuple_layout = self.layout_of(tuple_ty)?;
+        let fields_slice_place = self.project_field(&tuple_place, FieldIdx::ZERO)?;
+        self.allocate_fill_and_write_slice_ptr(
+            fields_slice_place,
+            fields.len() as u64,
+            |this, i, place| {
+                let field_ty = fields[i as usize];
+                this.write_field(field_ty, place, tuple_layout, None, i)
+            },
+        )
+    }
+
     pub(crate) fn write_array_type_info(
         &mut self,
         place: impl Writeable<'tcx, CtfeProvenance>,

From a575fe168f2293a9c8c2945f5c5795eaec2f4d34 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Tue, 10 Feb 2026 01:28:35 +0100
Subject: [PATCH 1557/1843] Erase type lifetime before writing type ID

---
 compiler/rustc_const_eval/src/const_eval/type_info.rs | 5 ++++-
 compiler/rustc_const_eval/src/interpret/intrinsics.rs | 7 ++++++-
 library/coretests/tests/mem/type_info.rs              | 2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 97206e138697..7741027981d0 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -273,7 +273,10 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
                     let ptr = self.mplace_to_ref(&name_place)?;
                     self.write_immediate(*ptr, &field_place)?
                 }
-                sym::ty => self.write_type_id(field_ty, &field_place)?,
+                sym::ty => {
+                    let field_ty = self.tcx.erase_and_anonymize_regions(field_ty);
+                    self.write_type_id(field_ty, &field_place)?
+                }
                 sym::offset => {
                     let offset = layout.fields.offset(idx as usize);
                     self.write_scalar(
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index 2ea5e4a25c11..49038315b546 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -13,7 +13,7 @@ use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
 use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
 use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{FloatTy, PolyExistentialPredicate, Ty, TyCtxt};
+use rustc_middle::ty::{FloatTy, PolyExistentialPredicate, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::{bug, span_bug, ty};
 use rustc_span::{Symbol, sym};
 use rustc_trait_selection::traits::{Obligation, ObligationCause, ObligationCtxt};
@@ -73,6 +73,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         ty: Ty<'tcx>,
         dest: &impl Writeable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()> {
+        debug_assert!(
+            !ty.has_erasable_regions(),
+            "type {ty:?} has regions that need erasing before writing a TypeId",
+        );
+
         let tcx = self.tcx;
         let type_id_hash = tcx.type_id_hash(ty).as_u128();
         let op = self.const_val_to_op(
diff --git a/library/coretests/tests/mem/type_info.rs b/library/coretests/tests/mem/type_info.rs
index 808ef68783af..2483b4c2aacd 100644
--- a/library/coretests/tests/mem/type_info.rs
+++ b/library/coretests/tests/mem/type_info.rs
@@ -91,7 +91,7 @@ fn test_structs() {
         assert!(ty.fields[1].ty == TypeId::of::());
         assert!(ty.fields[1].offset == offset_of!(TestStruct, second));
         assert!(ty.fields[2].name == "reference");
-        assert!(ty.fields[2].ty != TypeId::of::<&'static u16>()); // FIXME(type_info): should be ==
+        assert!(ty.fields[2].ty == TypeId::of::<&'static u16>());
         assert!(ty.fields[2].offset == offset_of!(TestStruct, reference));
     }
 

From 6ab6734d4badb8fb8956873df1c3e2e2389a42f4 Mon Sep 17 00:00:00 2001
From: Asuna 
Date: Tue, 10 Feb 2026 02:09:07 +0100
Subject: [PATCH 1558/1843] Move ADT related code to a sub module for type info

---
 .../src/const_eval/type_info.rs               | 271 +----------------
 .../src/const_eval/type_info/adt.rs           | 276 ++++++++++++++++++
 2 files changed, 280 insertions(+), 267 deletions(-)
 create mode 100644 compiler/rustc_const_eval/src/const_eval/type_info/adt.rs

diff --git a/compiler/rustc_const_eval/src/const_eval/type_info.rs b/compiler/rustc_const_eval/src/const_eval/type_info.rs
index 7741027981d0..0fd70d784d4f 100644
--- a/compiler/rustc_const_eval/src/const_eval/type_info.rs
+++ b/compiler/rustc_const_eval/src/const_eval/type_info.rs
@@ -1,14 +1,13 @@
+mod adt;
+
 use std::borrow::Cow;
 
 use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_ast::Mutability;
 use rustc_hir::LangItem;
+use rustc_middle::span_bug;
 use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{
-    self, AdtDef, AdtKind, Const, ConstKind, GenericArgKind, GenericArgs, Region, ScalarInt, Ty,
-    VariantDef,
-};
-use rustc_middle::{bug, span_bug};
+use rustc_middle::ty::{self, Const, ScalarInt, Ty};
 use rustc_span::{Symbol, sym};
 
 use crate::const_eval::CompileTimeMachine;
@@ -232,25 +231,6 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
-    // Write fields for struct, enum variants
-    fn write_variant_fields(
-        &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
-        variant_def: &'tcx VariantDef,
-        variant_layout: TyAndLayout<'tcx>,
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
-        self.allocate_fill_and_write_slice_ptr(
-            place,
-            variant_def.fields.len() as u64,
-            |this, i, place| {
-                let field_def = &variant_def.fields[FieldIdx::from_usize(i as usize)];
-                let field_ty = field_def.ty(*this.tcx, generics);
-                this.write_field(field_ty, place, variant_layout, Some(field_def.name), i)
-            },
-        )
-    }
-
     fn write_field(
         &mut self,
         field_ty: Ty<'tcx>,
@@ -355,249 +335,6 @@ impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
         interp_ok(())
     }
 
-    // FIXME(type_info): No semver considerations for now
-    pub(crate) fn write_adt_type_info(
-        &mut self,
-        place: &impl Writeable<'tcx, CtfeProvenance>,
-        adt: (Ty<'tcx>, AdtDef<'tcx>),
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx, VariantIdx> {
-        let (adt_ty, adt_def) = adt;
-        let variant_idx = match adt_def.adt_kind() {
-            AdtKind::Struct => {
-                let (variant, variant_place) = self.downcast(place, sym::Struct)?;
-                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
-                self.write_struct_type_info(
-                    place,
-                    (adt_ty, adt_def.variant(VariantIdx::ZERO)),
-                    generics,
-                )?;
-                variant
-            }
-            AdtKind::Union => {
-                let (variant, variant_place) = self.downcast(place, sym::Union)?;
-                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
-                self.write_union_type_info(
-                    place,
-                    (adt_ty, adt_def.variant(VariantIdx::ZERO)),
-                    generics,
-                )?;
-                variant
-            }
-            AdtKind::Enum => {
-                let (variant, variant_place) = self.downcast(place, sym::Enum)?;
-                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
-                self.write_enum_type_info(place, adt, generics)?;
-                variant
-            }
-        };
-        interp_ok(variant_idx)
-    }
-
-    pub(crate) fn write_struct_type_info(
-        &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
-        struct_: (Ty<'tcx>, &'tcx VariantDef),
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
-        let (struct_ty, struct_def) = struct_;
-        let struct_layout = self.layout_of(struct_ty)?;
-
-        for (field_idx, field) in
-            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
-        {
-            let field_place = self.project_field(&place, field_idx)?;
-
-            match field.name {
-                sym::generics => self.write_generics(field_place, generics)?,
-                sym::fields => {
-                    self.write_variant_fields(field_place, struct_def, struct_layout, generics)?
-                }
-                sym::non_exhaustive => {
-                    let is_non_exhaustive = struct_def.is_field_list_non_exhaustive();
-                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
-                }
-                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
-            }
-        }
-
-        interp_ok(())
-    }
-
-    pub(crate) fn write_union_type_info(
-        &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
-        union_: (Ty<'tcx>, &'tcx VariantDef),
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
-        let (union_ty, union_def) = union_;
-        let union_layout = self.layout_of(union_ty)?;
-
-        for (field_idx, field) in
-            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
-        {
-            let field_place = self.project_field(&place, field_idx)?;
-
-            match field.name {
-                sym::generics => self.write_generics(field_place, generics)?,
-                sym::fields => {
-                    self.write_variant_fields(field_place, union_def, union_layout, generics)?
-                }
-                sym::non_exhaustive => {
-                    let is_non_exhaustive = union_def.is_field_list_non_exhaustive();
-                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
-                }
-                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
-            }
-        }
-
-        interp_ok(())
-    }
-
-    pub(crate) fn write_enum_type_info(
-        &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
-        enum_: (Ty<'tcx>, AdtDef<'tcx>),
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
-        let (enum_ty, enum_def) = enum_;
-        let enum_layout = self.layout_of(enum_ty)?;
-
-        for (field_idx, field) in
-            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
-        {
-            let field_place = self.project_field(&place, field_idx)?;
-
-            match field.name {
-                sym::generics => self.write_generics(field_place, generics)?,
-                sym::variants => {
-                    self.allocate_fill_and_write_slice_ptr(
-                        field_place,
-                        enum_def.variants().len() as u64,
-                        |this, i, place| {
-                            let variant_idx = VariantIdx::from_usize(i as usize);
-                            let variant_def = &enum_def.variants()[variant_idx];
-                            let variant_layout = enum_layout.for_variant(this, variant_idx);
-                            this.write_enum_variant(place, (variant_layout, &variant_def), generics)
-                        },
-                    )?;
-                }
-                sym::non_exhaustive => {
-                    let is_non_exhaustive = enum_def.is_variant_list_non_exhaustive();
-                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
-                }
-                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
-            }
-        }
-
-        interp_ok(())
-    }
-
-    fn write_enum_variant(
-        &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
-        variant: (TyAndLayout<'tcx>, &'tcx VariantDef),
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
-        let (variant_layout, variant_def) = variant;
-
-        for (field_idx, field_def) in
-            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
-        {
-            let field_place = self.project_field(&place, field_idx)?;
-            match field_def.name {
-                sym::name => {
-                    let name_place = self.allocate_str_dedup(variant_def.name.as_str())?;
-                    let ptr = self.mplace_to_ref(&name_place)?;
-                    self.write_immediate(*ptr, &field_place)?
-                }
-                sym::fields => {
-                    self.write_variant_fields(field_place, &variant_def, variant_layout, generics)?
-                }
-                sym::non_exhaustive => {
-                    let is_non_exhaustive = variant_def.is_field_list_non_exhaustive();
-                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
-                }
-                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
-            }
-        }
-        interp_ok(())
-    }
-
-    fn write_generics(
-        &mut self,
-        place: impl Writeable<'tcx, CtfeProvenance>,
-        generics: &'tcx GenericArgs<'tcx>,
-    ) -> InterpResult<'tcx> {
-        self.allocate_fill_and_write_slice_ptr(place, generics.len() as u64, |this, i, place| {
-            match generics[i as usize].kind() {
-                GenericArgKind::Lifetime(region) => this.write_generic_lifetime(region, place),
-                GenericArgKind::Type(ty) => this.write_generic_type(ty, place),
-                GenericArgKind::Const(c) => this.write_generic_const(c, place),
-            }
-        })
-    }
-
-    fn write_generic_lifetime(
-        &mut self,
-        _region: Region<'tcx>,
-        place: MPlaceTy<'tcx>,
-    ) -> InterpResult<'tcx> {
-        let (variant_idx, _) = self.downcast(&place, sym::Lifetime)?;
-        self.write_discriminant(variant_idx, &place)?;
-        interp_ok(())
-    }
-
-    fn write_generic_type(&mut self, ty: Ty<'tcx>, place: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
-        let (variant_idx, variant_place) = self.downcast(&place, sym::Type)?;
-        let generic_type_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
-
-        for (field_idx, field_def) in generic_type_place
-            .layout()
-            .ty
-            .ty_adt_def()
-            .unwrap()
-            .non_enum_variant()
-            .fields
-            .iter_enumerated()
-        {
-            let field_place = self.project_field(&generic_type_place, field_idx)?;
-            match field_def.name {
-                sym::ty => self.write_type_id(ty, &field_place)?,
-                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
-            }
-        }
-
-        self.write_discriminant(variant_idx, &place)?;
-        interp_ok(())
-    }
-
-    fn write_generic_const(&mut self, c: Const<'tcx>, place: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
-        let ConstKind::Value(c) = c.kind() else { bug!("expected a computed const, got {c:?}") };
-
-        let (variant_idx, variant_place) = self.downcast(&place, sym::Const)?;
-        let const_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
-
-        for (field_idx, field_def) in const_place
-            .layout()
-            .ty
-            .ty_adt_def()
-            .unwrap()
-            .non_enum_variant()
-            .fields
-            .iter_enumerated()
-        {
-            let field_place = self.project_field(&const_place, field_idx)?;
-            match field_def.name {
-                sym::ty => self.write_type_id(c.ty, &field_place)?,
-                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
-            }
-        }
-
-        self.write_discriminant(variant_idx, &place)?;
-        interp_ok(())
-    }
-
     fn write_int_type_info(
         &mut self,
         place: impl Writeable<'tcx, CtfeProvenance>,
diff --git a/compiler/rustc_const_eval/src/const_eval/type_info/adt.rs b/compiler/rustc_const_eval/src/const_eval/type_info/adt.rs
new file mode 100644
index 000000000000..60f7b95e799a
--- /dev/null
+++ b/compiler/rustc_const_eval/src/const_eval/type_info/adt.rs
@@ -0,0 +1,276 @@
+use rustc_abi::{FieldIdx, VariantIdx};
+use rustc_middle::ty::layout::TyAndLayout;
+use rustc_middle::ty::{
+    AdtDef, AdtKind, Const, ConstKind, GenericArgKind, GenericArgs, Region, Ty, VariantDef,
+};
+use rustc_middle::{bug, span_bug};
+use rustc_span::sym;
+
+use crate::const_eval::CompileTimeMachine;
+use crate::interpret::{
+    CtfeProvenance, InterpCx, InterpResult, MPlaceTy, Projectable, Scalar, Writeable, interp_ok,
+};
+
+impl<'tcx> InterpCx<'tcx, CompileTimeMachine<'tcx>> {
+    // FIXME(type_info): No semver considerations for now
+    pub(crate) fn write_adt_type_info(
+        &mut self,
+        place: &impl Writeable<'tcx, CtfeProvenance>,
+        adt: (Ty<'tcx>, AdtDef<'tcx>),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx, VariantIdx> {
+        let (adt_ty, adt_def) = adt;
+        let variant_idx = match adt_def.adt_kind() {
+            AdtKind::Struct => {
+                let (variant, variant_place) = self.downcast(place, sym::Struct)?;
+                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                self.write_struct_type_info(
+                    place,
+                    (adt_ty, adt_def.variant(VariantIdx::ZERO)),
+                    generics,
+                )?;
+                variant
+            }
+            AdtKind::Union => {
+                let (variant, variant_place) = self.downcast(place, sym::Union)?;
+                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                self.write_union_type_info(
+                    place,
+                    (adt_ty, adt_def.variant(VariantIdx::ZERO)),
+                    generics,
+                )?;
+                variant
+            }
+            AdtKind::Enum => {
+                let (variant, variant_place) = self.downcast(place, sym::Enum)?;
+                let place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+                self.write_enum_type_info(place, adt, generics)?;
+                variant
+            }
+        };
+        interp_ok(variant_idx)
+    }
+
+    pub(crate) fn write_struct_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        struct_: (Ty<'tcx>, &'tcx VariantDef),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (struct_ty, struct_def) = struct_;
+        let struct_layout = self.layout_of(struct_ty)?;
+
+        for (field_idx, field) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+
+            match field.name {
+                sym::generics => self.write_generics(field_place, generics)?,
+                sym::fields => {
+                    self.write_variant_fields(field_place, struct_def, struct_layout, generics)?
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = struct_def.is_field_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
+            }
+        }
+
+        interp_ok(())
+    }
+
+    pub(crate) fn write_union_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        union_: (Ty<'tcx>, &'tcx VariantDef),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (union_ty, union_def) = union_;
+        let union_layout = self.layout_of(union_ty)?;
+
+        for (field_idx, field) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+
+            match field.name {
+                sym::generics => self.write_generics(field_place, generics)?,
+                sym::fields => {
+                    self.write_variant_fields(field_place, union_def, union_layout, generics)?
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = union_def.is_field_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
+            }
+        }
+
+        interp_ok(())
+    }
+
+    pub(crate) fn write_enum_type_info(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        enum_: (Ty<'tcx>, AdtDef<'tcx>),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (enum_ty, enum_def) = enum_;
+        let enum_layout = self.layout_of(enum_ty)?;
+
+        for (field_idx, field) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+
+            match field.name {
+                sym::generics => self.write_generics(field_place, generics)?,
+                sym::variants => {
+                    self.allocate_fill_and_write_slice_ptr(
+                        field_place,
+                        enum_def.variants().len() as u64,
+                        |this, i, place| {
+                            let variant_idx = VariantIdx::from_usize(i as usize);
+                            let variant_def = &enum_def.variants()[variant_idx];
+                            let variant_layout = enum_layout.for_variant(this, variant_idx);
+                            this.write_enum_variant(place, (variant_layout, &variant_def), generics)
+                        },
+                    )?;
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = enum_def.is_variant_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field.did), "unimplemented field {other}"),
+            }
+        }
+
+        interp_ok(())
+    }
+
+    fn write_enum_variant(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        variant: (TyAndLayout<'tcx>, &'tcx VariantDef),
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (variant_layout, variant_def) = variant;
+
+        for (field_idx, field_def) in
+            place.layout().ty.ty_adt_def().unwrap().non_enum_variant().fields.iter_enumerated()
+        {
+            let field_place = self.project_field(&place, field_idx)?;
+            match field_def.name {
+                sym::name => {
+                    let name_place = self.allocate_str_dedup(variant_def.name.as_str())?;
+                    let ptr = self.mplace_to_ref(&name_place)?;
+                    self.write_immediate(*ptr, &field_place)?
+                }
+                sym::fields => {
+                    self.write_variant_fields(field_place, &variant_def, variant_layout, generics)?
+                }
+                sym::non_exhaustive => {
+                    let is_non_exhaustive = variant_def.is_field_list_non_exhaustive();
+                    self.write_scalar(Scalar::from_bool(is_non_exhaustive), &field_place)?
+                }
+                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
+            }
+        }
+        interp_ok(())
+    }
+
+    // Write fields for struct, enum variants
+    fn write_variant_fields(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        variant_def: &'tcx VariantDef,
+        variant_layout: TyAndLayout<'tcx>,
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        self.allocate_fill_and_write_slice_ptr(
+            place,
+            variant_def.fields.len() as u64,
+            |this, i, place| {
+                let field_def = &variant_def.fields[FieldIdx::from_usize(i as usize)];
+                let field_ty = field_def.ty(*this.tcx, generics);
+                this.write_field(field_ty, place, variant_layout, Some(field_def.name), i)
+            },
+        )
+    }
+
+    fn write_generics(
+        &mut self,
+        place: impl Writeable<'tcx, CtfeProvenance>,
+        generics: &'tcx GenericArgs<'tcx>,
+    ) -> InterpResult<'tcx> {
+        self.allocate_fill_and_write_slice_ptr(place, generics.len() as u64, |this, i, place| {
+            match generics[i as usize].kind() {
+                GenericArgKind::Lifetime(region) => this.write_generic_lifetime(region, place),
+                GenericArgKind::Type(ty) => this.write_generic_type(ty, place),
+                GenericArgKind::Const(c) => this.write_generic_const(c, place),
+            }
+        })
+    }
+
+    fn write_generic_lifetime(
+        &mut self,
+        _region: Region<'tcx>,
+        place: MPlaceTy<'tcx>,
+    ) -> InterpResult<'tcx> {
+        let (variant_idx, _) = self.downcast(&place, sym::Lifetime)?;
+        self.write_discriminant(variant_idx, &place)?;
+        interp_ok(())
+    }
+
+    fn write_generic_type(&mut self, ty: Ty<'tcx>, place: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
+        let (variant_idx, variant_place) = self.downcast(&place, sym::Type)?;
+        let generic_type_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+
+        for (field_idx, field_def) in generic_type_place
+            .layout()
+            .ty
+            .ty_adt_def()
+            .unwrap()
+            .non_enum_variant()
+            .fields
+            .iter_enumerated()
+        {
+            let field_place = self.project_field(&generic_type_place, field_idx)?;
+            match field_def.name {
+                sym::ty => self.write_type_id(ty, &field_place)?,
+                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
+            }
+        }
+
+        self.write_discriminant(variant_idx, &place)?;
+        interp_ok(())
+    }
+
+    fn write_generic_const(&mut self, c: Const<'tcx>, place: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
+        let ConstKind::Value(c) = c.kind() else { bug!("expected a computed const, got {c:?}") };
+
+        let (variant_idx, variant_place) = self.downcast(&place, sym::Const)?;
+        let const_place = self.project_field(&variant_place, FieldIdx::ZERO)?;
+
+        for (field_idx, field_def) in const_place
+            .layout()
+            .ty
+            .ty_adt_def()
+            .unwrap()
+            .non_enum_variant()
+            .fields
+            .iter_enumerated()
+        {
+            let field_place = self.project_field(&const_place, field_idx)?;
+            match field_def.name {
+                sym::ty => self.write_type_id(c.ty, &field_place)?,
+                other => span_bug!(self.tcx.def_span(field_def.did), "unimplemented field {other}"),
+            }
+        }
+
+        self.write_discriminant(variant_idx, &place)?;
+        interp_ok(())
+    }
+}

From 78dc744e93028188c66d92dc2064b2107dd9992c Mon Sep 17 00:00:00 2001
From: Lizan Zhou 
Date: Tue, 10 Feb 2026 22:20:02 +0900
Subject: [PATCH 1559/1843] unwind/wasm: fix compile error by wrapping
 wasm_throw in unsafe block

---
 library/unwind/src/wasm.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/unwind/src/wasm.rs b/library/unwind/src/wasm.rs
index 2bff306af293..cb6e90ba180b 100644
--- a/library/unwind/src/wasm.rs
+++ b/library/unwind/src/wasm.rs
@@ -73,7 +73,7 @@ pub unsafe fn _Unwind_RaiseException(exception: *mut _Unwind_Exception) -> _Unwi
             // corresponds with llvm::WebAssembly::Tag::CPP_EXCEPTION
             //     in llvm-project/llvm/include/llvm/CodeGen/WasmEHFuncInfo.h
             const CPP_EXCEPTION_TAG: i32 = 0;
-            wasm_throw(CPP_EXCEPTION_TAG, exception.cast())
+            unsafe { wasm_throw(CPP_EXCEPTION_TAG, exception.cast()) }
         }
         _ => {
             let _ = exception;

From 11e4873b96961d14202a8c63d83a21be5b7f7b82 Mon Sep 17 00:00:00 2001
From: ron 
Date: Tue, 10 Feb 2026 10:22:05 -0500
Subject: [PATCH 1560/1843] Fix typos and grammar in compiler and build
 documentation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- compiler/rustc_codegen_llvm/src/debuginfo/doc.md: fix 5 issues
  (adaption → adaptation, "allow to" → "allow V-ing" x3, later → latter + plural)
- compiler/rustc_thread_pool/src/sleep/README.md: fix 2 issues
  (idle → active per context, then → than)
- src/bootstrap/README.md: fix 2 issues (rust → Rust, add missing article "the")
- src/ci/docker/README.md: fix 2 issues ("come form" → "come from", grammar)
---
 compiler/rustc_codegen_llvm/src/debuginfo/doc.md | 10 +++++-----
 compiler/rustc_thread_pool/src/sleep/README.md   |  4 ++--
 src/bootstrap/README.md                          |  4 ++--
 src/ci/docker/README.md                          |  4 ++--
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md
index aaec4e68c174..f5ea5ea77c4b 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md
@@ -16,7 +16,7 @@ module to generate correct metadata and insert it into the LLVM IR.
 As the exact format of metadata trees may change between different LLVM
 versions, we now use LLVM
 [DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html)
-to create metadata where possible. This will hopefully ease the adaption of
+to create metadata where possible. This will hopefully ease the adaptation of
 this module to future LLVM versions.
 
 The public API of the module is a set of functions that will insert the
@@ -87,19 +87,19 @@ describe the type anew. This behavior is encapsulated in the
 ## Source Locations and Line Information
 
 In addition to data type descriptions the debugging information must also
-allow to map machine code locations back to source code locations in order
+allow mapping machine code locations back to source code locations in order
 to be useful. This functionality is also handled in this module. The
-following functions allow to control source mappings:
+following functions allow controlling source mappings:
 
 + `set_source_location()`
 + `clear_source_location()`
 + `start_emitting_source_locations()`
 
-`set_source_location()` allows to set the current source location. All IR
+`set_source_location()` allows setting the current source location. All IR
 instructions created after a call to this function will be linked to the
 given source location, until another location is specified with
 `set_source_location()` or the source location is cleared with
-`clear_source_location()`. In the later case, subsequent IR instruction
+`clear_source_location()`. In the latter case, subsequent IR instructions
 will not be linked to any source location. As you can see, this is a
 stateful API (mimicking the one in LLVM), so be careful with source
 locations set by previous calls. It's probably best to not rely on any
diff --git a/compiler/rustc_thread_pool/src/sleep/README.md b/compiler/rustc_thread_pool/src/sleep/README.md
index 1e11da55f4a4..948c2922a434 100644
--- a/compiler/rustc_thread_pool/src/sleep/README.md
+++ b/compiler/rustc_thread_pool/src/sleep/README.md
@@ -32,7 +32,7 @@ There are three main thread states:
   waiting to be awoken.
 
 We sometimes refer to the final two states collectively as **inactive**.
-Threads begin as idle but transition to idle and finally sleeping when
+Threads begin as active but transition to idle and finally sleeping when
 they're unable to find work to do.
 
 ## Sleepy threads
@@ -160,7 +160,7 @@ not complete. It is possible -- if unlikely -- that enough activity occurs for
 Thread A to observe the same JEC value that it saw when getting sleepy. If the
 new work being published came from *inside* the thread-pool, then this race
 condition isn't too harmful. It means that we have fewer workers processing the
-work then we should, but we won't deadlock. This seems like an acceptable risk
+work than we should, but we won't deadlock. This seems like an acceptable risk
 given that this is unlikely in practice.
 
 However, if the work was posted as an *external* job, that is a problem. In that
diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md
index e46fed7c426a..ef2de9b37ed0 100644
--- a/src/bootstrap/README.md
+++ b/src/bootstrap/README.md
@@ -175,8 +175,8 @@ When you use bootstrap, you'll call it through the entry point script
 (`x`, `x.ps1`, or `x.py`). However, most of the code lives in `src/bootstrap`.
 `bootstrap` has a difficult problem: it is written in Rust, but yet it is run
 before the Rust compiler is built! To work around this, there are two components
-of bootstrap: the main one written in rust, and `bootstrap.py`. `bootstrap.py`
-is what gets run by entry point script. It takes care of downloading the prebuilt
+of bootstrap: the main one written in Rust, and `bootstrap.py`. `bootstrap.py`
+is what gets run by the entry point script. It takes care of downloading the prebuilt
 stage 0 compiler, std and Cargo binaries, which are then used to build the
 bootstrap binary.
 
diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md
index 4ee02e9bf005..5d3a74edc995 100644
--- a/src/ci/docker/README.md
+++ b/src/ci/docker/README.md
@@ -48,7 +48,7 @@ Refer to the [dev guide](https://rustc-dev-guide.rust-lang.org/tests/docker.html
 ## Docker Toolbox on Windows
 
 For Windows before Windows 10, the docker images can be run on Windows via
-[Docker Toolbox]. There are several preparation needs to be made before running
+[Docker Toolbox]. There are several preparations that need to be made before running
 a Docker image.
 
 1. Stop the virtual machine from the terminal with `docker-machine stop`
@@ -92,7 +92,7 @@ To run the image,
 A number of these images take quite a long time to compile as they're building
 whole gcc toolchains to do cross builds with. Much of this is relatively
 self-explanatory but some images use [crosstool-ng] which isn't quite as self
-explanatory. Below is a description of where these `*.defconfig` files come form,
+explanatory. Below is a description of where these `*.defconfig` files come from,
 how to generate them, and how the existing ones were generated.
 
 [crosstool-ng]: https://github.com/crosstool-ng/crosstool-ng

From da80677d6a84c1a3ea3e22785392c91ccae6a69b Mon Sep 17 00:00:00 2001
From: ron 
Date: Tue, 10 Feb 2026 10:22:19 -0500
Subject: [PATCH 1561/1843] Fix typos and grammar in top-level and src/doc
 documentation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- CONTRIBUTING.md: add missing verb "is"
- INSTALL.md: fix subject-verb agreement ("requires" → "require")
- RELEASES.md: fix 4 issues (previous → previously, remove extra "is",
  add hyphen in Rust-for-Linux, results → result)
- src/doc/not_found.md: fix misspelling ("Standary" → "Standard")
- src/doc/index.md: fix awkward grammar in Embedded Rust Book description
---
 CONTRIBUTING.md      | 2 +-
 INSTALL.md           | 2 +-
 RELEASES.md          | 8 ++++----
 src/doc/index.md     | 5 ++---
 src/doc/not_found.md | 2 +-
 5 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2b5699dcd098..3f052eca48e6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -10,7 +10,7 @@ the Zulip stream is the best place to *ask* for help.
 
 Documentation for contributing to the compiler or tooling is located in the [Guide to Rustc
 Development][rustc-dev-guide], commonly known as the [rustc-dev-guide]. Documentation for the
-standard library in the [Standard library developers Guide][std-dev-guide], commonly known as the [std-dev-guide].
+standard library is in the [Standard library developers Guide][std-dev-guide], commonly known as the [std-dev-guide].
 
 ## Making changes to subtrees and submodules
 
diff --git a/INSTALL.md b/INSTALL.md
index dd6c64b3df5f..0e998101fcd1 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -233,7 +233,7 @@ itself back on after some time).
 
 ### MSVC
 
-MSVC builds of Rust additionally requires an installation of:
+MSVC builds of Rust additionally require an installation of:
 
 - Visual Studio 2022 (or later) build tools so `rustc` can use its linker. Older
   Visual Studio versions such as 2019 *may* work but aren't actively tested.
diff --git a/RELEASES.md b/RELEASES.md
index 424e12ceec05..3c0a95f8462b 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -1546,7 +1546,7 @@ Compatibility Notes
 - [Check well-formedness of the source type's signature in fn pointer casts.](https://github.com/rust-lang/rust/pull/129021) This partly closes a soundness hole that comes when casting a function item to function pointer
 - [Use equality instead of subtyping when resolving type dependent paths.](https://github.com/rust-lang/rust/pull/129073)
 - Linking on macOS now correctly includes Rust's default deployment target. Due to a linker bug, you might have to pass `MACOSX_DEPLOYMENT_TARGET` or fix your `#[link]` attributes to point to the correct frameworks. See .
-- [Rust will now correctly raise an error for `repr(Rust)` written on non-`struct`/`enum`/`union` items, since it previous did not have any effect.](https://github.com/rust-lang/rust/pull/129422)
+- [Rust will now correctly raise an error for `repr(Rust)` written on non-`struct`/`enum`/`union` items, since it previously did not have any effect.](https://github.com/rust-lang/rust/pull/129422)
 - The future incompatibility lint `deprecated_cfg_attr_crate_type_name` [has been made into a hard error](https://github.com/rust-lang/rust/pull/129670). It was used to deny usage of `#![crate_type]` and `#![crate_name]` attributes in `#![cfg_attr]`, which required a hack in the compiler to be able to change the used crate type and crate name after cfg expansion.
   Users can use `--crate-type` instead of `#![cfg_attr(..., crate_type = "...")]` and `--crate-name` instead of `#![cfg_attr(..., crate_name = "...")]` when running `rustc`/`cargo rustc` on the command line.
   Use of those two attributes outside of `#![cfg_attr]` continue to be fully supported.
@@ -1722,7 +1722,7 @@ Cargo
 Compatibility Notes
 -------------------
  - We now [disallow setting some built-in cfgs via the command-line](https://github.com/rust-lang/rust/pull/126158) with the newly added [`explicit_builtin_cfgs_in_flags`](https://doc.rust-lang.org/rustc/lints/listing/deny-by-default.html#explicit-builtin-cfgs-in-flags) lint in order to prevent incoherent state, eg. `windows` cfg active but target is Linux based. The appropriate [`rustc` flag](https://doc.rust-lang.org/rustc/command-line-arguments.html) should be used instead.
-- The standard library has a new implementation of `binary_search` which is significantly improves performance ([#128254](https://github.com/rust-lang/rust/pull/128254)). However when a sorted slice has multiple values which compare equal, the new implementation may select a different value among the equal ones than the old implementation.
+- The standard library has a new implementation of `binary_search` which significantly improves performance ([#128254](https://github.com/rust-lang/rust/pull/128254)). However when a sorted slice has multiple values which compare equal, the new implementation may select a different value among the equal ones than the old implementation.
 - [illumos/Solaris now sets `MSG_NOSIGNAL` when writing to sockets](https://github.com/rust-lang/rust/pull/128259). This avoids killing the process with SIGPIPE when writing to a closed socket, which matches the existing behavior on other UNIX targets.
 - [Removes a problematic hack that always passed the --whole-archive linker flag for tests, which may cause linker errors for code accidentally relying on it.](https://github.com/rust-lang/rust/pull/128400)
 - The WebAssembly target features `multivalue` and `reference-types` are now
@@ -1872,7 +1872,7 @@ These changes do not affect any public interfaces of Rust, but they represent
 significant improvements to the performance or internals of rustc and related
 tools.
 
-- [Add a Rust-for Linux `auto` CI job to check kernel builds.](https://github.com/rust-lang/rust/pull/125209/)
+- [Add a Rust-for-Linux `auto` CI job to check kernel builds.](https://github.com/rust-lang/rust/pull/125209/)
 
 Version 1.80.1 (2024-08-08)
 ===========================
@@ -4510,7 +4510,7 @@ Compatibility Notes
   saturating to `0` instead][89926]. In the real world the panic happened mostly
   on platforms with buggy monotonic clock implementations rather than catching
   programming errors like reversing the start and end times. Such programming
-  errors will now results in `0` rather than a panic.
+  errors will now result in `0` rather than a panic.
 - In a future release we're planning to increase the baseline requirements for
   the Linux kernel to version 3.2, and for glibc to version 2.17. We'd love
   your feedback in [PR #95026][95026].
diff --git a/src/doc/index.md b/src/doc/index.md
index 892057a8f4db..418939353975 100644
--- a/src/doc/index.md
+++ b/src/doc/index.md
@@ -194,9 +194,8 @@ resources maintained by the [Embedded Working Group] useful.
 
 #### The Embedded Rust Book
 
-[The Embedded Rust Book] is targeted at developers familiar with embedded
-development and familiar with Rust, but have not used Rust for embedded
-development.
+[The Embedded Rust Book] is targeted at developers who are familiar with embedded
+development and Rust, but who have not used Rust for embedded development.
 
 [The Embedded Rust Book]: embedded-book/index.html
 [Rust project]: https://www.rust-lang.org
diff --git a/src/doc/not_found.md b/src/doc/not_found.md
index 9552759d2b8b..69e6ae3e2d03 100644
--- a/src/doc/not_found.md
+++ b/src/doc/not_found.md
@@ -41,7 +41,7 @@ Some things that might be helpful to you though: